diff options
Diffstat (limited to 'lib/ansible/plugins')
247 files changed, 29830 insertions, 0 deletions
diff --git a/lib/ansible/plugins/__init__.py b/lib/ansible/plugins/__init__.py new file mode 100644 index 0000000..4d1f3b1 --- /dev/null +++ b/lib/ansible/plugins/__init__.py @@ -0,0 +1,143 @@ +# (c) 2012, Daniel Hokka Zakrisson <daniel@hozac.com> +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> and others +# (c) 2017, Toshio Kuratomi <tkuratomi@ansible.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from abc import ABC + +import types +import typing as t + +from ansible import constants as C +from ansible.errors import AnsibleError +from ansible.module_utils._text import to_native +from ansible.module_utils.six import string_types +from ansible.utils.display import Display + +display = Display() + +if t.TYPE_CHECKING: + from .loader import PluginPathContext + +# Global so that all instances of a PluginLoader will share the caches +MODULE_CACHE = {} # type: dict[str, dict[str, types.ModuleType]] +PATH_CACHE = {} # type: dict[str, list[PluginPathContext] | None] +PLUGIN_PATH_CACHE = {} # type: dict[str, dict[str, dict[str, PluginPathContext]]] + + +def get_plugin_class(obj): + if isinstance(obj, string_types): + return obj.lower().replace('module', '') + else: + return obj.__class__.__name__.lower().replace('module', '') + + +class AnsiblePlugin(ABC): + + # allow extra passthrough parameters + allow_extras = False + + def __init__(self): + self._options = {} + self._defs = None + + def matches_name(self, possible_names): + possible_fqcns = set() + for name in possible_names: + if '.' not in name: + possible_fqcns.add(f"ansible.builtin.{name}") + elif name.startswith("ansible.legacy."): + possible_fqcns.add(name.removeprefix("ansible.legacy.")) + possible_fqcns.add(name) + return bool(possible_fqcns.intersection(set(self.ansible_aliases))) + + def get_option(self, option, hostvars=None): + if option not in self._options: + try: + option_value = C.config.get_config_value(option, plugin_type=self.plugin_type, plugin_name=self._load_name, variables=hostvars) + except AnsibleError as e: + raise KeyError(to_native(e)) + self.set_option(option, option_value) + return self._options.get(option) + + def get_options(self, hostvars=None): + options = {} + for option in self.option_definitions.keys(): + options[option] = self.get_option(option, hostvars=hostvars) + return options + + def set_option(self, option, value): + self._options[option] = value + + def set_options(self, task_keys=None, var_options=None, direct=None): + ''' + Sets the _options attribute with the configuration/keyword information for this plugin + + :arg task_keys: Dict with playbook keywords that affect this option + :arg var_options: Dict with either 'connection variables' + :arg direct: Dict with 'direct assignment' + ''' + self._options = C.config.get_plugin_options(self.plugin_type, self._load_name, keys=task_keys, variables=var_options, direct=direct) + + # allow extras/wildcards from vars that are not directly consumed in configuration + # this is needed to support things like winrm that can have extended protocol options we don't directly handle + if self.allow_extras and var_options and '_extras' in var_options: + self.set_option('_extras', var_options['_extras']) + + def has_option(self, option): + if not self._options: + self.set_options() + return option in self._options + + @property + def plugin_type(self): + return self.__class__.__name__.lower().replace('module', '') + + @property + def option_definitions(self): + if self._defs is None: + self._defs = C.config.get_configuration_definitions(plugin_type=self.plugin_type, name=self._load_name) + return self._defs + + def _check_required(self): + # FIXME: standardize required check based on config + pass + + +class AnsibleJinja2Plugin(AnsiblePlugin): + + def __init__(self, function): + + super(AnsibleJinja2Plugin, self).__init__() + self._function = function + + @property + def plugin_type(self): + return self.__class__.__name__.lower().replace('ansiblejinja2', '') + + def _no_options(self, *args, **kwargs): + raise NotImplementedError() + + has_option = get_option = get_options = option_definitions = set_option = set_options = _no_options + + @property + def j2_function(self): + return self._function diff --git a/lib/ansible/plugins/action/__init__.py b/lib/ansible/plugins/action/__init__.py new file mode 100644 index 0000000..d199207 --- /dev/null +++ b/lib/ansible/plugins/action/__init__.py @@ -0,0 +1,1431 @@ +# coding: utf-8 +# Copyright: (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# Copyright: (c) 2018, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import base64 +import json +import os +import random +import re +import shlex +import stat +import tempfile + +from abc import ABC, abstractmethod +from collections.abc import Sequence + +from ansible import constants as C +from ansible.errors import AnsibleError, AnsibleConnectionFailure, AnsibleActionSkip, AnsibleActionFail, AnsibleAuthenticationFailure +from ansible.executor.module_common import modify_module +from ansible.executor.interpreter_discovery import discover_interpreter, InterpreterDiscoveryRequiredError +from ansible.module_utils.common.arg_spec import ArgumentSpecValidator +from ansible.module_utils.errors import UnsupportedError +from ansible.module_utils.json_utils import _filter_non_json_lines +from ansible.module_utils.six import binary_type, string_types, text_type +from ansible.module_utils._text import to_bytes, to_native, to_text +from ansible.parsing.utils.jsonify import jsonify +from ansible.release import __version__ +from ansible.utils.collection_loader import resource_from_fqcr +from ansible.utils.display import Display +from ansible.utils.unsafe_proxy import wrap_var, AnsibleUnsafeText +from ansible.vars.clean import remove_internal_keys +from ansible.utils.plugin_docs import get_versioned_doclink + +display = Display() + + +class ActionBase(ABC): + + ''' + This class is the base class for all action plugins, and defines + code common to all actions. The base class handles the connection + by putting/getting files and executing commands based on the current + action in use. + ''' + + # A set of valid arguments + _VALID_ARGS = frozenset([]) # type: frozenset[str] + + def __init__(self, task, connection, play_context, loader, templar, shared_loader_obj): + self._task = task + self._connection = connection + self._play_context = play_context + self._loader = loader + self._templar = templar + self._shared_loader_obj = shared_loader_obj + self._cleanup_remote_tmp = False + + self._supports_check_mode = True + self._supports_async = False + + # interpreter discovery state + self._discovered_interpreter_key = None + self._discovered_interpreter = False + self._discovery_deprecation_warnings = [] + self._discovery_warnings = [] + + # Backwards compat: self._display isn't really needed, just import the global display and use that. + self._display = display + + self._used_interpreter = None + + @abstractmethod + def run(self, tmp=None, task_vars=None): + """ Action Plugins should implement this method to perform their + tasks. Everything else in this base class is a helper method for the + action plugin to do that. + + :kwarg tmp: Deprecated parameter. This is no longer used. An action plugin that calls + another one and wants to use the same remote tmp for both should set + self._connection._shell.tmpdir rather than this parameter. + :kwarg task_vars: The variables (host vars, group vars, config vars, + etc) associated with this task. + :returns: dictionary of results from the module + + Implementors of action modules may find the following variables especially useful: + + * Module parameters. These are stored in self._task.args + """ + + # does not default to {'changed': False, 'failed': False}, as it breaks async + result = {} + + if tmp is not None: + result['warning'] = ['ActionModule.run() no longer honors the tmp parameter. Action' + ' plugins should set self._connection._shell.tmpdir to share' + ' the tmpdir'] + del tmp + + if self._task.async_val and not self._supports_async: + raise AnsibleActionFail('async is not supported for this task.') + elif self._task.check_mode and not self._supports_check_mode: + raise AnsibleActionSkip('check mode is not supported for this task.') + elif self._task.async_val and self._task.check_mode: + raise AnsibleActionFail('check mode and async cannot be used on same task.') + + # Error if invalid argument is passed + if self._VALID_ARGS: + task_opts = frozenset(self._task.args.keys()) + bad_opts = task_opts.difference(self._VALID_ARGS) + if bad_opts: + raise AnsibleActionFail('Invalid options for %s: %s' % (self._task.action, ','.join(list(bad_opts)))) + + if self._connection._shell.tmpdir is None and self._early_needs_tmp_path(): + self._make_tmp_path() + + return result + + def validate_argument_spec(self, argument_spec=None, + mutually_exclusive=None, + required_together=None, + required_one_of=None, + required_if=None, + required_by=None, + ): + """Validate an argument spec against the task args + + This will return a tuple of (ValidationResult, dict) where the dict + is the validated, coerced, and normalized task args. + + Be cautious when directly passing ``new_module_args`` directly to a + module invocation, as it will contain the defaults, and not only + the args supplied from the task. If you do this, the module + should not define ``mututally_exclusive`` or similar. + + This code is roughly copied from the ``validate_argument_spec`` + action plugin for use by other action plugins. + """ + + new_module_args = self._task.args.copy() + + validator = ArgumentSpecValidator( + argument_spec, + mutually_exclusive=mutually_exclusive, + required_together=required_together, + required_one_of=required_one_of, + required_if=required_if, + required_by=required_by, + ) + validation_result = validator.validate(new_module_args) + + new_module_args.update(validation_result.validated_parameters) + + try: + error = validation_result.errors[0] + except IndexError: + error = None + + # Fail for validation errors, even in check mode + if error: + msg = validation_result.errors.msg + if isinstance(error, UnsupportedError): + msg = f"Unsupported parameters for ({self._load_name}) module: {msg}" + + raise AnsibleActionFail(msg) + + return validation_result, new_module_args + + def cleanup(self, force=False): + """Method to perform a clean up at the end of an action plugin execution + + By default this is designed to clean up the shell tmpdir, and is toggled based on whether + async is in use + + Action plugins may override this if they deem necessary, but should still call this method + via super + """ + if force or not self._task.async_val: + self._remove_tmp_path(self._connection._shell.tmpdir) + + def get_plugin_option(self, plugin, option, default=None): + """Helper to get an option from a plugin without having to use + the try/except dance everywhere to set a default + """ + try: + return plugin.get_option(option) + except (AttributeError, KeyError): + return default + + def get_become_option(self, option, default=None): + return self.get_plugin_option(self._connection.become, option, default=default) + + def get_connection_option(self, option, default=None): + return self.get_plugin_option(self._connection, option, default=default) + + def get_shell_option(self, option, default=None): + return self.get_plugin_option(self._connection._shell, option, default=default) + + def _remote_file_exists(self, path): + cmd = self._connection._shell.exists(path) + result = self._low_level_execute_command(cmd=cmd, sudoable=True) + if result['rc'] == 0: + return True + return False + + def _configure_module(self, module_name, module_args, task_vars): + ''' + Handles the loading and templating of the module code through the + modify_module() function. + ''' + if self._task.delegate_to: + use_vars = task_vars.get('ansible_delegated_vars')[self._task.delegate_to] + else: + use_vars = task_vars + + split_module_name = module_name.split('.') + collection_name = '.'.join(split_module_name[0:2]) if len(split_module_name) > 2 else '' + leaf_module_name = resource_from_fqcr(module_name) + + # Search module path(s) for named module. + for mod_type in self._connection.module_implementation_preferences: + # Check to determine if PowerShell modules are supported, and apply + # some fixes (hacks) to module name + args. + if mod_type == '.ps1': + # FIXME: This should be temporary and moved to an exec subsystem plugin where we can define the mapping + # for each subsystem. + win_collection = 'ansible.windows' + rewrite_collection_names = ['ansible.builtin', 'ansible.legacy', ''] + # async_status, win_stat, win_file, win_copy, and win_ping are not just like their + # python counterparts but they are compatible enough for our + # internal usage + # NB: we only rewrite the module if it's not being called by the user (eg, an action calling something else) + # and if it's unqualified or FQ to a builtin + if leaf_module_name in ('stat', 'file', 'copy', 'ping') and \ + collection_name in rewrite_collection_names and self._task.action != module_name: + module_name = '%s.win_%s' % (win_collection, leaf_module_name) + elif leaf_module_name == 'async_status' and collection_name in rewrite_collection_names: + module_name = '%s.%s' % (win_collection, leaf_module_name) + + # TODO: move this tweak down to the modules, not extensible here + # Remove extra quotes surrounding path parameters before sending to module. + if leaf_module_name in ['win_stat', 'win_file', 'win_copy', 'slurp'] and module_args and \ + hasattr(self._connection._shell, '_unquote'): + for key in ('src', 'dest', 'path'): + if key in module_args: + module_args[key] = self._connection._shell._unquote(module_args[key]) + + result = self._shared_loader_obj.module_loader.find_plugin_with_context(module_name, mod_type, collection_list=self._task.collections) + + if not result.resolved: + if result.redirect_list and len(result.redirect_list) > 1: + # take the last one in the redirect list, we may have successfully jumped through N other redirects + target_module_name = result.redirect_list[-1] + + raise AnsibleError("The module {0} was redirected to {1}, which could not be loaded.".format(module_name, target_module_name)) + + module_path = result.plugin_resolved_path + if module_path: + break + else: # This is a for-else: http://bit.ly/1ElPkyg + raise AnsibleError("The module %s was not found in configured module paths" % (module_name)) + + # insert shared code and arguments into the module + final_environment = dict() + self._compute_environment_string(final_environment) + + become_kwargs = {} + if self._connection.become: + become_kwargs['become'] = True + become_kwargs['become_method'] = self._connection.become.name + become_kwargs['become_user'] = self._connection.become.get_option('become_user', + playcontext=self._play_context) + become_kwargs['become_password'] = self._connection.become.get_option('become_pass', + playcontext=self._play_context) + become_kwargs['become_flags'] = self._connection.become.get_option('become_flags', + playcontext=self._play_context) + + # modify_module will exit early if interpreter discovery is required; re-run after if necessary + for dummy in (1, 2): + try: + (module_data, module_style, module_shebang) = modify_module(module_name, module_path, module_args, self._templar, + task_vars=use_vars, + module_compression=self._play_context.module_compression, + async_timeout=self._task.async_val, + environment=final_environment, + remote_is_local=bool(getattr(self._connection, '_remote_is_local', False)), + **become_kwargs) + break + except InterpreterDiscoveryRequiredError as idre: + self._discovered_interpreter = AnsibleUnsafeText(discover_interpreter( + action=self, + interpreter_name=idre.interpreter_name, + discovery_mode=idre.discovery_mode, + task_vars=use_vars)) + + # update the local task_vars with the discovered interpreter (which might be None); + # we'll propagate back to the controller in the task result + discovered_key = 'discovered_interpreter_%s' % idre.interpreter_name + + # update the local vars copy for the retry + use_vars['ansible_facts'][discovered_key] = self._discovered_interpreter + + # TODO: this condition prevents 'wrong host' from being updated + # but in future we would want to be able to update 'delegated host facts' + # irrespective of task settings + if not self._task.delegate_to or self._task.delegate_facts: + # store in local task_vars facts collection for the retry and any other usages in this worker + task_vars['ansible_facts'][discovered_key] = self._discovered_interpreter + # preserve this so _execute_module can propagate back to controller as a fact + self._discovered_interpreter_key = discovered_key + else: + task_vars['ansible_delegated_vars'][self._task.delegate_to]['ansible_facts'][discovered_key] = self._discovered_interpreter + + return (module_style, module_shebang, module_data, module_path) + + def _compute_environment_string(self, raw_environment_out=None): + ''' + Builds the environment string to be used when executing the remote task. + ''' + + final_environment = dict() + if self._task.environment is not None: + environments = self._task.environment + if not isinstance(environments, list): + environments = [environments] + + # The order of environments matters to make sure we merge + # in the parent's values first so those in the block then + # task 'win' in precedence + for environment in environments: + if environment is None or len(environment) == 0: + continue + temp_environment = self._templar.template(environment) + if not isinstance(temp_environment, dict): + raise AnsibleError("environment must be a dictionary, received %s (%s)" % (temp_environment, type(temp_environment))) + # very deliberately using update here instead of combine_vars, as + # these environment settings should not need to merge sub-dicts + final_environment.update(temp_environment) + + if len(final_environment) > 0: + final_environment = self._templar.template(final_environment) + + if isinstance(raw_environment_out, dict): + raw_environment_out.clear() + raw_environment_out.update(final_environment) + + return self._connection._shell.env_prefix(**final_environment) + + def _early_needs_tmp_path(self): + ''' + Determines if a tmp path should be created before the action is executed. + ''' + + return getattr(self, 'TRANSFERS_FILES', False) + + def _is_pipelining_enabled(self, module_style, wrap_async=False): + ''' + Determines if we are required and can do pipelining + ''' + + try: + is_enabled = self._connection.get_option('pipelining') + except (KeyError, AttributeError, ValueError): + is_enabled = self._play_context.pipelining + + # winrm supports async pipeline + # TODO: make other class property 'has_async_pipelining' to separate cases + always_pipeline = self._connection.always_pipeline_modules + + # su does not work with pipelining + # TODO: add has_pipelining class prop to become plugins + become_exception = (self._connection.become.name if self._connection.become else '') != 'su' + + # any of these require a true + conditions = [ + self._connection.has_pipelining, # connection class supports it + is_enabled or always_pipeline, # enabled via config or forced via connection (eg winrm) + module_style == "new", # old style modules do not support pipelining + not C.DEFAULT_KEEP_REMOTE_FILES, # user wants remote files + not wrap_async or always_pipeline, # async does not normally support pipelining unless it does (eg winrm) + become_exception, + ] + + return all(conditions) + + def _get_admin_users(self): + ''' + Returns a list of admin users that are configured for the current shell + plugin + ''' + + return self.get_shell_option('admin_users', ['root']) + + def _get_remote_addr(self, tvars): + ''' consistently get the 'remote_address' for the action plugin ''' + remote_addr = tvars.get('delegated_vars', {}).get('ansible_host', tvars.get('ansible_host', tvars.get('inventory_hostname', None))) + for variation in ('remote_addr', 'host'): + try: + remote_addr = self._connection.get_option(variation) + except KeyError: + continue + break + else: + # plugin does not have, fallback to play_context + remote_addr = self._play_context.remote_addr + return remote_addr + + def _get_remote_user(self): + ''' consistently get the 'remote_user' for the action plugin ''' + # TODO: use 'current user running ansible' as fallback when moving away from play_context + # pwd.getpwuid(os.getuid()).pw_name + remote_user = None + try: + remote_user = self._connection.get_option('remote_user') + except KeyError: + # plugin does not have remote_user option, fallback to default and/play_context + remote_user = getattr(self._connection, 'default_user', None) or self._play_context.remote_user + except AttributeError: + # plugin does not use config system, fallback to old play_context + remote_user = self._play_context.remote_user + return remote_user + + def _is_become_unprivileged(self): + ''' + The user is not the same as the connection user and is not part of the + shell configured admin users + ''' + # if we don't use become then we know we aren't switching to a + # different unprivileged user + if not self._connection.become: + return False + + # if we use become and the user is not an admin (or same user) then + # we need to return become_unprivileged as True + admin_users = self._get_admin_users() + remote_user = self._get_remote_user() + become_user = self.get_become_option('become_user') + return bool(become_user and become_user not in admin_users + [remote_user]) + + def _make_tmp_path(self, remote_user=None): + ''' + Create and return a temporary path on a remote box. + ''' + + # Network connection plugins (network_cli, netconf, etc.) execute on the controller, rather than the remote host. + # As such, we want to avoid using remote_user for paths as remote_user may not line up with the local user + # This is a hack and should be solved by more intelligent handling of remote_tmp in 2.7 + if getattr(self._connection, '_remote_is_local', False): + tmpdir = C.DEFAULT_LOCAL_TMP + else: + # NOTE: shell plugins should populate this setting anyways, but they dont do remote expansion, which + # we need for 'non posix' systems like cloud-init and solaris + tmpdir = self._remote_expand_user(self.get_shell_option('remote_tmp', default='~/.ansible/tmp'), sudoable=False) + + become_unprivileged = self._is_become_unprivileged() + basefile = self._connection._shell._generate_temp_dir_name() + cmd = self._connection._shell.mkdtemp(basefile=basefile, system=become_unprivileged, tmpdir=tmpdir) + result = self._low_level_execute_command(cmd, sudoable=False) + + # error handling on this seems a little aggressive? + if result['rc'] != 0: + if result['rc'] == 5: + output = 'Authentication failure.' + elif result['rc'] == 255 and self._connection.transport in ('ssh',): + + if display.verbosity > 3: + output = u'SSH encountered an unknown error. The output was:\n%s%s' % (result['stdout'], result['stderr']) + else: + output = (u'SSH encountered an unknown error during the connection. ' + 'We recommend you re-run the command using -vvvv, which will enable SSH debugging output to help diagnose the issue') + + elif u'No space left on device' in result['stderr']: + output = result['stderr'] + else: + output = ('Failed to create temporary directory. ' + 'In some cases, you may have been able to authenticate and did not have permissions on the target directory. ' + 'Consider changing the remote tmp path in ansible.cfg to a path rooted in "/tmp", for more error information use -vvv. ' + 'Failed command was: %s, exited with result %d' % (cmd, result['rc'])) + if 'stdout' in result and result['stdout'] != u'': + output = output + u", stdout output: %s" % result['stdout'] + if display.verbosity > 3 and 'stderr' in result and result['stderr'] != u'': + output += u", stderr output: %s" % result['stderr'] + raise AnsibleConnectionFailure(output) + else: + self._cleanup_remote_tmp = True + + try: + stdout_parts = result['stdout'].strip().split('%s=' % basefile, 1) + rc = self._connection._shell.join_path(stdout_parts[-1], u'').splitlines()[-1] + except IndexError: + # stdout was empty or just space, set to / to trigger error in next if + rc = '/' + + # Catch failure conditions, files should never be + # written to locations in /. + if rc == '/': + raise AnsibleError('failed to resolve remote temporary directory from %s: `%s` returned empty string' % (basefile, cmd)) + + self._connection._shell.tmpdir = rc + + return rc + + def _should_remove_tmp_path(self, tmp_path): + '''Determine if temporary path should be deleted or kept by user request/config''' + return tmp_path and self._cleanup_remote_tmp and not C.DEFAULT_KEEP_REMOTE_FILES and "-tmp-" in tmp_path + + def _remove_tmp_path(self, tmp_path, force=False): + '''Remove a temporary path we created. ''' + + if tmp_path is None and self._connection._shell.tmpdir: + tmp_path = self._connection._shell.tmpdir + + if force or self._should_remove_tmp_path(tmp_path): + cmd = self._connection._shell.remove(tmp_path, recurse=True) + # If we have gotten here we have a working connection configuration. + # If the connection breaks we could leave tmp directories out on the remote system. + tmp_rm_res = self._low_level_execute_command(cmd, sudoable=False) + + if tmp_rm_res.get('rc', 0) != 0: + display.warning('Error deleting remote temporary files (rc: %s, stderr: %s})' + % (tmp_rm_res.get('rc'), tmp_rm_res.get('stderr', 'No error string available.'))) + else: + self._connection._shell.tmpdir = None + + def _transfer_file(self, local_path, remote_path): + """ + Copy a file from the controller to a remote path + + :arg local_path: Path on controller to transfer + :arg remote_path: Path on the remote system to transfer into + + .. warning:: + * When you use this function you likely want to use use fixup_perms2() on the + remote_path to make sure that the remote file is readable when the user becomes + a non-privileged user. + * If you use fixup_perms2() on the file and copy or move the file into place, you will + need to then remove filesystem acls on the file once it has been copied into place by + the module. See how the copy module implements this for help. + """ + self._connection.put_file(local_path, remote_path) + return remote_path + + def _transfer_data(self, remote_path, data): + ''' + Copies the module data out to the temporary module path. + ''' + + if isinstance(data, dict): + data = jsonify(data) + + afd, afile = tempfile.mkstemp(dir=C.DEFAULT_LOCAL_TMP) + afo = os.fdopen(afd, 'wb') + try: + data = to_bytes(data, errors='surrogate_or_strict') + afo.write(data) + except Exception as e: + raise AnsibleError("failure writing module data to temporary file for transfer: %s" % to_native(e)) + + afo.flush() + afo.close() + + try: + self._transfer_file(afile, remote_path) + finally: + os.unlink(afile) + + return remote_path + + def _fixup_perms2(self, remote_paths, remote_user=None, execute=True): + """ + We need the files we upload to be readable (and sometimes executable) + by the user being sudo'd to but we want to limit other people's access + (because the files could contain passwords or other private + information. We achieve this in one of these ways: + + * If no sudo is performed or the remote_user is sudo'ing to + themselves, we don't have to change permissions. + * If the remote_user sudo's to a privileged user (for instance, root), + we don't have to change permissions + * If the remote_user sudo's to an unprivileged user then we attempt to + grant the unprivileged user access via file system acls. + * If granting file system acls fails we try to change the owner of the + file with chown which only works in case the remote_user is + privileged or the remote systems allows chown calls by unprivileged + users (e.g. HP-UX) + * If the above fails, we next try 'chmod +a' which is a macOS way of + setting ACLs on files. + * If the above fails, we check if ansible_common_remote_group is set. + If it is, we attempt to chgrp the file to its value. This is useful + if the remote_user has a group in common with the become_user. As the + remote_user, we can chgrp the file to that group and allow the + become_user to read it. + * If (the chown fails AND ansible_common_remote_group is not set) OR + (ansible_common_remote_group is set AND the chgrp (or following chmod) + returned non-zero), we can set the file to be world readable so that + the second unprivileged user can read the file. + Since this could allow other users to get access to private + information we only do this if ansible is configured with + "allow_world_readable_tmpfiles" in the ansible.cfg. Also note that + when ansible_common_remote_group is set this final fallback is very + unlikely to ever be triggered, so long as chgrp was successful. But + just because the chgrp was successful, does not mean Ansible can + necessarily access the files (if, for example, the variable was set + to a group that remote_user is in, and can chgrp to, but does not have + in common with become_user). + """ + if remote_user is None: + remote_user = self._get_remote_user() + + # Step 1: Are we on windows? + if getattr(self._connection._shell, "_IS_WINDOWS", False): + # This won't work on Powershell as-is, so we'll just completely + # skip until we have a need for it, at which point we'll have to do + # something different. + return remote_paths + + # Step 2: If we're not becoming an unprivileged user, we are roughly + # done. Make the files +x if we're asked to, and return. + if not self._is_become_unprivileged(): + if execute: + # Can't depend on the file being transferred with execute permissions. + # Only need user perms because no become was used here + res = self._remote_chmod(remote_paths, 'u+x') + if res['rc'] != 0: + raise AnsibleError( + 'Failed to set execute bit on remote files ' + '(rc: {0}, err: {1})'.format( + res['rc'], + to_native(res['stderr']))) + return remote_paths + + # If we're still here, we have an unprivileged user that's different + # than the ssh user. + become_user = self.get_become_option('become_user') + + # Try to use file system acls to make the files readable for sudo'd + # user + if execute: + chmod_mode = 'rx' + setfacl_mode = 'r-x' + # Apple patches their "file_cmds" chmod with ACL support + chmod_acl_mode = '{0} allow read,execute'.format(become_user) + # POSIX-draft ACL specification. Solaris, maybe others. + # See chmod(1) on something Solaris-based for syntax details. + posix_acl_mode = 'A+user:{0}:rx:allow'.format(become_user) + else: + chmod_mode = 'rX' + # TODO: this form fails silently on freebsd. We currently + # never call _fixup_perms2() with execute=False but if we + # start to we'll have to fix this. + setfacl_mode = 'r-X' + # Apple + chmod_acl_mode = '{0} allow read'.format(become_user) + # POSIX-draft + posix_acl_mode = 'A+user:{0}:r:allow'.format(become_user) + + # Step 3a: Are we able to use setfacl to add user ACLs to the file? + res = self._remote_set_user_facl( + remote_paths, + become_user, + setfacl_mode) + + if res['rc'] == 0: + return remote_paths + + # Step 3b: Set execute if we need to. We do this before anything else + # because some of the methods below might work but not let us set +x + # as part of them. + if execute: + res = self._remote_chmod(remote_paths, 'u+x') + if res['rc'] != 0: + raise AnsibleError( + 'Failed to set file mode or acl on remote temporary files ' + '(rc: {0}, err: {1})'.format( + res['rc'], + to_native(res['stderr']))) + + # Step 3c: File system ACLs failed above; try falling back to chown. + res = self._remote_chown(remote_paths, become_user) + if res['rc'] == 0: + return remote_paths + + # Check if we are an admin/root user. If we are and got here, it means + # we failed to chown as root and something weird has happened. + if remote_user in self._get_admin_users(): + raise AnsibleError( + 'Failed to change ownership of the temporary files Ansible ' + '(via chmod nor setfacl) needs to create despite connecting as a ' + 'privileged user. Unprivileged become user would be unable to read' + ' the file.') + + # Step 3d: Try macOS's special chmod + ACL + # macOS chmod's +a flag takes its own argument. As a slight hack, we + # pass that argument as the first element of remote_paths. So we end + # up running `chmod +a [that argument] [file 1] [file 2] ...` + try: + res = self._remote_chmod([chmod_acl_mode] + list(remote_paths), '+a') + except AnsibleAuthenticationFailure as e: + # Solaris-based chmod will return 5 when it sees an invalid mode, + # and +a is invalid there. Because it returns 5, which is the same + # thing sshpass returns on auth failure, our sshpass code will + # assume that auth failed. If we don't handle that case here, none + # of the other logic below will get run. This is fairly hacky and a + # corner case, but probably one that shows up pretty often in + # Solaris-based environments (and possibly others). + pass + else: + if res['rc'] == 0: + return remote_paths + + # Step 3e: Try Solaris/OpenSolaris/OpenIndiana-sans-setfacl chmod + # Similar to macOS above, Solaris 11.4 drops setfacl and takes file ACLs + # via chmod instead. OpenSolaris and illumos-based distros allow for + # using either setfacl or chmod, and compatibility depends on filesystem. + # It should be possible to debug this branch by installing OpenIndiana + # (use ZFS) and going unpriv -> unpriv. + res = self._remote_chmod(remote_paths, posix_acl_mode) + if res['rc'] == 0: + return remote_paths + + # we'll need this down here + become_link = get_versioned_doclink('user_guide/become.html') + + # Step 3f: Common group + # Otherwise, we're a normal user. We failed to chown the paths to the + # unprivileged user, but if we have a common group with them, we should + # be able to chown it to that. + # + # Note that we have no way of knowing if this will actually work... just + # because chgrp exits successfully does not mean that Ansible will work. + # We could check if the become user is in the group, but this would + # create an extra round trip. + # + # Also note that due to the above, this can prevent the + # world_readable_temp logic below from ever getting called. We + # leave this up to the user to rectify if they have both of these + # features enabled. + group = self.get_shell_option('common_remote_group') + if group is not None: + res = self._remote_chgrp(remote_paths, group) + if res['rc'] == 0: + # warn user that something might go weirdly here. + if self.get_shell_option('world_readable_temp'): + display.warning( + 'Both common_remote_group and ' + 'allow_world_readable_tmpfiles are set. chgrp was ' + 'successful, but there is no guarantee that Ansible ' + 'will be able to read the files after this operation, ' + 'particularly if common_remote_group was set to a ' + 'group of which the unprivileged become user is not a ' + 'member. In this situation, ' + 'allow_world_readable_tmpfiles is a no-op. See this ' + 'URL for more details: %s' + '#risks-of-becoming-an-unprivileged-user' % become_link) + if execute: + group_mode = 'g+rwx' + else: + group_mode = 'g+rw' + res = self._remote_chmod(remote_paths, group_mode) + if res['rc'] == 0: + return remote_paths + + # Step 4: World-readable temp directory + if self.get_shell_option('world_readable_temp'): + # chown and fs acls failed -- do things this insecure way only if + # the user opted in in the config file + display.warning( + 'Using world-readable permissions for temporary files Ansible ' + 'needs to create when becoming an unprivileged user. This may ' + 'be insecure. For information on securing this, see %s' + '#risks-of-becoming-an-unprivileged-user' % become_link) + res = self._remote_chmod(remote_paths, 'a+%s' % chmod_mode) + if res['rc'] == 0: + return remote_paths + raise AnsibleError( + 'Failed to set file mode on remote files ' + '(rc: {0}, err: {1})'.format( + res['rc'], + to_native(res['stderr']))) + + raise AnsibleError( + 'Failed to set permissions on the temporary files Ansible needs ' + 'to create when becoming an unprivileged user ' + '(rc: %s, err: %s}). For information on working around this, see %s' + '#risks-of-becoming-an-unprivileged-user' % ( + res['rc'], + to_native(res['stderr']), become_link)) + + def _remote_chmod(self, paths, mode, sudoable=False): + ''' + Issue a remote chmod command + ''' + cmd = self._connection._shell.chmod(paths, mode) + res = self._low_level_execute_command(cmd, sudoable=sudoable) + return res + + def _remote_chown(self, paths, user, sudoable=False): + ''' + Issue a remote chown command + ''' + cmd = self._connection._shell.chown(paths, user) + res = self._low_level_execute_command(cmd, sudoable=sudoable) + return res + + def _remote_chgrp(self, paths, group, sudoable=False): + ''' + Issue a remote chgrp command + ''' + cmd = self._connection._shell.chgrp(paths, group) + res = self._low_level_execute_command(cmd, sudoable=sudoable) + return res + + def _remote_set_user_facl(self, paths, user, mode, sudoable=False): + ''' + Issue a remote call to setfacl + ''' + cmd = self._connection._shell.set_user_facl(paths, user, mode) + res = self._low_level_execute_command(cmd, sudoable=sudoable) + return res + + def _execute_remote_stat(self, path, all_vars, follow, tmp=None, checksum=True): + ''' + Get information from remote file. + ''' + if tmp is not None: + display.warning('_execute_remote_stat no longer honors the tmp parameter. Action' + ' plugins should set self._connection._shell.tmpdir to share' + ' the tmpdir') + del tmp # No longer used + + module_args = dict( + path=path, + follow=follow, + get_checksum=checksum, + checksum_algorithm='sha1', + ) + mystat = self._execute_module(module_name='ansible.legacy.stat', module_args=module_args, task_vars=all_vars, + wrap_async=False) + + if mystat.get('failed'): + msg = mystat.get('module_stderr') + if not msg: + msg = mystat.get('module_stdout') + if not msg: + msg = mystat.get('msg') + raise AnsibleError('Failed to get information on remote file (%s): %s' % (path, msg)) + + if not mystat['stat']['exists']: + # empty might be matched, 1 should never match, also backwards compatible + mystat['stat']['checksum'] = '1' + + # happens sometimes when it is a dir and not on bsd + if 'checksum' not in mystat['stat']: + mystat['stat']['checksum'] = '' + elif not isinstance(mystat['stat']['checksum'], string_types): + raise AnsibleError("Invalid checksum returned by stat: expected a string type but got %s" % type(mystat['stat']['checksum'])) + + return mystat['stat'] + + def _remote_checksum(self, path, all_vars, follow=False): + """Deprecated. Use _execute_remote_stat() instead. + + Produces a remote checksum given a path, + Returns a number 0-4 for specific errors instead of checksum, also ensures it is different + 0 = unknown error + 1 = file does not exist, this might not be an error + 2 = permissions issue + 3 = its a directory, not a file + 4 = stat module failed, likely due to not finding python + 5 = appropriate json module not found + """ + self._display.deprecated("The '_remote_checksum()' method is deprecated. " + "The plugin author should update the code to use '_execute_remote_stat()' instead", "2.16") + x = "0" # unknown error has occurred + try: + remote_stat = self._execute_remote_stat(path, all_vars, follow=follow) + if remote_stat['exists'] and remote_stat['isdir']: + x = "3" # its a directory not a file + else: + x = remote_stat['checksum'] # if 1, file is missing + except AnsibleError as e: + errormsg = to_text(e) + if errormsg.endswith(u'Permission denied'): + x = "2" # cannot read file + elif errormsg.endswith(u'MODULE FAILURE'): + x = "4" # python not found or module uncaught exception + elif 'json' in errormsg: + x = "5" # json module needed + finally: + return x # pylint: disable=lost-exception + + def _remote_expand_user(self, path, sudoable=True, pathsep=None): + ''' takes a remote path and performs tilde/$HOME expansion on the remote host ''' + + # We only expand ~/path and ~username/path + if not path.startswith('~'): + return path + + # Per Jborean, we don't have to worry about Windows as we don't have a notion of user's home + # dir there. + split_path = path.split(os.path.sep, 1) + expand_path = split_path[0] + + if expand_path == '~': + # Network connection plugins (network_cli, netconf, etc.) execute on the controller, rather than the remote host. + # As such, we want to avoid using remote_user for paths as remote_user may not line up with the local user + # This is a hack and should be solved by more intelligent handling of remote_tmp in 2.7 + become_user = self.get_become_option('become_user') + if getattr(self._connection, '_remote_is_local', False): + pass + elif sudoable and self._connection.become and become_user: + expand_path = '~%s' % become_user + else: + # use remote user instead, if none set default to current user + expand_path = '~%s' % (self._get_remote_user() or '') + + # use shell to construct appropriate command and execute + cmd = self._connection._shell.expand_user(expand_path) + data = self._low_level_execute_command(cmd, sudoable=False) + + try: + initial_fragment = data['stdout'].strip().splitlines()[-1] + except IndexError: + initial_fragment = None + + if not initial_fragment: + # Something went wrong trying to expand the path remotely. Try using pwd, if not, return + # the original string + cmd = self._connection._shell.pwd() + pwd = self._low_level_execute_command(cmd, sudoable=False).get('stdout', '').strip() + if pwd: + expanded = pwd + else: + expanded = path + + elif len(split_path) > 1: + expanded = self._connection._shell.join_path(initial_fragment, *split_path[1:]) + else: + expanded = initial_fragment + + if '..' in os.path.dirname(expanded).split('/'): + raise AnsibleError("'%s' returned an invalid relative home directory path containing '..'" % self._get_remote_addr({})) + + return expanded + + def _strip_success_message(self, data): + ''' + Removes the BECOME-SUCCESS message from the data. + ''' + if data.strip().startswith('BECOME-SUCCESS-'): + data = re.sub(r'^((\r)?\n)?BECOME-SUCCESS.*(\r)?\n', '', data) + return data + + def _update_module_args(self, module_name, module_args, task_vars): + + # set check mode in the module arguments, if required + if self._task.check_mode: + if not self._supports_check_mode: + raise AnsibleError("check mode is not supported for this operation") + module_args['_ansible_check_mode'] = True + else: + module_args['_ansible_check_mode'] = False + + # set no log in the module arguments, if required + no_target_syslog = C.config.get_config_value('DEFAULT_NO_TARGET_SYSLOG', variables=task_vars) + module_args['_ansible_no_log'] = self._task.no_log or no_target_syslog + + # set debug in the module arguments, if required + module_args['_ansible_debug'] = C.DEFAULT_DEBUG + + # let module know we are in diff mode + module_args['_ansible_diff'] = self._task.diff + + # let module know our verbosity + module_args['_ansible_verbosity'] = display.verbosity + + # give the module information about the ansible version + module_args['_ansible_version'] = __version__ + + # give the module information about its name + module_args['_ansible_module_name'] = module_name + + # set the syslog facility to be used in the module + module_args['_ansible_syslog_facility'] = task_vars.get('ansible_syslog_facility', C.DEFAULT_SYSLOG_FACILITY) + + # let module know about filesystems that selinux treats specially + module_args['_ansible_selinux_special_fs'] = C.DEFAULT_SELINUX_SPECIAL_FS + + # what to do when parameter values are converted to strings + module_args['_ansible_string_conversion_action'] = C.STRING_CONVERSION_ACTION + + # give the module the socket for persistent connections + module_args['_ansible_socket'] = getattr(self._connection, 'socket_path') + if not module_args['_ansible_socket']: + module_args['_ansible_socket'] = task_vars.get('ansible_socket') + + # make sure all commands use the designated shell executable + module_args['_ansible_shell_executable'] = self._play_context.executable + + # make sure modules are aware if they need to keep the remote files + module_args['_ansible_keep_remote_files'] = C.DEFAULT_KEEP_REMOTE_FILES + + # make sure all commands use the designated temporary directory if created + if self._is_become_unprivileged(): # force fallback on remote_tmp as user cannot normally write to dir + module_args['_ansible_tmpdir'] = None + else: + module_args['_ansible_tmpdir'] = self._connection._shell.tmpdir + + # make sure the remote_tmp value is sent through in case modules needs to create their own + module_args['_ansible_remote_tmp'] = self.get_shell_option('remote_tmp', default='~/.ansible/tmp') + + def _execute_module(self, module_name=None, module_args=None, tmp=None, task_vars=None, persist_files=False, delete_remote_tmp=None, wrap_async=False): + ''' + Transfer and run a module along with its arguments. + ''' + if tmp is not None: + display.warning('_execute_module no longer honors the tmp parameter. Action plugins' + ' should set self._connection._shell.tmpdir to share the tmpdir') + del tmp # No longer used + if delete_remote_tmp is not None: + display.warning('_execute_module no longer honors the delete_remote_tmp parameter.' + ' Action plugins should check self._connection._shell.tmpdir to' + ' see if a tmpdir existed before they were called to determine' + ' if they are responsible for removing it.') + del delete_remote_tmp # No longer used + + tmpdir = self._connection._shell.tmpdir + + # We set the module_style to new here so the remote_tmp is created + # before the module args are built if remote_tmp is needed (async). + # If the module_style turns out to not be new and we didn't create the + # remote tmp here, it will still be created. This must be done before + # calling self._update_module_args() so the module wrapper has the + # correct remote_tmp value set + if not self._is_pipelining_enabled("new", wrap_async) and tmpdir is None: + self._make_tmp_path() + tmpdir = self._connection._shell.tmpdir + + if task_vars is None: + task_vars = dict() + + # if a module name was not specified for this execution, use the action from the task + if module_name is None: + module_name = self._task.action + if module_args is None: + module_args = self._task.args + + self._update_module_args(module_name, module_args, task_vars) + + remove_async_dir = None + if wrap_async or self._task.async_val: + async_dir = self.get_shell_option('async_dir', default="~/.ansible_async") + remove_async_dir = len(self._task.environment) + self._task.environment.append({"ANSIBLE_ASYNC_DIR": async_dir}) + + # FUTURE: refactor this along with module build process to better encapsulate "smart wrapper" functionality + (module_style, shebang, module_data, module_path) = self._configure_module(module_name=module_name, module_args=module_args, task_vars=task_vars) + display.vvv("Using module file %s" % module_path) + if not shebang and module_style != 'binary': + raise AnsibleError("module (%s) is missing interpreter line" % module_name) + + self._used_interpreter = shebang + remote_module_path = None + + if not self._is_pipelining_enabled(module_style, wrap_async): + # we might need remote tmp dir + if tmpdir is None: + self._make_tmp_path() + tmpdir = self._connection._shell.tmpdir + + remote_module_filename = self._connection._shell.get_remote_filename(module_path) + remote_module_path = self._connection._shell.join_path(tmpdir, 'AnsiballZ_%s' % remote_module_filename) + + args_file_path = None + if module_style in ('old', 'non_native_want_json', 'binary'): + # we'll also need a tmp file to hold our module arguments + args_file_path = self._connection._shell.join_path(tmpdir, 'args') + + if remote_module_path or module_style != 'new': + display.debug("transferring module to remote %s" % remote_module_path) + if module_style == 'binary': + self._transfer_file(module_path, remote_module_path) + else: + self._transfer_data(remote_module_path, module_data) + if module_style == 'old': + # we need to dump the module args to a k=v string in a file on + # the remote system, which can be read and parsed by the module + args_data = "" + for k, v in module_args.items(): + args_data += '%s=%s ' % (k, shlex.quote(text_type(v))) + self._transfer_data(args_file_path, args_data) + elif module_style in ('non_native_want_json', 'binary'): + self._transfer_data(args_file_path, json.dumps(module_args)) + display.debug("done transferring module to remote") + + environment_string = self._compute_environment_string() + + # remove the ANSIBLE_ASYNC_DIR env entry if we added a temporary one for + # the async_wrapper task. + if remove_async_dir is not None: + del self._task.environment[remove_async_dir] + + remote_files = [] + if tmpdir and remote_module_path: + remote_files = [tmpdir, remote_module_path] + + if args_file_path: + remote_files.append(args_file_path) + + sudoable = True + in_data = None + cmd = "" + + if wrap_async and not self._connection.always_pipeline_modules: + # configure, upload, and chmod the async_wrapper module + (async_module_style, shebang, async_module_data, async_module_path) = self._configure_module( + module_name='ansible.legacy.async_wrapper', module_args=dict(), task_vars=task_vars) + async_module_remote_filename = self._connection._shell.get_remote_filename(async_module_path) + remote_async_module_path = self._connection._shell.join_path(tmpdir, async_module_remote_filename) + self._transfer_data(remote_async_module_path, async_module_data) + remote_files.append(remote_async_module_path) + + async_limit = self._task.async_val + async_jid = str(random.randint(0, 999999999999)) + + # call the interpreter for async_wrapper directly + # this permits use of a script for an interpreter on non-Linux platforms + interpreter = shebang.replace('#!', '').strip() + async_cmd = [interpreter, remote_async_module_path, async_jid, async_limit, remote_module_path] + + if environment_string: + async_cmd.insert(0, environment_string) + + if args_file_path: + async_cmd.append(args_file_path) + else: + # maintain a fixed number of positional parameters for async_wrapper + async_cmd.append('_') + + if not self._should_remove_tmp_path(tmpdir): + async_cmd.append("-preserve_tmp") + + cmd = " ".join(to_text(x) for x in async_cmd) + + else: + + if self._is_pipelining_enabled(module_style): + in_data = module_data + display.vvv("Pipelining is enabled.") + else: + cmd = remote_module_path + + cmd = self._connection._shell.build_module_command(environment_string, shebang, cmd, arg_path=args_file_path).strip() + + # Fix permissions of the tmpdir path and tmpdir files. This should be called after all + # files have been transferred. + if remote_files: + # remove none/empty + remote_files = [x for x in remote_files if x] + self._fixup_perms2(remote_files, self._get_remote_user()) + + # actually execute + res = self._low_level_execute_command(cmd, sudoable=sudoable, in_data=in_data) + + # parse the main result + data = self._parse_returned_data(res) + + # NOTE: INTERNAL KEYS ONLY ACCESSIBLE HERE + # get internal info before cleaning + if data.pop("_ansible_suppress_tmpdir_delete", False): + self._cleanup_remote_tmp = False + + # NOTE: yum returns results .. but that made it 'compatible' with squashing, so we allow mappings, for now + if 'results' in data and (not isinstance(data['results'], Sequence) or isinstance(data['results'], string_types)): + data['ansible_module_results'] = data['results'] + del data['results'] + display.warning("Found internal 'results' key in module return, renamed to 'ansible_module_results'.") + + # remove internal keys + remove_internal_keys(data) + + if wrap_async: + # async_wrapper will clean up its tmpdir on its own so we want the controller side to + # forget about it now + self._connection._shell.tmpdir = None + + # FIXME: for backwards compat, figure out if still makes sense + data['changed'] = True + + # pre-split stdout/stderr into lines if needed + if 'stdout' in data and 'stdout_lines' not in data: + # if the value is 'False', a default won't catch it. + txt = data.get('stdout', None) or u'' + data['stdout_lines'] = txt.splitlines() + if 'stderr' in data and 'stderr_lines' not in data: + # if the value is 'False', a default won't catch it. + txt = data.get('stderr', None) or u'' + data['stderr_lines'] = txt.splitlines() + + # propagate interpreter discovery results back to the controller + if self._discovered_interpreter_key: + if data.get('ansible_facts') is None: + data['ansible_facts'] = {} + + data['ansible_facts'][self._discovered_interpreter_key] = self._discovered_interpreter + + if self._discovery_warnings: + if data.get('warnings') is None: + data['warnings'] = [] + data['warnings'].extend(self._discovery_warnings) + + if self._discovery_deprecation_warnings: + if data.get('deprecations') is None: + data['deprecations'] = [] + data['deprecations'].extend(self._discovery_deprecation_warnings) + + # mark the entire module results untrusted as a template right here, since the current action could + # possibly template one of these values. + data = wrap_var(data) + + display.debug("done with _execute_module (%s, %s)" % (module_name, module_args)) + return data + + def _parse_returned_data(self, res): + try: + filtered_output, warnings = _filter_non_json_lines(res.get('stdout', u''), objects_only=True) + for w in warnings: + display.warning(w) + + data = json.loads(filtered_output) + data['_ansible_parsed'] = True + except ValueError: + # not valid json, lets try to capture error + data = dict(failed=True, _ansible_parsed=False) + data['module_stdout'] = res.get('stdout', u'') + if 'stderr' in res: + data['module_stderr'] = res['stderr'] + if res['stderr'].startswith(u'Traceback'): + data['exception'] = res['stderr'] + + # in some cases a traceback will arrive on stdout instead of stderr, such as when using ssh with -tt + if 'exception' not in data and data['module_stdout'].startswith(u'Traceback'): + data['exception'] = data['module_stdout'] + + # The default + data['msg'] = "MODULE FAILURE" + + # try to figure out if we are missing interpreter + if self._used_interpreter is not None: + interpreter = re.escape(self._used_interpreter.lstrip('!#')) + match = re.compile('%s: (?:No such file or directory|not found)' % interpreter) + if match.search(data['module_stderr']) or match.search(data['module_stdout']): + data['msg'] = "The module failed to execute correctly, you probably need to set the interpreter." + + # always append hint + data['msg'] += '\nSee stdout/stderr for the exact error' + + if 'rc' in res: + data['rc'] = res['rc'] + return data + + # FIXME: move to connection base + def _low_level_execute_command(self, cmd, sudoable=True, in_data=None, executable=None, encoding_errors='surrogate_then_replace', chdir=None): + ''' + This is the function which executes the low level shell command, which + may be commands to create/remove directories for temporary files, or to + run the module code or python directly when pipelining. + + :kwarg encoding_errors: If the value returned by the command isn't + utf-8 then we have to figure out how to transform it to unicode. + If the value is just going to be displayed to the user (or + discarded) then the default of 'replace' is fine. If the data is + used as a key or is going to be written back out to a file + verbatim, then this won't work. May have to use some sort of + replacement strategy (python3 could use surrogateescape) + :kwarg chdir: cd into this directory before executing the command. + ''' + + display.debug("_low_level_execute_command(): starting") + # if not cmd: + # # this can happen with powershell modules when there is no analog to a Windows command (like chmod) + # display.debug("_low_level_execute_command(): no command, exiting") + # return dict(stdout='', stderr='', rc=254) + + if chdir: + display.debug("_low_level_execute_command(): changing cwd to %s for this command" % chdir) + cmd = self._connection._shell.append_command('cd %s' % chdir, cmd) + + # https://github.com/ansible/ansible/issues/68054 + if executable: + self._connection._shell.executable = executable + + ruser = self._get_remote_user() + buser = self.get_become_option('become_user') + if (sudoable and self._connection.become and # if sudoable and have become + resource_from_fqcr(self._connection.transport) != 'network_cli' and # if not using network_cli + (C.BECOME_ALLOW_SAME_USER or (buser != ruser or not any((ruser, buser))))): # if we allow same user PE or users are different and either is set + display.debug("_low_level_execute_command(): using become for this command") + cmd = self._connection.become.build_become_command(cmd, self._connection._shell) + + if self._connection.allow_executable: + if executable is None: + executable = self._play_context.executable + # mitigation for SSH race which can drop stdout (https://github.com/ansible/ansible/issues/13876) + # only applied for the default executable to avoid interfering with the raw action + cmd = self._connection._shell.append_command(cmd, 'sleep 0') + if executable: + cmd = executable + ' -c ' + shlex.quote(cmd) + + display.debug("_low_level_execute_command(): executing: %s" % (cmd,)) + + # Change directory to basedir of task for command execution when connection is local + if self._connection.transport == 'local': + self._connection.cwd = to_bytes(self._loader.get_basedir(), errors='surrogate_or_strict') + + rc, stdout, stderr = self._connection.exec_command(cmd, in_data=in_data, sudoable=sudoable) + + # stdout and stderr may be either a file-like or a bytes object. + # Convert either one to a text type + if isinstance(stdout, binary_type): + out = to_text(stdout, errors=encoding_errors) + elif not isinstance(stdout, text_type): + out = to_text(b''.join(stdout.readlines()), errors=encoding_errors) + else: + out = stdout + + if isinstance(stderr, binary_type): + err = to_text(stderr, errors=encoding_errors) + elif not isinstance(stderr, text_type): + err = to_text(b''.join(stderr.readlines()), errors=encoding_errors) + else: + err = stderr + + if rc is None: + rc = 0 + + # be sure to remove the BECOME-SUCCESS message now + out = self._strip_success_message(out) + + display.debug(u"_low_level_execute_command() done: rc=%d, stdout=%s, stderr=%s" % (rc, out, err)) + return dict(rc=rc, stdout=out, stdout_lines=out.splitlines(), stderr=err, stderr_lines=err.splitlines()) + + def _get_diff_data(self, destination, source, task_vars, source_file=True): + + # Note: Since we do not diff the source and destination before we transform from bytes into + # text the diff between source and destination may not be accurate. To fix this, we'd need + # to move the diffing from the callback plugins into here. + # + # Example of data which would cause trouble is src_content == b'\xff' and dest_content == + # b'\xfe'. Neither of those are valid utf-8 so both get turned into the replacement + # character: diff['before'] = u'�' ; diff['after'] = u'�' When the callback plugin later + # diffs before and after it shows an empty diff. + + diff = {} + display.debug("Going to peek to see if file has changed permissions") + peek_result = self._execute_module( + module_name='ansible.legacy.file', module_args=dict(path=destination, _diff_peek=True), + task_vars=task_vars, persist_files=True) + + if peek_result.get('failed', False): + display.warning(u"Failed to get diff between '%s' and '%s': %s" % (os.path.basename(source), destination, to_text(peek_result.get(u'msg', u'')))) + return diff + + if peek_result.get('rc', 0) == 0: + + if peek_result.get('state') in (None, 'absent'): + diff['before'] = u'' + elif peek_result.get('appears_binary'): + diff['dst_binary'] = 1 + elif peek_result.get('size') and C.MAX_FILE_SIZE_FOR_DIFF > 0 and peek_result['size'] > C.MAX_FILE_SIZE_FOR_DIFF: + diff['dst_larger'] = C.MAX_FILE_SIZE_FOR_DIFF + else: + display.debug(u"Slurping the file %s" % source) + dest_result = self._execute_module( + module_name='ansible.legacy.slurp', module_args=dict(path=destination), + task_vars=task_vars, persist_files=True) + if 'content' in dest_result: + dest_contents = dest_result['content'] + if dest_result['encoding'] == u'base64': + dest_contents = base64.b64decode(dest_contents) + else: + raise AnsibleError("unknown encoding in content option, failed: %s" % to_native(dest_result)) + diff['before_header'] = destination + diff['before'] = to_text(dest_contents) + + if source_file: + st = os.stat(source) + if C.MAX_FILE_SIZE_FOR_DIFF > 0 and st[stat.ST_SIZE] > C.MAX_FILE_SIZE_FOR_DIFF: + diff['src_larger'] = C.MAX_FILE_SIZE_FOR_DIFF + else: + display.debug("Reading local copy of the file %s" % source) + try: + with open(source, 'rb') as src: + src_contents = src.read() + except Exception as e: + raise AnsibleError("Unexpected error while reading source (%s) for diff: %s " % (source, to_native(e))) + + if b"\x00" in src_contents: + diff['src_binary'] = 1 + else: + diff['after_header'] = source + diff['after'] = to_text(src_contents) + else: + display.debug(u"source of file passed in") + diff['after_header'] = u'dynamically generated' + diff['after'] = source + + if self._task.no_log: + if 'before' in diff: + diff["before"] = u"" + if 'after' in diff: + diff["after"] = u" [[ Diff output has been hidden because 'no_log: true' was specified for this result ]]\n" + + return diff + + def _find_needle(self, dirname, needle): + ''' + find a needle in haystack of paths, optionally using 'dirname' as a subdir. + This will build the ordered list of paths to search and pass them to dwim + to get back the first existing file found. + ''' + + # dwim already deals with playbook basedirs + path_stack = self._task.get_search_path() + + # if missing it will return a file not found exception + return self._loader.path_dwim_relative_stack(path_stack, dirname, needle) diff --git a/lib/ansible/plugins/action/add_host.py b/lib/ansible/plugins/action/add_host.py new file mode 100644 index 0000000..e569739 --- /dev/null +++ b/lib/ansible/plugins/action/add_host.py @@ -0,0 +1,98 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# Copyright 2012, Seth Vidal <skvidal@fedoraproject.org> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from collections.abc import Mapping + +from ansible.errors import AnsibleActionFail +from ansible.module_utils.six import string_types +from ansible.plugins.action import ActionBase +from ansible.parsing.utils.addresses import parse_address +from ansible.utils.display import Display +from ansible.utils.vars import combine_vars + +display = Display() + + +class ActionModule(ActionBase): + ''' Create inventory hosts and groups in the memory inventory''' + + # We need to be able to modify the inventory + BYPASS_HOST_LOOP = True + TRANSFERS_FILES = False + + def run(self, tmp=None, task_vars=None): + + self._supports_check_mode = True + + result = super(ActionModule, self).run(tmp, task_vars) + del tmp # tmp no longer has any effect + + args = self._task.args + raw = args.pop('_raw_params', {}) + if isinstance(raw, Mapping): + # TODO: create 'conflict' detection in base class to deal with repeats and aliases and warn user + args = combine_vars(raw, args) + else: + raise AnsibleActionFail('Invalid raw parameters passed, requires a dictonary/mapping got a %s' % type(raw)) + + # Parse out any hostname:port patterns + new_name = args.get('name', args.get('hostname', args.get('host', None))) + if new_name is None: + raise AnsibleActionFail('name, host or hostname needs to be provided') + + display.vv("creating host via 'add_host': hostname=%s" % new_name) + + try: + name, port = parse_address(new_name, allow_ranges=False) + except Exception: + # not a parsable hostname, but might still be usable + name = new_name + port = None + + if port: + args['ansible_ssh_port'] = port + + groups = args.get('groupname', args.get('groups', args.get('group', ''))) + # add it to the group if that was specified + new_groups = [] + if groups: + if isinstance(groups, list): + group_list = groups + elif isinstance(groups, string_types): + group_list = groups.split(",") + else: + raise AnsibleActionFail("Groups must be specified as a list.", obj=self._task) + + for group_name in group_list: + if group_name not in new_groups: + new_groups.append(group_name.strip()) + + # Add any variables to the new_host + host_vars = dict() + special_args = frozenset(('name', 'hostname', 'groupname', 'groups')) + for k in args.keys(): + if k not in special_args: + host_vars[k] = args[k] + + result['changed'] = False + result['add_host'] = dict(host_name=name, groups=new_groups, host_vars=host_vars) + return result diff --git a/lib/ansible/plugins/action/assemble.py b/lib/ansible/plugins/action/assemble.py new file mode 100644 index 0000000..06fa2df --- /dev/null +++ b/lib/ansible/plugins/action/assemble.py @@ -0,0 +1,166 @@ +# (c) 2013-2016, Michael DeHaan <michael.dehaan@gmail.com> +# Stephen Fromm <sfromm@gmail.com> +# Brian Coca <briancoca+dev@gmail.com> +# Toshio Kuratomi <tkuratomi@ansible.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import codecs +import os +import os.path +import re +import tempfile + +from ansible import constants as C +from ansible.errors import AnsibleError, AnsibleAction, _AnsibleActionDone, AnsibleActionFail +from ansible.module_utils._text import to_native, to_text +from ansible.module_utils.parsing.convert_bool import boolean +from ansible.plugins.action import ActionBase +from ansible.utils.hashing import checksum_s + + +class ActionModule(ActionBase): + + TRANSFERS_FILES = True + + def _assemble_from_fragments(self, src_path, delimiter=None, compiled_regexp=None, ignore_hidden=False, decrypt=True): + ''' assemble a file from a directory of fragments ''' + + tmpfd, temp_path = tempfile.mkstemp(dir=C.DEFAULT_LOCAL_TMP) + tmp = os.fdopen(tmpfd, 'wb') + delimit_me = False + add_newline = False + + for f in (to_text(p, errors='surrogate_or_strict') for p in sorted(os.listdir(src_path))): + if compiled_regexp and not compiled_regexp.search(f): + continue + fragment = u"%s/%s" % (src_path, f) + if not os.path.isfile(fragment) or (ignore_hidden and os.path.basename(fragment).startswith('.')): + continue + + with open(self._loader.get_real_file(fragment, decrypt=decrypt), 'rb') as fragment_fh: + fragment_content = fragment_fh.read() + + # always put a newline between fragments if the previous fragment didn't end with a newline. + if add_newline: + tmp.write(b'\n') + + # delimiters should only appear between fragments + if delimit_me: + if delimiter: + # un-escape anything like newlines + delimiter = codecs.escape_decode(delimiter)[0] + tmp.write(delimiter) + # always make sure there's a newline after the + # delimiter, so lines don't run together + if delimiter[-1] != b'\n': + tmp.write(b'\n') + + tmp.write(fragment_content) + delimit_me = True + if fragment_content.endswith(b'\n'): + add_newline = False + else: + add_newline = True + + tmp.close() + return temp_path + + def run(self, tmp=None, task_vars=None): + + self._supports_check_mode = False + + result = super(ActionModule, self).run(tmp, task_vars) + del tmp # tmp no longer has any effect + + if task_vars is None: + task_vars = dict() + + src = self._task.args.get('src', None) + dest = self._task.args.get('dest', None) + delimiter = self._task.args.get('delimiter', None) + remote_src = self._task.args.get('remote_src', 'yes') + regexp = self._task.args.get('regexp', None) + follow = self._task.args.get('follow', False) + ignore_hidden = self._task.args.get('ignore_hidden', False) + decrypt = self._task.args.pop('decrypt', True) + + try: + if src is None or dest is None: + raise AnsibleActionFail("src and dest are required") + + if boolean(remote_src, strict=False): + # call assemble via ansible.legacy to allow library/ overrides of the module without collection search + result.update(self._execute_module(module_name='ansible.legacy.assemble', task_vars=task_vars)) + raise _AnsibleActionDone() + else: + try: + src = self._find_needle('files', src) + except AnsibleError as e: + raise AnsibleActionFail(to_native(e)) + + if not os.path.isdir(src): + raise AnsibleActionFail(u"Source (%s) is not a directory" % src) + + _re = None + if regexp is not None: + _re = re.compile(regexp) + + # Does all work assembling the file + path = self._assemble_from_fragments(src, delimiter, _re, ignore_hidden, decrypt) + + path_checksum = checksum_s(path) + dest = self._remote_expand_user(dest) + dest_stat = self._execute_remote_stat(dest, all_vars=task_vars, follow=follow) + + diff = {} + + # setup args for running modules + new_module_args = self._task.args.copy() + + # clean assemble specific options + for opt in ['remote_src', 'regexp', 'delimiter', 'ignore_hidden', 'decrypt']: + if opt in new_module_args: + del new_module_args[opt] + new_module_args['dest'] = dest + + if path_checksum != dest_stat['checksum']: + + if self._play_context.diff: + diff = self._get_diff_data(dest, path, task_vars) + + remote_path = self._connection._shell.join_path(self._connection._shell.tmpdir, 'src') + xfered = self._transfer_file(path, remote_path) + + # fix file permissions when the copy is done as a different user + self._fixup_perms2((self._connection._shell.tmpdir, remote_path)) + + new_module_args.update(dict(src=xfered,)) + + res = self._execute_module(module_name='ansible.legacy.copy', module_args=new_module_args, task_vars=task_vars) + if diff: + res['diff'] = diff + result.update(res) + else: + result.update(self._execute_module(module_name='ansible.legacy.file', module_args=new_module_args, task_vars=task_vars)) + + except AnsibleAction as e: + result.update(e.result) + finally: + self._remove_tmp_path(self._connection._shell.tmpdir) + + return result diff --git a/lib/ansible/plugins/action/assert.py b/lib/ansible/plugins/action/assert.py new file mode 100644 index 0000000..7721a6b --- /dev/null +++ b/lib/ansible/plugins/action/assert.py @@ -0,0 +1,94 @@ +# Copyright 2012, Dag Wieers <dag@wieers.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible.errors import AnsibleError +from ansible.playbook.conditional import Conditional +from ansible.plugins.action import ActionBase +from ansible.module_utils.six import string_types +from ansible.module_utils.parsing.convert_bool import boolean + + +class ActionModule(ActionBase): + ''' Fail with custom message ''' + + TRANSFERS_FILES = False + _VALID_ARGS = frozenset(('fail_msg', 'msg', 'quiet', 'success_msg', 'that')) + + def run(self, tmp=None, task_vars=None): + if task_vars is None: + task_vars = dict() + + result = super(ActionModule, self).run(tmp, task_vars) + del tmp # tmp no longer has any effect + + if 'that' not in self._task.args: + raise AnsibleError('conditional required in "that" string') + + fail_msg = None + success_msg = None + + fail_msg = self._task.args.get('fail_msg', self._task.args.get('msg')) + if fail_msg is None: + fail_msg = 'Assertion failed' + elif isinstance(fail_msg, list): + if not all(isinstance(x, string_types) for x in fail_msg): + raise AnsibleError('Type of one of the elements in fail_msg or msg list is not string type') + elif not isinstance(fail_msg, (string_types, list)): + raise AnsibleError('Incorrect type for fail_msg or msg, expected a string or list and got %s' % type(fail_msg)) + + success_msg = self._task.args.get('success_msg') + if success_msg is None: + success_msg = 'All assertions passed' + elif isinstance(success_msg, list): + if not all(isinstance(x, string_types) for x in success_msg): + raise AnsibleError('Type of one of the elements in success_msg list is not string type') + elif not isinstance(success_msg, (string_types, list)): + raise AnsibleError('Incorrect type for success_msg, expected a string or list and got %s' % type(success_msg)) + + quiet = boolean(self._task.args.get('quiet', False), strict=False) + + # make sure the 'that' items are a list + thats = self._task.args['that'] + if not isinstance(thats, list): + thats = [thats] + + # Now we iterate over the that items, temporarily assigning them + # to the task's when value so we can evaluate the conditional using + # the built in evaluate function. The when has already been evaluated + # by this point, and is not used again, so we don't care about mangling + # that value now + cond = Conditional(loader=self._loader) + if not quiet: + result['_ansible_verbose_always'] = True + + for that in thats: + cond.when = [that] + test_result = cond.evaluate_conditional(templar=self._templar, all_vars=task_vars) + if not test_result: + result['failed'] = True + result['evaluated_to'] = test_result + result['assertion'] = that + + result['msg'] = fail_msg + + return result + + result['changed'] = False + result['msg'] = success_msg + return result diff --git a/lib/ansible/plugins/action/async_status.py b/lib/ansible/plugins/action/async_status.py new file mode 100644 index 0000000..ad839f1 --- /dev/null +++ b/lib/ansible/plugins/action/async_status.py @@ -0,0 +1,53 @@ +# Copyright: (c) 2018, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible.errors import AnsibleActionFail +from ansible.plugins.action import ActionBase +from ansible.utils.vars import merge_hash + + +class ActionModule(ActionBase): + + def _get_async_dir(self): + + # async directory based on the shell option + async_dir = self.get_shell_option('async_dir', default="~/.ansible_async") + + return self._remote_expand_user(async_dir) + + def run(self, tmp=None, task_vars=None): + + results = super(ActionModule, self).run(tmp, task_vars) + + validation_result, new_module_args = self.validate_argument_spec( + argument_spec={ + 'jid': {'type': 'str', 'required': True}, + 'mode': {'type': 'str', 'choices': ['status', 'cleanup'], 'default': 'status'}, + }, + ) + + # initialize response + results['started'] = results['finished'] = 0 + results['stdout'] = results['stderr'] = '' + results['stdout_lines'] = results['stderr_lines'] = [] + + jid = new_module_args["jid"] + mode = new_module_args["mode"] + + results['ansible_job_id'] = jid + async_dir = self._get_async_dir() + log_path = self._connection._shell.join_path(async_dir, jid) + + if mode == 'cleanup': + results['erased'] = log_path + else: + results['results_file'] = log_path + results['started'] = 1 + + new_module_args['_async_dir'] = async_dir + results = merge_hash(results, self._execute_module(module_name='ansible.legacy.async_status', task_vars=task_vars, module_args=new_module_args)) + + return results diff --git a/lib/ansible/plugins/action/command.py b/lib/ansible/plugins/action/command.py new file mode 100644 index 0000000..82a85dc --- /dev/null +++ b/lib/ansible/plugins/action/command.py @@ -0,0 +1,28 @@ +# Copyright: (c) 2017, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible import constants as C +from ansible.plugins.action import ActionBase +from ansible.utils.vars import merge_hash + + +class ActionModule(ActionBase): + + def run(self, tmp=None, task_vars=None): + self._supports_async = True + results = super(ActionModule, self).run(tmp, task_vars) + del tmp # tmp no longer has any effect + + wrap_async = self._task.async_val and not self._connection.has_native_async + # explicitly call `ansible.legacy.command` for backcompat to allow library/ override of `command` while not allowing + # collections search for an unqualified `command` module + results = merge_hash(results, self._execute_module(module_name='ansible.legacy.command', task_vars=task_vars, wrap_async=wrap_async)) + + if not wrap_async: + # remove a temporary path we created + self._remove_tmp_path(self._connection._shell.tmpdir) + + return results diff --git a/lib/ansible/plugins/action/copy.py b/lib/ansible/plugins/action/copy.py new file mode 100644 index 0000000..cb3d15b --- /dev/null +++ b/lib/ansible/plugins/action/copy.py @@ -0,0 +1,599 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# (c) 2017 Toshio Kuratomi <tkuraotmi@ansible.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import json +import os +import os.path +import stat +import tempfile +import traceback + +from ansible import constants as C +from ansible.errors import AnsibleError, AnsibleFileNotFound +from ansible.module_utils.basic import FILE_COMMON_ARGUMENTS +from ansible.module_utils._text import to_bytes, to_native, to_text +from ansible.module_utils.parsing.convert_bool import boolean +from ansible.plugins.action import ActionBase +from ansible.utils.hashing import checksum + + +# Supplement the FILE_COMMON_ARGUMENTS with arguments that are specific to file +REAL_FILE_ARGS = frozenset(FILE_COMMON_ARGUMENTS.keys()).union( + ('state', 'path', '_original_basename', 'recurse', 'force', + '_diff_peek', 'src')) + + +def _create_remote_file_args(module_args): + """remove keys that are not relevant to file""" + return dict((k, v) for k, v in module_args.items() if k in REAL_FILE_ARGS) + + +def _create_remote_copy_args(module_args): + """remove action plugin only keys""" + return dict((k, v) for k, v in module_args.items() if k not in ('content', 'decrypt')) + + +def _walk_dirs(topdir, base_path=None, local_follow=False, trailing_slash_detector=None): + """ + Walk a filesystem tree returning enough information to copy the files + + :arg topdir: The directory that the filesystem tree is rooted at + :kwarg base_path: The initial directory structure to strip off of the + files for the destination directory. If this is None (the default), + the base_path is set to ``top_dir``. + :kwarg local_follow: Whether to follow symlinks on the source. When set + to False, no symlinks are dereferenced. When set to True (the + default), the code will dereference most symlinks. However, symlinks + can still be present if needed to break a circular link. + :kwarg trailing_slash_detector: Function to determine if a path has + a trailing directory separator. Only needed when dealing with paths on + a remote machine (in which case, pass in a function that is aware of the + directory separator conventions on the remote machine). + :returns: dictionary of tuples. All of the path elements in the structure are text strings. + This separates all the files, directories, and symlinks along with + important information about each:: + + { 'files': [('/absolute/path/to/copy/from', 'relative/path/to/copy/to'), ...], + 'directories': [('/absolute/path/to/copy/from', 'relative/path/to/copy/to'), ...], + 'symlinks': [('/symlink/target/path', 'relative/path/to/copy/to'), ...], + } + + The ``symlinks`` field is only populated if ``local_follow`` is set to False + *or* a circular symlink cannot be dereferenced. + + """ + # Convert the path segments into byte strings + + r_files = {'files': [], 'directories': [], 'symlinks': []} + + def _recurse(topdir, rel_offset, parent_dirs, rel_base=u''): + """ + This is a closure (function utilizing variables from it's parent + function's scope) so that we only need one copy of all the containers. + Note that this function uses side effects (See the Variables used from + outer scope). + + :arg topdir: The directory we are walking for files + :arg rel_offset: Integer defining how many characters to strip off of + the beginning of a path + :arg parent_dirs: Directories that we're copying that this directory is in. + :kwarg rel_base: String to prepend to the path after ``rel_offset`` is + applied to form the relative path. + + Variables used from the outer scope + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + :r_files: Dictionary of files in the hierarchy. See the return value + for :func:`walk` for the structure of this dictionary. + :local_follow: Read-only inside of :func:`_recurse`. Whether to follow symlinks + """ + for base_path, sub_folders, files in os.walk(topdir): + for filename in files: + filepath = os.path.join(base_path, filename) + dest_filepath = os.path.join(rel_base, filepath[rel_offset:]) + + if os.path.islink(filepath): + # Dereference the symlnk + real_file = os.path.realpath(filepath) + if local_follow and os.path.isfile(real_file): + # Add the file pointed to by the symlink + r_files['files'].append((real_file, dest_filepath)) + else: + # Mark this file as a symlink to copy + r_files['symlinks'].append((os.readlink(filepath), dest_filepath)) + else: + # Just a normal file + r_files['files'].append((filepath, dest_filepath)) + + for dirname in sub_folders: + dirpath = os.path.join(base_path, dirname) + dest_dirpath = os.path.join(rel_base, dirpath[rel_offset:]) + real_dir = os.path.realpath(dirpath) + dir_stats = os.stat(real_dir) + + if os.path.islink(dirpath): + if local_follow: + if (dir_stats.st_dev, dir_stats.st_ino) in parent_dirs: + # Just insert the symlink if the target directory + # exists inside of the copy already + r_files['symlinks'].append((os.readlink(dirpath), dest_dirpath)) + else: + # Walk the dirpath to find all parent directories. + new_parents = set() + parent_dir_list = os.path.dirname(dirpath).split(os.path.sep) + for parent in range(len(parent_dir_list), 0, -1): + parent_stat = os.stat(u'/'.join(parent_dir_list[:parent])) + if (parent_stat.st_dev, parent_stat.st_ino) in parent_dirs: + # Reached the point at which the directory + # tree is already known. Don't add any + # more or we might go to an ancestor that + # isn't being copied. + break + new_parents.add((parent_stat.st_dev, parent_stat.st_ino)) + + if (dir_stats.st_dev, dir_stats.st_ino) in new_parents: + # This was a a circular symlink. So add it as + # a symlink + r_files['symlinks'].append((os.readlink(dirpath), dest_dirpath)) + else: + # Walk the directory pointed to by the symlink + r_files['directories'].append((real_dir, dest_dirpath)) + offset = len(real_dir) + 1 + _recurse(real_dir, offset, parent_dirs.union(new_parents), rel_base=dest_dirpath) + else: + # Add the symlink to the destination + r_files['symlinks'].append((os.readlink(dirpath), dest_dirpath)) + else: + # Just a normal directory + r_files['directories'].append((dirpath, dest_dirpath)) + + # Check if the source ends with a "/" so that we know which directory + # level to work at (similar to rsync) + source_trailing_slash = False + if trailing_slash_detector: + source_trailing_slash = trailing_slash_detector(topdir) + else: + source_trailing_slash = topdir.endswith(os.path.sep) + + # Calculate the offset needed to strip the base_path to make relative + # paths + if base_path is None: + base_path = topdir + if not source_trailing_slash: + base_path = os.path.dirname(base_path) + if topdir.startswith(base_path): + offset = len(base_path) + + # Make sure we're making the new paths relative + if trailing_slash_detector and not trailing_slash_detector(base_path): + offset += 1 + elif not base_path.endswith(os.path.sep): + offset += 1 + + if os.path.islink(topdir) and not local_follow: + r_files['symlinks'] = (os.readlink(topdir), os.path.basename(topdir)) + return r_files + + dir_stats = os.stat(topdir) + parents = frozenset(((dir_stats.st_dev, dir_stats.st_ino),)) + # Actually walk the directory hierarchy + _recurse(topdir, offset, parents) + + return r_files + + +class ActionModule(ActionBase): + + TRANSFERS_FILES = True + + def _ensure_invocation(self, result): + # NOTE: adding invocation arguments here needs to be kept in sync with + # any no_log specified in the argument_spec in the module. + # This is not automatic. + # NOTE: do not add to this. This should be made a generic function for action plugins. + # This should also use the same argspec as the module instead of keeping it in sync. + if 'invocation' not in result: + if self._play_context.no_log: + result['invocation'] = "CENSORED: no_log is set" + else: + # NOTE: Should be removed in the future. For now keep this broken + # behaviour, have a look in the PR 51582 + result['invocation'] = self._task.args.copy() + result['invocation']['module_args'] = self._task.args.copy() + + if isinstance(result['invocation'], dict): + if 'content' in result['invocation']: + result['invocation']['content'] = 'CENSORED: content is a no_log parameter' + if result['invocation'].get('module_args', {}).get('content') is not None: + result['invocation']['module_args']['content'] = 'VALUE_SPECIFIED_IN_NO_LOG_PARAMETER' + + return result + + def _copy_file(self, source_full, source_rel, content, content_tempfile, + dest, task_vars, follow): + decrypt = boolean(self._task.args.get('decrypt', True), strict=False) + force = boolean(self._task.args.get('force', 'yes'), strict=False) + raw = boolean(self._task.args.get('raw', 'no'), strict=False) + + result = {} + result['diff'] = [] + + # If the local file does not exist, get_real_file() raises AnsibleFileNotFound + try: + source_full = self._loader.get_real_file(source_full, decrypt=decrypt) + except AnsibleFileNotFound as e: + result['failed'] = True + result['msg'] = "could not find src=%s, %s" % (source_full, to_text(e)) + return result + + # Get the local mode and set if user wanted it preserved + # https://github.com/ansible/ansible-modules-core/issues/1124 + lmode = None + if self._task.args.get('mode', None) == 'preserve': + lmode = '0%03o' % stat.S_IMODE(os.stat(source_full).st_mode) + + # This is kind of optimization - if user told us destination is + # dir, do path manipulation right away, otherwise we still check + # for dest being a dir via remote call below. + if self._connection._shell.path_has_trailing_slash(dest): + dest_file = self._connection._shell.join_path(dest, source_rel) + else: + dest_file = dest + + # Attempt to get remote file info + dest_status = self._execute_remote_stat(dest_file, all_vars=task_vars, follow=follow, checksum=force) + + if dest_status['exists'] and dest_status['isdir']: + # The dest is a directory. + if content is not None: + # If source was defined as content remove the temporary file and fail out. + self._remove_tempfile_if_content_defined(content, content_tempfile) + result['failed'] = True + result['msg'] = "can not use content with a dir as dest" + return result + else: + # Append the relative source location to the destination and get remote stats again + dest_file = self._connection._shell.join_path(dest, source_rel) + dest_status = self._execute_remote_stat(dest_file, all_vars=task_vars, follow=follow, checksum=force) + + if dest_status['exists'] and not force: + # remote_file exists so continue to next iteration. + return None + + # Generate a hash of the local file. + local_checksum = checksum(source_full) + + if local_checksum != dest_status['checksum']: + # The checksums don't match and we will change or error out. + + if self._play_context.diff and not raw: + result['diff'].append(self._get_diff_data(dest_file, source_full, task_vars)) + + if self._play_context.check_mode: + self._remove_tempfile_if_content_defined(content, content_tempfile) + result['changed'] = True + return result + + # Define a remote directory that we will copy the file to. + tmp_src = self._connection._shell.join_path(self._connection._shell.tmpdir, 'source') + + remote_path = None + + if not raw: + remote_path = self._transfer_file(source_full, tmp_src) + else: + self._transfer_file(source_full, dest_file) + + # We have copied the file remotely and no longer require our content_tempfile + self._remove_tempfile_if_content_defined(content, content_tempfile) + self._loader.cleanup_tmp_file(source_full) + + # FIXME: I don't think this is needed when PIPELINING=0 because the source is created + # world readable. Access to the directory itself is controlled via fixup_perms2() as + # part of executing the module. Check that umask with scp/sftp/piped doesn't cause + # a problem before acting on this idea. (This idea would save a round-trip) + # fix file permissions when the copy is done as a different user + if remote_path: + self._fixup_perms2((self._connection._shell.tmpdir, remote_path)) + + if raw: + # Continue to next iteration if raw is defined. + return None + + # Run the copy module + + # src and dest here come after original and override them + # we pass dest only to make sure it includes trailing slash in case of recursive copy + new_module_args = _create_remote_copy_args(self._task.args) + new_module_args.update( + dict( + src=tmp_src, + dest=dest, + _original_basename=source_rel, + follow=follow + ) + ) + if not self._task.args.get('checksum'): + new_module_args['checksum'] = local_checksum + + if lmode: + new_module_args['mode'] = lmode + + module_return = self._execute_module(module_name='ansible.legacy.copy', module_args=new_module_args, task_vars=task_vars) + + else: + # no need to transfer the file, already correct hash, but still need to call + # the file module in case we want to change attributes + self._remove_tempfile_if_content_defined(content, content_tempfile) + self._loader.cleanup_tmp_file(source_full) + + if raw: + return None + + # Fix for https://github.com/ansible/ansible-modules-core/issues/1568. + # If checksums match, and follow = True, find out if 'dest' is a link. If so, + # change it to point to the source of the link. + if follow: + dest_status_nofollow = self._execute_remote_stat(dest_file, all_vars=task_vars, follow=False) + if dest_status_nofollow['islnk'] and 'lnk_source' in dest_status_nofollow.keys(): + dest = dest_status_nofollow['lnk_source'] + + # Build temporary module_args. + new_module_args = _create_remote_file_args(self._task.args) + new_module_args.update( + dict( + dest=dest, + _original_basename=source_rel, + recurse=False, + state='file', + ) + ) + # src is sent to the file module in _original_basename, not in src + try: + del new_module_args['src'] + except KeyError: + pass + + if lmode: + new_module_args['mode'] = lmode + + # Execute the file module. + module_return = self._execute_module(module_name='ansible.legacy.file', module_args=new_module_args, task_vars=task_vars) + + if not module_return.get('checksum'): + module_return['checksum'] = local_checksum + + result.update(module_return) + return result + + def _create_content_tempfile(self, content): + ''' Create a tempfile containing defined content ''' + fd, content_tempfile = tempfile.mkstemp(dir=C.DEFAULT_LOCAL_TMP) + f = os.fdopen(fd, 'wb') + content = to_bytes(content) + try: + f.write(content) + except Exception as err: + os.remove(content_tempfile) + raise Exception(err) + finally: + f.close() + return content_tempfile + + def _remove_tempfile_if_content_defined(self, content, content_tempfile): + if content is not None: + os.remove(content_tempfile) + + def run(self, tmp=None, task_vars=None): + ''' handler for file transfer operations ''' + if task_vars is None: + task_vars = dict() + + result = super(ActionModule, self).run(tmp, task_vars) + del tmp # tmp no longer has any effect + + source = self._task.args.get('src', None) + content = self._task.args.get('content', None) + dest = self._task.args.get('dest', None) + remote_src = boolean(self._task.args.get('remote_src', False), strict=False) + local_follow = boolean(self._task.args.get('local_follow', True), strict=False) + + result['failed'] = True + if not source and content is None: + result['msg'] = 'src (or content) is required' + elif not dest: + result['msg'] = 'dest is required' + elif source and content is not None: + result['msg'] = 'src and content are mutually exclusive' + elif content is not None and dest is not None and dest.endswith("/"): + result['msg'] = "can not use content with a dir as dest" + else: + del result['failed'] + + if result.get('failed'): + return self._ensure_invocation(result) + + # Define content_tempfile in case we set it after finding content populated. + content_tempfile = None + + # If content is defined make a tmp file and write the content into it. + if content is not None: + try: + # If content comes to us as a dict it should be decoded json. + # We need to encode it back into a string to write it out. + if isinstance(content, dict) or isinstance(content, list): + content_tempfile = self._create_content_tempfile(json.dumps(content)) + else: + content_tempfile = self._create_content_tempfile(content) + source = content_tempfile + except Exception as err: + result['failed'] = True + result['msg'] = "could not write content temp file: %s" % to_native(err) + return self._ensure_invocation(result) + + # if we have first_available_file in our vars + # look up the files and use the first one we find as src + elif remote_src: + result.update(self._execute_module(module_name='ansible.legacy.copy', task_vars=task_vars)) + return self._ensure_invocation(result) + else: + # find_needle returns a path that may not have a trailing slash on + # a directory so we need to determine that now (we use it just + # like rsync does to figure out whether to include the directory + # or only the files inside the directory + trailing_slash = source.endswith(os.path.sep) + try: + # find in expected paths + source = self._find_needle('files', source) + except AnsibleError as e: + result['failed'] = True + result['msg'] = to_text(e) + result['exception'] = traceback.format_exc() + return self._ensure_invocation(result) + + if trailing_slash != source.endswith(os.path.sep): + if source[-1] == os.path.sep: + source = source[:-1] + else: + source = source + os.path.sep + + # A list of source file tuples (full_path, relative_path) which will try to copy to the destination + source_files = {'files': [], 'directories': [], 'symlinks': []} + + # If source is a directory populate our list else source is a file and translate it to a tuple. + if os.path.isdir(to_bytes(source, errors='surrogate_or_strict')): + # Get a list of the files we want to replicate on the remote side + source_files = _walk_dirs(source, local_follow=local_follow, + trailing_slash_detector=self._connection._shell.path_has_trailing_slash) + + # If it's recursive copy, destination is always a dir, + # explicitly mark it so (note - copy module relies on this). + if not self._connection._shell.path_has_trailing_slash(dest): + dest = self._connection._shell.join_path(dest, '') + # FIXME: Can we optimize cases where there's only one file, no + # symlinks and any number of directories? In the original code, + # empty directories are not copied.... + else: + source_files['files'] = [(source, os.path.basename(source))] + + changed = False + module_return = dict(changed=False) + + # A register for if we executed a module. + # Used to cut down on command calls when not recursive. + module_executed = False + + # expand any user home dir specifier + dest = self._remote_expand_user(dest) + + implicit_directories = set() + for source_full, source_rel in source_files['files']: + # copy files over. This happens first as directories that have + # a file do not need to be created later + + # We only follow symlinks for files in the non-recursive case + if source_files['directories']: + follow = False + else: + follow = boolean(self._task.args.get('follow', False), strict=False) + + module_return = self._copy_file(source_full, source_rel, content, content_tempfile, dest, task_vars, follow) + if module_return is None: + continue + + if module_return.get('failed'): + result.update(module_return) + return self._ensure_invocation(result) + + paths = os.path.split(source_rel) + dir_path = '' + for dir_component in paths: + os.path.join(dir_path, dir_component) + implicit_directories.add(dir_path) + if 'diff' in result and not result['diff']: + del result['diff'] + module_executed = True + changed = changed or module_return.get('changed', False) + + for src, dest_path in source_files['directories']: + # Find directories that are leaves as they might not have been + # created yet. + if dest_path in implicit_directories: + continue + + # Use file module to create these + new_module_args = _create_remote_file_args(self._task.args) + new_module_args['path'] = os.path.join(dest, dest_path) + new_module_args['state'] = 'directory' + new_module_args['mode'] = self._task.args.get('directory_mode', None) + new_module_args['recurse'] = False + del new_module_args['src'] + + module_return = self._execute_module(module_name='ansible.legacy.file', module_args=new_module_args, task_vars=task_vars) + + if module_return.get('failed'): + result.update(module_return) + return self._ensure_invocation(result) + + module_executed = True + changed = changed or module_return.get('changed', False) + + for target_path, dest_path in source_files['symlinks']: + # Copy symlinks over + new_module_args = _create_remote_file_args(self._task.args) + new_module_args['path'] = os.path.join(dest, dest_path) + new_module_args['src'] = target_path + new_module_args['state'] = 'link' + new_module_args['force'] = True + + # Only follow remote symlinks in the non-recursive case + if source_files['directories']: + new_module_args['follow'] = False + + # file module cannot deal with 'preserve' mode and is meaningless + # for symlinks anyway, so just don't pass it. + if new_module_args.get('mode', None) == 'preserve': + new_module_args.pop('mode') + + module_return = self._execute_module(module_name='ansible.legacy.file', module_args=new_module_args, task_vars=task_vars) + module_executed = True + + if module_return.get('failed'): + result.update(module_return) + return self._ensure_invocation(result) + + changed = changed or module_return.get('changed', False) + + if module_executed and len(source_files['files']) == 1: + result.update(module_return) + + # the file module returns the file path as 'path', but + # the copy module uses 'dest', so add it if it's not there + if 'path' in result and 'dest' not in result: + result['dest'] = result['path'] + else: + result.update(dict(dest=dest, src=source, changed=changed)) + + # Delete tmp path + self._remove_tmp_path(self._connection._shell.tmpdir) + + return self._ensure_invocation(result) diff --git a/lib/ansible/plugins/action/debug.py b/lib/ansible/plugins/action/debug.py new file mode 100644 index 0000000..2584fd3 --- /dev/null +++ b/lib/ansible/plugins/action/debug.py @@ -0,0 +1,80 @@ +# Copyright 2012, Dag Wieers <dag@wieers.com> +# Copyright 2016, Toshio Kuratomi <tkuratomi@ansible.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible.errors import AnsibleUndefinedVariable +from ansible.module_utils.six import string_types +from ansible.module_utils._text import to_text +from ansible.plugins.action import ActionBase + + +class ActionModule(ActionBase): + ''' Print statements during execution ''' + + TRANSFERS_FILES = False + _VALID_ARGS = frozenset(('msg', 'var', 'verbosity')) + + def run(self, tmp=None, task_vars=None): + if task_vars is None: + task_vars = dict() + + if 'msg' in self._task.args and 'var' in self._task.args: + return {"failed": True, "msg": "'msg' and 'var' are incompatible options"} + + result = super(ActionModule, self).run(tmp, task_vars) + del tmp # tmp no longer has any effect + + # get task verbosity + verbosity = int(self._task.args.get('verbosity', 0)) + + if verbosity <= self._display.verbosity: + if 'msg' in self._task.args: + result['msg'] = self._task.args['msg'] + + elif 'var' in self._task.args: + try: + results = self._templar.template(self._task.args['var'], convert_bare=True, fail_on_undefined=True) + if results == self._task.args['var']: + # if results is not str/unicode type, raise an exception + if not isinstance(results, string_types): + raise AnsibleUndefinedVariable + # If var name is same as result, try to template it + results = self._templar.template("{{" + results + "}}", convert_bare=True, fail_on_undefined=True) + except AnsibleUndefinedVariable as e: + results = u"VARIABLE IS NOT DEFINED!" + if self._display.verbosity > 0: + results += u": %s" % to_text(e) + + if isinstance(self._task.args['var'], (list, dict)): + # If var is a list or dict, use the type as key to display + result[to_text(type(self._task.args['var']))] = results + else: + result[self._task.args['var']] = results + else: + result['msg'] = 'Hello world!' + + # force flag to make debug output module always verbose + result['_ansible_verbose_always'] = True + else: + result['skipped_reason'] = "Verbosity threshold not met." + result['skipped'] = True + + result['failed'] = False + + return result diff --git a/lib/ansible/plugins/action/fail.py b/lib/ansible/plugins/action/fail.py new file mode 100644 index 0000000..8d3450c --- /dev/null +++ b/lib/ansible/plugins/action/fail.py @@ -0,0 +1,43 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# (c) 2012, Dag Wieers <dag@wieers.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible.plugins.action import ActionBase + + +class ActionModule(ActionBase): + ''' Fail with custom message ''' + + TRANSFERS_FILES = False + _VALID_ARGS = frozenset(('msg',)) + + def run(self, tmp=None, task_vars=None): + if task_vars is None: + task_vars = dict() + + result = super(ActionModule, self).run(tmp, task_vars) + del tmp # tmp no longer has any effect + + msg = 'Failed as requested from task' + if self._task.args and 'msg' in self._task.args: + msg = self._task.args.get('msg') + + result['failed'] = True + result['msg'] = msg + return result diff --git a/lib/ansible/plugins/action/fetch.py b/lib/ansible/plugins/action/fetch.py new file mode 100644 index 0000000..992ba5a --- /dev/null +++ b/lib/ansible/plugins/action/fetch.py @@ -0,0 +1,207 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import os +import base64 +from ansible.errors import AnsibleError, AnsibleActionFail, AnsibleActionSkip +from ansible.module_utils.common.text.converters import to_bytes, to_text +from ansible.module_utils.six import string_types +from ansible.module_utils.parsing.convert_bool import boolean +from ansible.plugins.action import ActionBase +from ansible.utils.display import Display +from ansible.utils.hashing import checksum, checksum_s, md5, secure_hash +from ansible.utils.path import makedirs_safe, is_subpath + +display = Display() + + +class ActionModule(ActionBase): + + def run(self, tmp=None, task_vars=None): + ''' handler for fetch operations ''' + if task_vars is None: + task_vars = dict() + + result = super(ActionModule, self).run(tmp, task_vars) + del tmp # tmp no longer has any effect + + try: + if self._play_context.check_mode: + raise AnsibleActionSkip('check mode not (yet) supported for this module') + + source = self._task.args.get('src', None) + original_dest = dest = self._task.args.get('dest', None) + flat = boolean(self._task.args.get('flat'), strict=False) + fail_on_missing = boolean(self._task.args.get('fail_on_missing', True), strict=False) + validate_checksum = boolean(self._task.args.get('validate_checksum', True), strict=False) + + msg = '' + # validate source and dest are strings FIXME: use basic.py and module specs + if not isinstance(source, string_types): + msg = "Invalid type supplied for source option, it must be a string" + + if not isinstance(dest, string_types): + msg = "Invalid type supplied for dest option, it must be a string" + + if source is None or dest is None: + msg = "src and dest are required" + + if msg: + raise AnsibleActionFail(msg) + + source = self._connection._shell.join_path(source) + source = self._remote_expand_user(source) + + remote_stat = {} + remote_checksum = None + if not self._connection.become: + # Get checksum for the remote file. Don't bother if using become as slurp will be used. + # Follow symlinks because fetch always follows symlinks + try: + remote_stat = self._execute_remote_stat(source, all_vars=task_vars, follow=True) + except AnsibleError as ae: + result['changed'] = False + result['file'] = source + if fail_on_missing: + result['failed'] = True + result['msg'] = to_text(ae) + else: + result['msg'] = "%s, ignored" % to_text(ae, errors='surrogate_or_replace') + + return result + + remote_checksum = remote_stat.get('checksum') + if remote_stat.get('exists'): + if remote_stat.get('isdir'): + result['failed'] = True + result['changed'] = False + result['msg'] = "remote file is a directory, fetch cannot work on directories" + + # Historically, these don't fail because you may want to transfer + # a log file that possibly MAY exist but keep going to fetch other + # log files. Today, this is better achieved by adding + # ignore_errors or failed_when to the task. Control the behaviour + # via fail_when_missing + if not fail_on_missing: + result['msg'] += ", not transferring, ignored" + del result['changed'] + del result['failed'] + + return result + + # use slurp if permissions are lacking or privilege escalation is needed + remote_data = None + if remote_checksum in (None, '1', ''): + slurpres = self._execute_module(module_name='ansible.legacy.slurp', module_args=dict(src=source), task_vars=task_vars) + if slurpres.get('failed'): + if not fail_on_missing: + result['file'] = source + result['changed'] = False + else: + result.update(slurpres) + + if 'not found' in slurpres.get('msg', ''): + result['msg'] = "the remote file does not exist, not transferring, ignored" + elif slurpres.get('msg', '').startswith('source is a directory'): + result['msg'] = "remote file is a directory, fetch cannot work on directories" + + return result + else: + if slurpres['encoding'] == 'base64': + remote_data = base64.b64decode(slurpres['content']) + if remote_data is not None: + remote_checksum = checksum_s(remote_data) + + # calculate the destination name + if os.path.sep not in self._connection._shell.join_path('a', ''): + source = self._connection._shell._unquote(source) + source_local = source.replace('\\', '/') + else: + source_local = source + + # ensure we only use file name, avoid relative paths + if not is_subpath(dest, original_dest): + # TODO: ? dest = os.path.expanduser(dest.replace(('../',''))) + raise AnsibleActionFail("Detected directory traversal, expected to be contained in '%s' but got '%s'" % (original_dest, dest)) + + if flat: + if os.path.isdir(to_bytes(dest, errors='surrogate_or_strict')) and not dest.endswith(os.sep): + raise AnsibleActionFail("dest is an existing directory, use a trailing slash if you want to fetch src into that directory") + if dest.endswith(os.sep): + # if the path ends with "/", we'll use the source filename as the + # destination filename + base = os.path.basename(source_local) + dest = os.path.join(dest, base) + if not dest.startswith("/"): + # if dest does not start with "/", we'll assume a relative path + dest = self._loader.path_dwim(dest) + else: + # files are saved in dest dir, with a subdir for each host, then the filename + if 'inventory_hostname' in task_vars: + target_name = task_vars['inventory_hostname'] + else: + target_name = self._play_context.remote_addr + dest = "%s/%s/%s" % (self._loader.path_dwim(dest), target_name, source_local) + + dest = os.path.normpath(dest) + + # calculate checksum for the local file + local_checksum = checksum(dest) + + if remote_checksum != local_checksum: + # create the containing directories, if needed + makedirs_safe(os.path.dirname(dest)) + + # fetch the file and check for changes + if remote_data is None: + self._connection.fetch_file(source, dest) + else: + try: + f = open(to_bytes(dest, errors='surrogate_or_strict'), 'wb') + f.write(remote_data) + f.close() + except (IOError, OSError) as e: + raise AnsibleActionFail("Failed to fetch the file: %s" % e) + new_checksum = secure_hash(dest) + # For backwards compatibility. We'll return None on FIPS enabled systems + try: + new_md5 = md5(dest) + except ValueError: + new_md5 = None + + if validate_checksum and new_checksum != remote_checksum: + result.update(dict(failed=True, md5sum=new_md5, + msg="checksum mismatch", file=source, dest=dest, remote_md5sum=None, + checksum=new_checksum, remote_checksum=remote_checksum)) + else: + result.update({'changed': True, 'md5sum': new_md5, 'dest': dest, + 'remote_md5sum': None, 'checksum': new_checksum, + 'remote_checksum': remote_checksum}) + else: + # For backwards compatibility. We'll return None on FIPS enabled systems + try: + local_md5 = md5(dest) + except ValueError: + local_md5 = None + result.update(dict(changed=False, md5sum=local_md5, file=source, dest=dest, checksum=local_checksum)) + + finally: + self._remove_tmp_path(self._connection._shell.tmpdir) + + return result diff --git a/lib/ansible/plugins/action/gather_facts.py b/lib/ansible/plugins/action/gather_facts.py new file mode 100644 index 0000000..3ff7beb --- /dev/null +++ b/lib/ansible/plugins/action/gather_facts.py @@ -0,0 +1,152 @@ +# Copyright (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import os +import time + +from ansible import constants as C +from ansible.executor.module_common import get_action_args_with_defaults +from ansible.module_utils.parsing.convert_bool import boolean +from ansible.plugins.action import ActionBase +from ansible.utils.vars import merge_hash + + +class ActionModule(ActionBase): + + def _get_module_args(self, fact_module, task_vars): + + mod_args = self._task.args.copy() + + # deal with 'setup specific arguments' + if fact_module not in C._ACTION_SETUP: + # TODO: remove in favor of controller side argspec detecing valid arguments + # network facts modules must support gather_subset + try: + name = self._connection.ansible_name.removeprefix('ansible.netcommon.') + except AttributeError: + name = self._connection._load_name.split('.')[-1] + if name not in ('network_cli', 'httpapi', 'netconf'): + subset = mod_args.pop('gather_subset', None) + if subset not in ('all', ['all']): + self._display.warning('Ignoring subset(%s) for %s' % (subset, fact_module)) + + timeout = mod_args.pop('gather_timeout', None) + if timeout is not None: + self._display.warning('Ignoring timeout(%s) for %s' % (timeout, fact_module)) + + fact_filter = mod_args.pop('filter', None) + if fact_filter is not None: + self._display.warning('Ignoring filter(%s) for %s' % (fact_filter, fact_module)) + + # Strip out keys with ``None`` values, effectively mimicking ``omit`` behavior + # This ensures we don't pass a ``None`` value as an argument expecting a specific type + mod_args = dict((k, v) for k, v in mod_args.items() if v is not None) + + # handle module defaults + resolved_fact_module = self._shared_loader_obj.module_loader.find_plugin_with_context( + fact_module, collection_list=self._task.collections + ).resolved_fqcn + + mod_args = get_action_args_with_defaults( + resolved_fact_module, mod_args, self._task.module_defaults, self._templar, + action_groups=self._task._parent._play._action_groups + ) + + return mod_args + + def _combine_task_result(self, result, task_result): + filtered_res = { + 'ansible_facts': task_result.get('ansible_facts', {}), + 'warnings': task_result.get('warnings', []), + 'deprecations': task_result.get('deprecations', []), + } + + # on conflict the last plugin processed wins, but try to do deep merge and append to lists. + return merge_hash(result, filtered_res, list_merge='append_rp') + + def run(self, tmp=None, task_vars=None): + + self._supports_check_mode = True + + result = super(ActionModule, self).run(tmp, task_vars) + result['ansible_facts'] = {} + + # copy the value with list() so we don't mutate the config + modules = list(C.config.get_config_value('FACTS_MODULES', variables=task_vars)) + + parallel = task_vars.pop('ansible_facts_parallel', self._task.args.pop('parallel', None)) + if 'smart' in modules: + connection_map = C.config.get_config_value('CONNECTION_FACTS_MODULES', variables=task_vars) + network_os = self._task.args.get('network_os', task_vars.get('ansible_network_os', task_vars.get('ansible_facts', {}).get('network_os'))) + modules.extend([connection_map.get(network_os or self._connection.ansible_name, 'ansible.legacy.setup')]) + modules.pop(modules.index('smart')) + + failed = {} + skipped = {} + + if parallel is None and len(modules) >= 1: + parallel = True + else: + parallel = boolean(parallel) + + if parallel: + # serially execute each module + for fact_module in modules: + # just one module, no need for fancy async + mod_args = self._get_module_args(fact_module, task_vars) + res = self._execute_module(module_name=fact_module, module_args=mod_args, task_vars=task_vars, wrap_async=False) + if res.get('failed', False): + failed[fact_module] = res + elif res.get('skipped', False): + skipped[fact_module] = res + else: + result = self._combine_task_result(result, res) + + self._remove_tmp_path(self._connection._shell.tmpdir) + else: + # do it async + jobs = {} + for fact_module in modules: + mod_args = self._get_module_args(fact_module, task_vars) + self._display.vvvv("Running %s" % fact_module) + jobs[fact_module] = (self._execute_module(module_name=fact_module, module_args=mod_args, task_vars=task_vars, wrap_async=True)) + + while jobs: + for module in jobs: + poll_args = {'jid': jobs[module]['ansible_job_id'], '_async_dir': os.path.dirname(jobs[module]['results_file'])} + res = self._execute_module(module_name='ansible.legacy.async_status', module_args=poll_args, task_vars=task_vars, wrap_async=False) + if res.get('finished', 0) == 1: + if res.get('failed', False): + failed[module] = res + elif res.get('skipped', False): + skipped[module] = res + else: + result = self._combine_task_result(result, res) + del jobs[module] + break + else: + time.sleep(0.1) + else: + time.sleep(0.5) + + if skipped: + result['msg'] = "The following modules were skipped: %s\n" % (', '.join(skipped.keys())) + result['skipped_modules'] = skipped + if len(skipped) == len(modules): + result['skipped'] = True + + if failed: + result['failed'] = True + result['msg'] = "The following modules failed to execute: %s\n" % (', '.join(failed.keys())) + result['failed_modules'] = failed + + # tell executor facts were gathered + result['ansible_facts']['_ansible_facts_gathered'] = True + + # hack to keep --verbose from showing all the setup module result + result['_ansible_verbose_override'] = True + + return result diff --git a/lib/ansible/plugins/action/group_by.py b/lib/ansible/plugins/action/group_by.py new file mode 100644 index 0000000..0958ad8 --- /dev/null +++ b/lib/ansible/plugins/action/group_by.py @@ -0,0 +1,51 @@ +# Copyright 2012, Jeroen Hoekx <jeroen@hoekx.be> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible.plugins.action import ActionBase +from ansible.module_utils.six import string_types + + +class ActionModule(ActionBase): + ''' Create inventory groups based on variables ''' + + # We need to be able to modify the inventory + TRANSFERS_FILES = False + _VALID_ARGS = frozenset(('key', 'parents')) + + def run(self, tmp=None, task_vars=None): + if task_vars is None: + task_vars = dict() + + result = super(ActionModule, self).run(tmp, task_vars) + del tmp # tmp no longer has any effect + + if 'key' not in self._task.args: + result['failed'] = True + result['msg'] = "the 'key' param is required when using group_by" + return result + + group_name = self._task.args.get('key') + parent_groups = self._task.args.get('parents', ['all']) + if isinstance(parent_groups, string_types): + parent_groups = [parent_groups] + + result['changed'] = False + result['add_group'] = group_name.replace(' ', '-') + result['parent_groups'] = [name.replace(' ', '-') for name in parent_groups] + return result diff --git a/lib/ansible/plugins/action/include_vars.py b/lib/ansible/plugins/action/include_vars.py new file mode 100644 index 0000000..3c3cb9e --- /dev/null +++ b/lib/ansible/plugins/action/include_vars.py @@ -0,0 +1,290 @@ +# Copyright: (c) 2016, Allen Sanabria <asanabria@linuxdynasty.org> +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from os import path, walk +import re + +import ansible.constants as C +from ansible.errors import AnsibleError +from ansible.module_utils.six import string_types +from ansible.module_utils._text import to_native, to_text +from ansible.plugins.action import ActionBase +from ansible.utils.vars import combine_vars + + +class ActionModule(ActionBase): + + TRANSFERS_FILES = False + + VALID_FILE_EXTENSIONS = ['yaml', 'yml', 'json'] + VALID_DIR_ARGUMENTS = ['dir', 'depth', 'files_matching', 'ignore_files', 'extensions', 'ignore_unknown_extensions'] + VALID_FILE_ARGUMENTS = ['file', '_raw_params'] + VALID_ALL = ['name', 'hash_behaviour'] + + def _set_dir_defaults(self): + if not self.depth: + self.depth = 0 + + if self.files_matching: + self.matcher = re.compile(r'{0}'.format(self.files_matching)) + else: + self.matcher = None + + if not self.ignore_files: + self.ignore_files = list() + + if isinstance(self.ignore_files, string_types): + self.ignore_files = self.ignore_files.split() + + elif isinstance(self.ignore_files, dict): + return { + 'failed': True, + 'message': '{0} must be a list'.format(self.ignore_files) + } + + def _set_args(self): + """ Set instance variables based on the arguments that were passed """ + + self.hash_behaviour = self._task.args.get('hash_behaviour', None) + self.return_results_as_name = self._task.args.get('name', None) + self.source_dir = self._task.args.get('dir', None) + self.source_file = self._task.args.get('file', None) + if not self.source_dir and not self.source_file: + self.source_file = self._task.args.get('_raw_params') + if self.source_file: + self.source_file = self.source_file.rstrip('\n') + + self.depth = self._task.args.get('depth', None) + self.files_matching = self._task.args.get('files_matching', None) + self.ignore_unknown_extensions = self._task.args.get('ignore_unknown_extensions', False) + self.ignore_files = self._task.args.get('ignore_files', None) + self.valid_extensions = self._task.args.get('extensions', self.VALID_FILE_EXTENSIONS) + + # convert/validate extensions list + if isinstance(self.valid_extensions, string_types): + self.valid_extensions = list(self.valid_extensions) + if not isinstance(self.valid_extensions, list): + raise AnsibleError('Invalid type for "extensions" option, it must be a list') + + def run(self, tmp=None, task_vars=None): + """ Load yml files recursively from a directory. + """ + del tmp # tmp no longer has any effect + + if task_vars is None: + task_vars = dict() + + self.show_content = True + self.included_files = [] + + # Validate arguments + dirs = 0 + files = 0 + for arg in self._task.args: + if arg in self.VALID_DIR_ARGUMENTS: + dirs += 1 + elif arg in self.VALID_FILE_ARGUMENTS: + files += 1 + elif arg in self.VALID_ALL: + pass + else: + raise AnsibleError('{0} is not a valid option in include_vars'.format(to_native(arg))) + + if dirs and files: + raise AnsibleError("You are mixing file only and dir only arguments, these are incompatible") + + # set internal vars from args + self._set_args() + + results = dict() + failed = False + if self.source_dir: + self._set_dir_defaults() + self._set_root_dir() + if not path.exists(self.source_dir): + failed = True + err_msg = ('{0} directory does not exist'.format(to_native(self.source_dir))) + elif not path.isdir(self.source_dir): + failed = True + err_msg = ('{0} is not a directory'.format(to_native(self.source_dir))) + else: + for root_dir, filenames in self._traverse_dir_depth(): + failed, err_msg, updated_results = (self._load_files_in_dir(root_dir, filenames)) + if failed: + break + results.update(updated_results) + else: + try: + self.source_file = self._find_needle('vars', self.source_file) + failed, err_msg, updated_results = ( + self._load_files(self.source_file) + ) + if not failed: + results.update(updated_results) + + except AnsibleError as e: + failed = True + err_msg = to_native(e) + + if self.return_results_as_name: + scope = dict() + scope[self.return_results_as_name] = results + results = scope + + result = super(ActionModule, self).run(task_vars=task_vars) + + if failed: + result['failed'] = failed + result['message'] = err_msg + elif self.hash_behaviour is not None and self.hash_behaviour != C.DEFAULT_HASH_BEHAVIOUR: + merge_hashes = self.hash_behaviour == 'merge' + for key, value in results.items(): + old_value = task_vars.get(key, None) + results[key] = combine_vars(old_value, value, merge=merge_hashes) + + result['ansible_included_var_files'] = self.included_files + result['ansible_facts'] = results + result['_ansible_no_log'] = not self.show_content + + return result + + def _set_root_dir(self): + if self._task._role: + if self.source_dir.split('/')[0] == 'vars': + path_to_use = ( + path.join(self._task._role._role_path, self.source_dir) + ) + if path.exists(path_to_use): + self.source_dir = path_to_use + else: + path_to_use = ( + path.join( + self._task._role._role_path, 'vars', self.source_dir + ) + ) + self.source_dir = path_to_use + else: + if hasattr(self._task._ds, '_data_source'): + current_dir = ( + "/".join(self._task._ds._data_source.split('/')[:-1]) + ) + self.source_dir = path.join(current_dir, self.source_dir) + + def _log_walk(self, error): + self._display.vvv('Issue with walking through "%s": %s' % (to_native(error.filename), to_native(error))) + + def _traverse_dir_depth(self): + """ Recursively iterate over a directory and sort the files in + alphabetical order. Do not iterate pass the set depth. + The default depth is unlimited. + """ + current_depth = 0 + sorted_walk = list(walk(self.source_dir, onerror=self._log_walk)) + sorted_walk.sort(key=lambda x: x[0]) + for current_root, current_dir, current_files in sorted_walk: + current_depth += 1 + if current_depth <= self.depth or self.depth == 0: + current_files.sort() + yield (current_root, current_files) + else: + break + + def _ignore_file(self, filename): + """ Return True if a file matches the list of ignore_files. + Args: + filename (str): The filename that is being matched against. + + Returns: + Boolean + """ + for file_type in self.ignore_files: + try: + if re.search(r'{0}$'.format(file_type), filename): + return True + except Exception: + err_msg = 'Invalid regular expression: {0}'.format(file_type) + raise AnsibleError(err_msg) + return False + + def _is_valid_file_ext(self, source_file): + """ Verify if source file has a valid extension + Args: + source_file (str): The full path of source file or source file. + Returns: + Bool + """ + file_ext = path.splitext(source_file) + return bool(len(file_ext) > 1 and file_ext[-1][1:] in self.valid_extensions) + + def _load_files(self, filename, validate_extensions=False): + """ Loads a file and converts the output into a valid Python dict. + Args: + filename (str): The source file. + + Returns: + Tuple (bool, str, dict) + """ + results = dict() + failed = False + err_msg = '' + if validate_extensions and not self._is_valid_file_ext(filename): + failed = True + err_msg = ('{0} does not have a valid extension: {1}'.format(to_native(filename), ', '.join(self.valid_extensions))) + else: + b_data, show_content = self._loader._get_file_contents(filename) + data = to_text(b_data, errors='surrogate_or_strict') + + self.show_content = show_content + data = self._loader.load(data, file_name=filename, show_content=show_content) + if not data: + data = dict() + if not isinstance(data, dict): + failed = True + err_msg = ('{0} must be stored as a dictionary/hash'.format(to_native(filename))) + else: + self.included_files.append(filename) + results.update(data) + + return failed, err_msg, results + + def _load_files_in_dir(self, root_dir, var_files): + """ Load the found yml files and update/overwrite the dictionary. + Args: + root_dir (str): The base directory of the list of files that is being passed. + var_files: (list): List of files to iterate over and load into a dictionary. + + Returns: + Tuple (bool, str, dict) + """ + results = dict() + failed = False + err_msg = '' + for filename in var_files: + stop_iter = False + # Never include main.yml from a role, as that is the default included by the role + if self._task._role: + if path.join(self._task._role._role_path, filename) == path.join(root_dir, 'vars', 'main.yml'): + stop_iter = True + continue + + filepath = path.join(root_dir, filename) + if self.files_matching: + if not self.matcher.search(filename): + stop_iter = True + + if not stop_iter and not failed: + if self.ignore_unknown_extensions: + if path.exists(filepath) and not self._ignore_file(filename) and self._is_valid_file_ext(filename): + failed, err_msg, loaded_data = self._load_files(filepath, validate_extensions=True) + if not failed: + results.update(loaded_data) + else: + if path.exists(filepath) and not self._ignore_file(filename): + failed, err_msg, loaded_data = self._load_files(filepath, validate_extensions=True) + if not failed: + results.update(loaded_data) + + return failed, err_msg, results diff --git a/lib/ansible/plugins/action/normal.py b/lib/ansible/plugins/action/normal.py new file mode 100644 index 0000000..cb91521 --- /dev/null +++ b/lib/ansible/plugins/action/normal.py @@ -0,0 +1,59 @@ +# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible import constants as C +from ansible.plugins.action import ActionBase +from ansible.utils.vars import merge_hash + + +class ActionModule(ActionBase): + + def run(self, tmp=None, task_vars=None): + + # individual modules might disagree but as the generic the action plugin, pass at this point. + self._supports_check_mode = True + self._supports_async = True + + result = super(ActionModule, self).run(tmp, task_vars) + del tmp # tmp no longer has any effect + + if not result.get('skipped'): + + if result.get('invocation', {}).get('module_args'): + # avoid passing to modules in case of no_log + # should not be set anymore but here for backwards compatibility + del result['invocation']['module_args'] + + # FUTURE: better to let _execute_module calculate this internally? + wrap_async = self._task.async_val and not self._connection.has_native_async + + # do work! + result = merge_hash(result, self._execute_module(task_vars=task_vars, wrap_async=wrap_async)) + + # hack to keep --verbose from showing all the setup module result + # moved from setup module as now we filter out all _ansible_ from result + # FIXME: is this still accurate with gather_facts etc, or does it need support for FQ and other names? + if self._task.action in C._ACTION_SETUP: + result['_ansible_verbose_override'] = True + + if not wrap_async: + # remove a temporary path we created + self._remove_tmp_path(self._connection._shell.tmpdir) + + return result diff --git a/lib/ansible/plugins/action/package.py b/lib/ansible/plugins/action/package.py new file mode 100644 index 0000000..6c43659 --- /dev/null +++ b/lib/ansible/plugins/action/package.py @@ -0,0 +1,96 @@ +# (c) 2015, Ansible Inc, +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible.errors import AnsibleAction, AnsibleActionFail +from ansible.executor.module_common import get_action_args_with_defaults +from ansible.module_utils.facts.system.pkg_mgr import PKG_MGRS +from ansible.plugins.action import ActionBase +from ansible.utils.display import Display + +display = Display() + + +class ActionModule(ActionBase): + + TRANSFERS_FILES = False + + BUILTIN_PKG_MGR_MODULES = {manager['name'] for manager in PKG_MGRS} + + def run(self, tmp=None, task_vars=None): + ''' handler for package operations ''' + + self._supports_check_mode = True + self._supports_async = True + + result = super(ActionModule, self).run(tmp, task_vars) + del tmp # tmp no longer has any effect + + module = self._task.args.get('use', 'auto') + + if module == 'auto': + try: + if self._task.delegate_to: # if we delegate, we should use delegated host's facts + module = self._templar.template("{{hostvars['%s']['ansible_facts']['pkg_mgr']}}" % self._task.delegate_to) + else: + module = self._templar.template('{{ansible_facts.pkg_mgr}}') + except Exception: + pass # could not get it from template! + + try: + if module == 'auto': + facts = self._execute_module( + module_name='ansible.legacy.setup', + module_args=dict(filter='ansible_pkg_mgr', gather_subset='!all'), + task_vars=task_vars) + display.debug("Facts %s" % facts) + module = facts.get('ansible_facts', {}).get('ansible_pkg_mgr', 'auto') + + if module != 'auto': + if not self._shared_loader_obj.module_loader.has_plugin(module): + raise AnsibleActionFail('Could not find a module for %s.' % module) + else: + # run the 'package' module + new_module_args = self._task.args.copy() + if 'use' in new_module_args: + del new_module_args['use'] + + # get defaults for specific module + context = self._shared_loader_obj.module_loader.find_plugin_with_context(module, collection_list=self._task.collections) + new_module_args = get_action_args_with_defaults( + context.resolved_fqcn, new_module_args, self._task.module_defaults, self._templar, + action_groups=self._task._parent._play._action_groups + ) + + if module in self.BUILTIN_PKG_MGR_MODULES: + # prefix with ansible.legacy to eliminate external collisions while still allowing library/ override + module = 'ansible.legacy.' + module + + display.vvvv("Running %s" % module) + result.update(self._execute_module(module_name=module, module_args=new_module_args, task_vars=task_vars, wrap_async=self._task.async_val)) + else: + raise AnsibleActionFail('Could not detect which package manager to use. Try gathering facts or setting the "use" option.') + + except AnsibleAction as e: + result.update(e.result) + finally: + if not self._task.async_val: + # remove a temporary path we created + self._remove_tmp_path(self._connection._shell.tmpdir) + + return result diff --git a/lib/ansible/plugins/action/pause.py b/lib/ansible/plugins/action/pause.py new file mode 100644 index 0000000..4c98cbb --- /dev/null +++ b/lib/ansible/plugins/action/pause.py @@ -0,0 +1,311 @@ +# Copyright 2012, Tim Bielawa <tbielawa@redhat.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import datetime +import signal +import sys +import termios +import time +import tty + +from os import ( + getpgrp, + isatty, + tcgetpgrp, +) +from ansible.errors import AnsibleError +from ansible.module_utils._text import to_text, to_native +from ansible.module_utils.parsing.convert_bool import boolean +from ansible.plugins.action import ActionBase +from ansible.utils.display import Display + +display = Display() + +try: + import curses + import io + + # Nest the try except since curses.error is not available if curses did not import + try: + curses.setupterm() + HAS_CURSES = True + except (curses.error, TypeError, io.UnsupportedOperation): + HAS_CURSES = False +except ImportError: + HAS_CURSES = False + +MOVE_TO_BOL = b'\r' +CLEAR_TO_EOL = b'\x1b[K' +if HAS_CURSES: + # curses.tigetstr() returns None in some circumstances + MOVE_TO_BOL = curses.tigetstr('cr') or MOVE_TO_BOL + CLEAR_TO_EOL = curses.tigetstr('el') or CLEAR_TO_EOL + + +def setraw(fd, when=termios.TCSAFLUSH): + """Put terminal into a raw mode. + + Copied from ``tty`` from CPython 3.11.0, and modified to not remove OPOST from OFLAG + + OPOST is kept to prevent an issue with multi line prompts from being corrupted now that display + is proxied via the queue from forks. The problem is a race condition, in that we proxy the display + over the fork, but before it can be displayed, this plugin will have continued executing, potentially + setting stdout and stdin to raw which remove output post processing that commonly converts NL to CRLF + """ + mode = termios.tcgetattr(fd) + mode[tty.IFLAG] = mode[tty.IFLAG] & ~(termios.BRKINT | termios.ICRNL | termios.INPCK | termios.ISTRIP | termios.IXON) + # mode[tty.OFLAG] = mode[tty.OFLAG] & ~(termios.OPOST) + mode[tty.CFLAG] = mode[tty.CFLAG] & ~(termios.CSIZE | termios.PARENB) + mode[tty.CFLAG] = mode[tty.CFLAG] | termios.CS8 + mode[tty.LFLAG] = mode[tty.LFLAG] & ~(termios.ECHO | termios.ICANON | termios.IEXTEN | termios.ISIG) + mode[tty.CC][termios.VMIN] = 1 + mode[tty.CC][termios.VTIME] = 0 + termios.tcsetattr(fd, when, mode) + + +class AnsibleTimeoutExceeded(Exception): + pass + + +def timeout_handler(signum, frame): + raise AnsibleTimeoutExceeded + + +def clear_line(stdout): + stdout.write(b'\x1b[%s' % MOVE_TO_BOL) + stdout.write(b'\x1b[%s' % CLEAR_TO_EOL) + + +def is_interactive(fd=None): + if fd is None: + return False + + if isatty(fd): + # Compare the current process group to the process group associated + # with terminal of the given file descriptor to determine if the process + # is running in the background. + return getpgrp() == tcgetpgrp(fd) + else: + return False + + +class ActionModule(ActionBase): + ''' pauses execution for a length or time, or until input is received ''' + + BYPASS_HOST_LOOP = True + + def run(self, tmp=None, task_vars=None): + ''' run the pause action module ''' + if task_vars is None: + task_vars = dict() + + result = super(ActionModule, self).run(tmp, task_vars) + del tmp # tmp no longer has any effect + + validation_result, new_module_args = self.validate_argument_spec( + argument_spec={ + 'echo': {'type': 'bool', 'default': True}, + 'minutes': {'type': int}, # Don't break backwards compat, allow floats, by using int callable + 'seconds': {'type': int}, # Don't break backwards compat, allow floats, by using int callable + 'prompt': {'type': 'str'}, + }, + mutually_exclusive=( + ('minutes', 'seconds'), + ), + ) + + duration_unit = 'minutes' + prompt = None + seconds = None + echo = new_module_args['echo'] + echo_prompt = '' + result.update(dict( + changed=False, + rc=0, + stderr='', + stdout='', + start=None, + stop=None, + delta=None, + echo=echo + )) + + # Add a note saying the output is hidden if echo is disabled + if not echo: + echo_prompt = ' (output is hidden)' + + if new_module_args['prompt']: + prompt = "[%s]\n%s%s:" % (self._task.get_name().strip(), new_module_args['prompt'], echo_prompt) + else: + # If no custom prompt is specified, set a default prompt + prompt = "[%s]\n%s%s:" % (self._task.get_name().strip(), 'Press enter to continue, Ctrl+C to interrupt', echo_prompt) + + if new_module_args['minutes'] is not None: + seconds = new_module_args['minutes'] * 60 + elif new_module_args['seconds'] is not None: + seconds = new_module_args['seconds'] + duration_unit = 'seconds' + + ######################################################################## + # Begin the hard work! + + start = time.time() + result['start'] = to_text(datetime.datetime.now()) + result['user_input'] = b'' + + stdin_fd = None + old_settings = None + try: + if seconds is not None: + if seconds < 1: + seconds = 1 + + # setup the alarm handler + signal.signal(signal.SIGALRM, timeout_handler) + signal.alarm(seconds) + + # show the timer and control prompts + display.display("Pausing for %d seconds%s" % (seconds, echo_prompt)) + display.display("(ctrl+C then 'C' = continue early, ctrl+C then 'A' = abort)\r"), + + # show the prompt specified in the task + if new_module_args['prompt']: + display.display(prompt) + + else: + display.display(prompt) + + # save the attributes on the existing (duped) stdin so + # that we can restore them later after we set raw mode + stdin_fd = None + stdout_fd = None + try: + stdin = self._connection._new_stdin.buffer + stdout = sys.stdout.buffer + stdin_fd = stdin.fileno() + stdout_fd = stdout.fileno() + except (ValueError, AttributeError): + # ValueError: someone is using a closed file descriptor as stdin + # AttributeError: someone is using a null file descriptor as stdin on windoze + stdin = None + interactive = is_interactive(stdin_fd) + if interactive: + # grab actual Ctrl+C sequence + try: + intr = termios.tcgetattr(stdin_fd)[6][termios.VINTR] + except Exception: + # unsupported/not present, use default + intr = b'\x03' # value for Ctrl+C + + # get backspace sequences + try: + backspace = termios.tcgetattr(stdin_fd)[6][termios.VERASE] + except Exception: + backspace = [b'\x7f', b'\x08'] + + old_settings = termios.tcgetattr(stdin_fd) + setraw(stdin_fd) + + # Only set stdout to raw mode if it is a TTY. This is needed when redirecting + # stdout to a file since a file cannot be set to raw mode. + if isatty(stdout_fd): + setraw(stdout_fd) + + # Only echo input if no timeout is specified + if not seconds and echo: + new_settings = termios.tcgetattr(stdin_fd) + new_settings[3] = new_settings[3] | termios.ECHO + termios.tcsetattr(stdin_fd, termios.TCSANOW, new_settings) + + # flush the buffer to make sure no previous key presses + # are read in below + termios.tcflush(stdin, termios.TCIFLUSH) + + while True: + if not interactive: + if seconds is None: + display.warning("Not waiting for response to prompt as stdin is not interactive") + if seconds is not None: + # Give the signal handler enough time to timeout + time.sleep(seconds + 1) + break + + try: + key_pressed = stdin.read(1) + + if key_pressed == intr: # value for Ctrl+C + clear_line(stdout) + raise KeyboardInterrupt + + if not seconds: + # read key presses and act accordingly + if key_pressed in (b'\r', b'\n'): + clear_line(stdout) + break + elif key_pressed in backspace: + # delete a character if backspace is pressed + result['user_input'] = result['user_input'][:-1] + clear_line(stdout) + if echo: + stdout.write(result['user_input']) + stdout.flush() + else: + result['user_input'] += key_pressed + + except KeyboardInterrupt: + signal.alarm(0) + display.display("Press 'C' to continue the play or 'A' to abort \r"), + if self._c_or_a(stdin): + clear_line(stdout) + break + + clear_line(stdout) + + raise AnsibleError('user requested abort!') + + except AnsibleTimeoutExceeded: + # this is the exception we expect when the alarm signal + # fires, so we simply ignore it to move into the cleanup + pass + finally: + # cleanup and save some information + # restore the old settings for the duped stdin stdin_fd + if not (None in (stdin_fd, old_settings)) and isatty(stdin_fd): + termios.tcsetattr(stdin_fd, termios.TCSADRAIN, old_settings) + + duration = time.time() - start + result['stop'] = to_text(datetime.datetime.now()) + result['delta'] = int(duration) + + if duration_unit == 'minutes': + duration = round(duration / 60.0, 2) + else: + duration = round(duration, 2) + result['stdout'] = "Paused for %s %s" % (duration, duration_unit) + + result['user_input'] = to_text(result['user_input'], errors='surrogate_or_strict') + return result + + def _c_or_a(self, stdin): + while True: + key_pressed = stdin.read(1) + if key_pressed.lower() == b'a': + return False + elif key_pressed.lower() == b'c': + return True diff --git a/lib/ansible/plugins/action/raw.py b/lib/ansible/plugins/action/raw.py new file mode 100644 index 0000000..b82ed34 --- /dev/null +++ b/lib/ansible/plugins/action/raw.py @@ -0,0 +1,50 @@ +# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible.plugins.action import ActionBase + + +class ActionModule(ActionBase): + TRANSFERS_FILES = False + + def run(self, tmp=None, task_vars=None): + if task_vars is None: + task_vars = dict() + + if self._task.environment and any(self._task.environment): + self._display.warning('raw module does not support the environment keyword') + + result = super(ActionModule, self).run(tmp, task_vars) + del tmp # tmp no longer has any effect + + if self._play_context.check_mode: + # in --check mode, always skip this module execution + result['skipped'] = True + return result + + executable = self._task.args.get('executable', False) + result.update(self._low_level_execute_command(self._task.args.get('_raw_params'), executable=executable)) + + result['changed'] = True + + if 'rc' in result and result['rc'] != 0: + result['failed'] = True + result['msg'] = 'non-zero return code' + + return result diff --git a/lib/ansible/plugins/action/reboot.py b/lib/ansible/plugins/action/reboot.py new file mode 100644 index 0000000..40447d1 --- /dev/null +++ b/lib/ansible/plugins/action/reboot.py @@ -0,0 +1,465 @@ +# Copyright: (c) 2016-2018, Matt Davis <mdavis@ansible.com> +# Copyright: (c) 2018, Sam Doran <sdoran@redhat.com> +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import random +import time + +from datetime import datetime, timedelta + +from ansible.errors import AnsibleError, AnsibleConnectionFailure +from ansible.module_utils._text import to_native, to_text +from ansible.module_utils.common.validation import check_type_list, check_type_str +from ansible.plugins.action import ActionBase +from ansible.utils.display import Display + +display = Display() + + +class TimedOutException(Exception): + pass + + +class ActionModule(ActionBase): + TRANSFERS_FILES = False + _VALID_ARGS = frozenset(( + 'boot_time_command', + 'connect_timeout', + 'msg', + 'post_reboot_delay', + 'pre_reboot_delay', + 'reboot_command', + 'reboot_timeout', + 'search_paths', + 'test_command', + )) + + DEFAULT_REBOOT_TIMEOUT = 600 + DEFAULT_CONNECT_TIMEOUT = None + DEFAULT_PRE_REBOOT_DELAY = 0 + DEFAULT_POST_REBOOT_DELAY = 0 + DEFAULT_TEST_COMMAND = 'whoami' + DEFAULT_BOOT_TIME_COMMAND = 'cat /proc/sys/kernel/random/boot_id' + DEFAULT_REBOOT_MESSAGE = 'Reboot initiated by Ansible' + DEFAULT_SHUTDOWN_COMMAND = 'shutdown' + DEFAULT_SHUTDOWN_COMMAND_ARGS = '-r {delay_min} "{message}"' + DEFAULT_SUDOABLE = True + + DEPRECATED_ARGS = {} # type: dict[str, str] + + BOOT_TIME_COMMANDS = { + 'freebsd': '/sbin/sysctl kern.boottime', + 'openbsd': '/sbin/sysctl kern.boottime', + 'macosx': 'who -b', + 'solaris': 'who -b', + 'sunos': 'who -b', + 'vmkernel': 'grep booted /var/log/vmksummary.log | tail -n 1', + 'aix': 'who -b', + } + + SHUTDOWN_COMMANDS = { + 'alpine': 'reboot', + 'vmkernel': 'reboot', + } + + SHUTDOWN_COMMAND_ARGS = { + 'alpine': '', + 'void': '-r +{delay_min} "{message}"', + 'freebsd': '-r +{delay_sec}s "{message}"', + 'linux': DEFAULT_SHUTDOWN_COMMAND_ARGS, + 'macosx': '-r +{delay_min} "{message}"', + 'openbsd': '-r +{delay_min} "{message}"', + 'solaris': '-y -g {delay_sec} -i 6 "{message}"', + 'sunos': '-y -g {delay_sec} -i 6 "{message}"', + 'vmkernel': '-d {delay_sec}', + 'aix': '-Fr', + } + + TEST_COMMANDS = { + 'solaris': 'who', + 'vmkernel': 'who', + } + + def __init__(self, *args, **kwargs): + super(ActionModule, self).__init__(*args, **kwargs) + + @property + def pre_reboot_delay(self): + return self._check_delay('pre_reboot_delay', self.DEFAULT_PRE_REBOOT_DELAY) + + @property + def post_reboot_delay(self): + return self._check_delay('post_reboot_delay', self.DEFAULT_POST_REBOOT_DELAY) + + def _check_delay(self, key, default): + """Ensure that the value is positive or zero""" + value = int(self._task.args.get(key, self._task.args.get(key + '_sec', default))) + if value < 0: + value = 0 + return value + + def _get_value_from_facts(self, variable_name, distribution, default_value): + """Get dist+version specific args first, then distribution, then family, lastly use default""" + attr = getattr(self, variable_name) + value = attr.get( + distribution['name'] + distribution['version'], + attr.get( + distribution['name'], + attr.get( + distribution['family'], + getattr(self, default_value)))) + return value + + def get_shutdown_command_args(self, distribution): + reboot_command = self._task.args.get('reboot_command') + if reboot_command is not None: + try: + reboot_command = check_type_str(reboot_command, allow_conversion=False) + except TypeError as e: + raise AnsibleError("Invalid value given for 'reboot_command': %s." % to_native(e)) + + # No args were provided + try: + return reboot_command.split(' ', 1)[1] + except IndexError: + return '' + else: + args = self._get_value_from_facts('SHUTDOWN_COMMAND_ARGS', distribution, 'DEFAULT_SHUTDOWN_COMMAND_ARGS') + + # Convert seconds to minutes. If less that 60, set it to 0. + delay_min = self.pre_reboot_delay // 60 + reboot_message = self._task.args.get('msg', self.DEFAULT_REBOOT_MESSAGE) + return args.format(delay_sec=self.pre_reboot_delay, delay_min=delay_min, message=reboot_message) + + def get_distribution(self, task_vars): + # FIXME: only execute the module if we don't already have the facts we need + distribution = {} + display.debug('{action}: running setup module to get distribution'.format(action=self._task.action)) + module_output = self._execute_module( + task_vars=task_vars, + module_name='ansible.legacy.setup', + module_args={'gather_subset': 'min'}) + try: + if module_output.get('failed', False): + raise AnsibleError('Failed to determine system distribution. {0}, {1}'.format( + to_native(module_output['module_stdout']).strip(), + to_native(module_output['module_stderr']).strip())) + distribution['name'] = module_output['ansible_facts']['ansible_distribution'].lower() + distribution['version'] = to_text(module_output['ansible_facts']['ansible_distribution_version'].split('.')[0]) + distribution['family'] = to_text(module_output['ansible_facts']['ansible_os_family'].lower()) + display.debug("{action}: distribution: {dist}".format(action=self._task.action, dist=distribution)) + return distribution + except KeyError as ke: + raise AnsibleError('Failed to get distribution information. Missing "{0}" in output.'.format(ke.args[0])) + + def get_shutdown_command(self, task_vars, distribution): + reboot_command = self._task.args.get('reboot_command') + if reboot_command is not None: + try: + reboot_command = check_type_str(reboot_command, allow_conversion=False) + except TypeError as e: + raise AnsibleError("Invalid value given for 'reboot_command': %s." % to_native(e)) + shutdown_bin = reboot_command.split(' ', 1)[0] + else: + shutdown_bin = self._get_value_from_facts('SHUTDOWN_COMMANDS', distribution, 'DEFAULT_SHUTDOWN_COMMAND') + + if shutdown_bin[0] == '/': + return shutdown_bin + else: + default_search_paths = ['/sbin', '/bin', '/usr/sbin', '/usr/bin', '/usr/local/sbin'] + search_paths = self._task.args.get('search_paths', default_search_paths) + + try: + # Convert bare strings to a list + search_paths = check_type_list(search_paths) + except TypeError: + err_msg = "'search_paths' must be a string or flat list of strings, got {0}" + raise AnsibleError(err_msg.format(search_paths)) + + display.debug('{action}: running find module looking in {paths} to get path for "{command}"'.format( + action=self._task.action, + command=shutdown_bin, + paths=search_paths)) + + find_result = self._execute_module( + task_vars=task_vars, + # prevent collection search by calling with ansible.legacy (still allows library/ override of find) + module_name='ansible.legacy.find', + module_args={ + 'paths': search_paths, + 'patterns': [shutdown_bin], + 'file_type': 'any' + } + ) + + full_path = [x['path'] for x in find_result['files']] + if not full_path: + raise AnsibleError('Unable to find command "{0}" in search paths: {1}'.format(shutdown_bin, search_paths)) + return full_path[0] + + def deprecated_args(self): + for arg, version in self.DEPRECATED_ARGS.items(): + if self._task.args.get(arg) is not None: + display.warning("Since Ansible {version}, {arg} is no longer a valid option for {action}".format( + version=version, + arg=arg, + action=self._task.action)) + + def get_system_boot_time(self, distribution): + boot_time_command = self._get_value_from_facts('BOOT_TIME_COMMANDS', distribution, 'DEFAULT_BOOT_TIME_COMMAND') + if self._task.args.get('boot_time_command'): + boot_time_command = self._task.args.get('boot_time_command') + + try: + check_type_str(boot_time_command, allow_conversion=False) + except TypeError as e: + raise AnsibleError("Invalid value given for 'boot_time_command': %s." % to_native(e)) + + display.debug("{action}: getting boot time with command: '{command}'".format(action=self._task.action, command=boot_time_command)) + command_result = self._low_level_execute_command(boot_time_command, sudoable=self.DEFAULT_SUDOABLE) + + if command_result['rc'] != 0: + stdout = command_result['stdout'] + stderr = command_result['stderr'] + raise AnsibleError("{action}: failed to get host boot time info, rc: {rc}, stdout: {out}, stderr: {err}".format( + action=self._task.action, + rc=command_result['rc'], + out=to_native(stdout), + err=to_native(stderr))) + display.debug("{action}: last boot time: {boot}".format(action=self._task.action, boot=command_result['stdout'].strip())) + return command_result['stdout'].strip() + + def check_boot_time(self, distribution, previous_boot_time): + display.vvv("{action}: attempting to get system boot time".format(action=self._task.action)) + connect_timeout = self._task.args.get('connect_timeout', self._task.args.get('connect_timeout_sec', self.DEFAULT_CONNECT_TIMEOUT)) + + # override connection timeout from defaults to custom value + if connect_timeout: + try: + display.debug("{action}: setting connect_timeout to {value}".format(action=self._task.action, value=connect_timeout)) + self._connection.set_option("connection_timeout", connect_timeout) + self._connection.reset() + except AttributeError: + display.warning("Connection plugin does not allow the connection timeout to be overridden") + + # try and get boot time + try: + current_boot_time = self.get_system_boot_time(distribution) + except Exception as e: + raise e + + # FreeBSD returns an empty string immediately before reboot so adding a length + # check to prevent prematurely assuming system has rebooted + if len(current_boot_time) == 0 or current_boot_time == previous_boot_time: + raise ValueError("boot time has not changed") + + def run_test_command(self, distribution, **kwargs): + test_command = self._task.args.get('test_command', self._get_value_from_facts('TEST_COMMANDS', distribution, 'DEFAULT_TEST_COMMAND')) + display.vvv("{action}: attempting post-reboot test command".format(action=self._task.action)) + display.debug("{action}: attempting post-reboot test command '{command}'".format(action=self._task.action, command=test_command)) + try: + command_result = self._low_level_execute_command(test_command, sudoable=self.DEFAULT_SUDOABLE) + except Exception: + # may need to reset the connection in case another reboot occurred + # which has invalidated our connection + try: + self._connection.reset() + except AttributeError: + pass + raise + + if command_result['rc'] != 0: + msg = 'Test command failed: {err} {out}'.format( + err=to_native(command_result['stderr']), + out=to_native(command_result['stdout'])) + raise RuntimeError(msg) + + display.vvv("{action}: system successfully rebooted".format(action=self._task.action)) + + def do_until_success_or_timeout(self, action, reboot_timeout, action_desc, distribution, action_kwargs=None): + max_end_time = datetime.utcnow() + timedelta(seconds=reboot_timeout) + if action_kwargs is None: + action_kwargs = {} + + fail_count = 0 + max_fail_sleep = 12 + + while datetime.utcnow() < max_end_time: + try: + action(distribution=distribution, **action_kwargs) + if action_desc: + display.debug('{action}: {desc} success'.format(action=self._task.action, desc=action_desc)) + return + except Exception as e: + if isinstance(e, AnsibleConnectionFailure): + try: + self._connection.reset() + except AnsibleConnectionFailure: + pass + # Use exponential backoff with a max timout, plus a little bit of randomness + random_int = random.randint(0, 1000) / 1000 + fail_sleep = 2 ** fail_count + random_int + if fail_sleep > max_fail_sleep: + + fail_sleep = max_fail_sleep + random_int + if action_desc: + try: + error = to_text(e).splitlines()[-1] + except IndexError as e: + error = to_text(e) + display.debug("{action}: {desc} fail '{err}', retrying in {sleep:.4} seconds...".format( + action=self._task.action, + desc=action_desc, + err=error, + sleep=fail_sleep)) + fail_count += 1 + time.sleep(fail_sleep) + + raise TimedOutException('Timed out waiting for {desc} (timeout={timeout})'.format(desc=action_desc, timeout=reboot_timeout)) + + def perform_reboot(self, task_vars, distribution): + result = {} + reboot_result = {} + shutdown_command = self.get_shutdown_command(task_vars, distribution) + shutdown_command_args = self.get_shutdown_command_args(distribution) + reboot_command = '{0} {1}'.format(shutdown_command, shutdown_command_args) + + try: + display.vvv("{action}: rebooting server...".format(action=self._task.action)) + display.debug("{action}: rebooting server with command '{command}'".format(action=self._task.action, command=reboot_command)) + reboot_result = self._low_level_execute_command(reboot_command, sudoable=self.DEFAULT_SUDOABLE) + except AnsibleConnectionFailure as e: + # If the connection is closed too quickly due to the system being shutdown, carry on + display.debug('{action}: AnsibleConnectionFailure caught and handled: {error}'.format(action=self._task.action, error=to_text(e))) + reboot_result['rc'] = 0 + + result['start'] = datetime.utcnow() + + if reboot_result['rc'] != 0: + result['failed'] = True + result['rebooted'] = False + result['msg'] = "Reboot command failed. Error was: '{stdout}, {stderr}'".format( + stdout=to_native(reboot_result['stdout'].strip()), + stderr=to_native(reboot_result['stderr'].strip())) + return result + + result['failed'] = False + return result + + def validate_reboot(self, distribution, original_connection_timeout=None, action_kwargs=None): + display.vvv('{action}: validating reboot'.format(action=self._task.action)) + result = {} + + try: + # keep on checking system boot_time with short connection responses + reboot_timeout = int(self._task.args.get('reboot_timeout', self._task.args.get('reboot_timeout_sec', self.DEFAULT_REBOOT_TIMEOUT))) + + self.do_until_success_or_timeout( + action=self.check_boot_time, + action_desc="last boot time check", + reboot_timeout=reboot_timeout, + distribution=distribution, + action_kwargs=action_kwargs) + + # Get the connect_timeout set on the connection to compare to the original + try: + connect_timeout = self._connection.get_option('connection_timeout') + except KeyError: + pass + else: + if original_connection_timeout != connect_timeout: + try: + display.debug("{action}: setting connect_timeout back to original value of {value}".format( + action=self._task.action, + value=original_connection_timeout)) + self._connection.set_option("connection_timeout", original_connection_timeout) + self._connection.reset() + except (AnsibleError, AttributeError) as e: + # reset the connection to clear the custom connection timeout + display.debug("{action}: failed to reset connection_timeout back to default: {error}".format(action=self._task.action, + error=to_text(e))) + + # finally run test command to ensure everything is working + # FUTURE: add a stability check (system must remain up for N seconds) to deal with self-multi-reboot updates + self.do_until_success_or_timeout( + action=self.run_test_command, + action_desc="post-reboot test command", + reboot_timeout=reboot_timeout, + distribution=distribution, + action_kwargs=action_kwargs) + + result['rebooted'] = True + result['changed'] = True + + except TimedOutException as toex: + result['failed'] = True + result['rebooted'] = True + result['msg'] = to_text(toex) + return result + + return result + + def run(self, tmp=None, task_vars=None): + self._supports_check_mode = True + self._supports_async = True + + # If running with local connection, fail so we don't reboot ourself + if self._connection.transport == 'local': + msg = 'Running {0} with local connection would reboot the control node.'.format(self._task.action) + return {'changed': False, 'elapsed': 0, 'rebooted': False, 'failed': True, 'msg': msg} + + if self._play_context.check_mode: + return {'changed': True, 'elapsed': 0, 'rebooted': True} + + if task_vars is None: + task_vars = {} + + self.deprecated_args() + + result = super(ActionModule, self).run(tmp, task_vars) + + if result.get('skipped', False) or result.get('failed', False): + return result + + distribution = self.get_distribution(task_vars) + + # Get current boot time + try: + previous_boot_time = self.get_system_boot_time(distribution) + except Exception as e: + result['failed'] = True + result['reboot'] = False + result['msg'] = to_text(e) + return result + + # Get the original connection_timeout option var so it can be reset after + original_connection_timeout = None + try: + original_connection_timeout = self._connection.get_option('connection_timeout') + display.debug("{action}: saving original connect_timeout of {timeout}".format(action=self._task.action, timeout=original_connection_timeout)) + except KeyError: + display.debug("{action}: connect_timeout connection option has not been set".format(action=self._task.action)) + # Initiate reboot + reboot_result = self.perform_reboot(task_vars, distribution) + + if reboot_result['failed']: + result = reboot_result + elapsed = datetime.utcnow() - reboot_result['start'] + result['elapsed'] = elapsed.seconds + return result + + if self.post_reboot_delay != 0: + display.debug("{action}: waiting an additional {delay} seconds".format(action=self._task.action, delay=self.post_reboot_delay)) + display.vvv("{action}: waiting an additional {delay} seconds".format(action=self._task.action, delay=self.post_reboot_delay)) + time.sleep(self.post_reboot_delay) + + # Make sure reboot was successful + result = self.validate_reboot(distribution, original_connection_timeout, action_kwargs={'previous_boot_time': previous_boot_time}) + + elapsed = datetime.utcnow() - reboot_result['start'] + result['elapsed'] = elapsed.seconds + + return result diff --git a/lib/ansible/plugins/action/script.py b/lib/ansible/plugins/action/script.py new file mode 100644 index 0000000..1bbb800 --- /dev/null +++ b/lib/ansible/plugins/action/script.py @@ -0,0 +1,160 @@ +# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import os +import re +import shlex + +from ansible.errors import AnsibleError, AnsibleAction, _AnsibleActionDone, AnsibleActionFail, AnsibleActionSkip +from ansible.executor.powershell import module_manifest as ps_manifest +from ansible.module_utils._text import to_bytes, to_native, to_text +from ansible.plugins.action import ActionBase + + +class ActionModule(ActionBase): + + TRANSFERS_FILES = True + + # On Windows platform, absolute paths begin with a (back)slash + # after chopping off a potential drive letter. + windows_absolute_path_detection = re.compile(r'^(?:[a-zA-Z]\:)?(\\|\/)') + + def run(self, tmp=None, task_vars=None): + ''' handler for file transfer operations ''' + if task_vars is None: + task_vars = dict() + + result = super(ActionModule, self).run(tmp, task_vars) + del tmp # tmp no longer has any effect + + try: + creates = self._task.args.get('creates') + if creates: + # do not run the command if the line contains creates=filename + # and the filename already exists. This allows idempotence + # of command executions. + if self._remote_file_exists(creates): + raise AnsibleActionSkip("%s exists, matching creates option" % creates) + + removes = self._task.args.get('removes') + if removes: + # do not run the command if the line contains removes=filename + # and the filename does not exist. This allows idempotence + # of command executions. + if not self._remote_file_exists(removes): + raise AnsibleActionSkip("%s does not exist, matching removes option" % removes) + + # The chdir must be absolute, because a relative path would rely on + # remote node behaviour & user config. + chdir = self._task.args.get('chdir') + if chdir: + # Powershell is the only Windows-path aware shell + if getattr(self._connection._shell, "_IS_WINDOWS", False) and \ + not self.windows_absolute_path_detection.match(chdir): + raise AnsibleActionFail('chdir %s must be an absolute path for a Windows remote node' % chdir) + # Every other shell is unix-path-aware. + if not getattr(self._connection._shell, "_IS_WINDOWS", False) and not chdir.startswith('/'): + raise AnsibleActionFail('chdir %s must be an absolute path for a Unix-aware remote node' % chdir) + + # Split out the script as the first item in raw_params using + # shlex.split() in order to support paths and files with spaces in the name. + # Any arguments passed to the script will be added back later. + raw_params = to_native(self._task.args.get('_raw_params', ''), errors='surrogate_or_strict') + parts = [to_text(s, errors='surrogate_or_strict') for s in shlex.split(raw_params.strip())] + source = parts[0] + + # Support executable paths and files with spaces in the name. + executable = to_native(self._task.args.get('executable', ''), errors='surrogate_or_strict') + + try: + source = self._loader.get_real_file(self._find_needle('files', source), decrypt=self._task.args.get('decrypt', True)) + except AnsibleError as e: + raise AnsibleActionFail(to_native(e)) + + if self._task.check_mode: + # check mode is supported if 'creates' or 'removes' are provided + # the task has already been skipped if a change would not occur + if self._task.args.get('creates') or self._task.args.get('removes'): + result['changed'] = True + raise _AnsibleActionDone(result=result) + # If the script doesn't return changed in the result, it defaults to True, + # but since the script may override 'changed', just skip instead of guessing. + else: + result['changed'] = False + raise AnsibleActionSkip('Check mode is not supported for this task.', result=result) + + # now we execute script, always assume changed. + result['changed'] = True + + # transfer the file to a remote tmp location + tmp_src = self._connection._shell.join_path(self._connection._shell.tmpdir, + os.path.basename(source)) + + # Convert raw_params to text for the purpose of replacing the script since + # parts and tmp_src are both unicode strings and raw_params will be different + # depending on Python version. + # + # Once everything is encoded consistently, replace the script path on the remote + # system with the remainder of the raw_params. This preserves quoting in parameters + # that would have been removed by shlex.split(). + target_command = to_text(raw_params).strip().replace(parts[0], tmp_src) + + self._transfer_file(source, tmp_src) + + # set file permissions, more permissive when the copy is done as a different user + self._fixup_perms2((self._connection._shell.tmpdir, tmp_src), execute=True) + + # add preparation steps to one ssh roundtrip executing the script + env_dict = dict() + env_string = self._compute_environment_string(env_dict) + + if executable: + script_cmd = ' '.join([env_string, executable, target_command]) + else: + script_cmd = ' '.join([env_string, target_command]) + + script_cmd = self._connection._shell.wrap_for_exec(script_cmd) + + exec_data = None + # PowerShell runs the script in a special wrapper to enable things + # like become and environment args + if getattr(self._connection._shell, "_IS_WINDOWS", False): + # FUTURE: use a more public method to get the exec payload + pc = self._play_context + exec_data = ps_manifest._create_powershell_wrapper( + to_bytes(script_cmd), source, {}, env_dict, self._task.async_val, + pc.become, pc.become_method, pc.become_user, + pc.become_pass, pc.become_flags, "script", task_vars, None + ) + # build the necessary exec wrapper command + # FUTURE: this still doesn't let script work on Windows with non-pipelined connections or + # full manual exec of KEEP_REMOTE_FILES + script_cmd = self._connection._shell.build_module_command(env_string='', shebang='#!powershell', cmd='') + + result.update(self._low_level_execute_command(cmd=script_cmd, in_data=exec_data, sudoable=True, chdir=chdir)) + + if 'rc' in result and result['rc'] != 0: + raise AnsibleActionFail('non-zero return code') + + except AnsibleAction as e: + result.update(e.result) + finally: + self._remove_tmp_path(self._connection._shell.tmpdir) + + return result diff --git a/lib/ansible/plugins/action/service.py b/lib/ansible/plugins/action/service.py new file mode 100644 index 0000000..c061687 --- /dev/null +++ b/lib/ansible/plugins/action/service.py @@ -0,0 +1,103 @@ +# (c) 2015, Ansible Inc, +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +from ansible.errors import AnsibleAction, AnsibleActionFail +from ansible.executor.module_common import get_action_args_with_defaults +from ansible.plugins.action import ActionBase + + +class ActionModule(ActionBase): + + TRANSFERS_FILES = False + + UNUSED_PARAMS = { + 'systemd': ['pattern', 'runlevel', 'sleep', 'arguments', 'args'], + } + + # HACK: list of unqualified service manager names that are/were built-in, we'll prefix these with `ansible.legacy` to + # avoid collisions with collections search + BUILTIN_SVC_MGR_MODULES = set(['openwrt_init', 'service', 'systemd', 'sysvinit']) + + def run(self, tmp=None, task_vars=None): + ''' handler for package operations ''' + + self._supports_check_mode = True + self._supports_async = True + + result = super(ActionModule, self).run(tmp, task_vars) + del tmp # tmp no longer has any effect + + module = self._task.args.get('use', 'auto').lower() + + if module == 'auto': + try: + if self._task.delegate_to: # if we delegate, we should use delegated host's facts + module = self._templar.template("{{hostvars['%s']['ansible_facts']['service_mgr']}}" % self._task.delegate_to) + else: + module = self._templar.template('{{ansible_facts.service_mgr}}') + except Exception: + pass # could not get it from template! + + try: + if module == 'auto': + facts = self._execute_module( + module_name='ansible.legacy.setup', + module_args=dict(gather_subset='!all', filter='ansible_service_mgr'), task_vars=task_vars) + self._display.debug("Facts %s" % facts) + module = facts.get('ansible_facts', {}).get('ansible_service_mgr', 'auto') + + if not module or module == 'auto' or not self._shared_loader_obj.module_loader.has_plugin(module): + module = 'ansible.legacy.service' + + if module != 'auto': + # run the 'service' module + new_module_args = self._task.args.copy() + if 'use' in new_module_args: + del new_module_args['use'] + + if module in self.UNUSED_PARAMS: + for unused in self.UNUSED_PARAMS[module]: + if unused in new_module_args: + del new_module_args[unused] + self._display.warning('Ignoring "%s" as it is not used in "%s"' % (unused, module)) + + # get defaults for specific module + context = self._shared_loader_obj.module_loader.find_plugin_with_context(module, collection_list=self._task.collections) + new_module_args = get_action_args_with_defaults( + context.resolved_fqcn, new_module_args, self._task.module_defaults, self._templar, + action_groups=self._task._parent._play._action_groups + ) + + # collection prefix known internal modules to avoid collisions from collections search, while still allowing library/ overrides + if module in self.BUILTIN_SVC_MGR_MODULES: + module = 'ansible.legacy.' + module + + self._display.vvvv("Running %s" % module) + result.update(self._execute_module(module_name=module, module_args=new_module_args, task_vars=task_vars, wrap_async=self._task.async_val)) + else: + raise AnsibleActionFail('Could not detect which service manager to use. Try gathering facts or setting the "use" option.') + + except AnsibleAction as e: + result.update(e.result) + finally: + if not self._task.async_val: + self._remove_tmp_path(self._connection._shell.tmpdir) + + return result diff --git a/lib/ansible/plugins/action/set_fact.py b/lib/ansible/plugins/action/set_fact.py new file mode 100644 index 0000000..ae92de8 --- /dev/null +++ b/lib/ansible/plugins/action/set_fact.py @@ -0,0 +1,68 @@ +# Copyright 2013 Dag Wieers <dag@wieers.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible.errors import AnsibleActionFail +from ansible.module_utils.six import string_types +from ansible.module_utils.parsing.convert_bool import boolean +from ansible.plugins.action import ActionBase +from ansible.utils.vars import isidentifier + +import ansible.constants as C + + +class ActionModule(ActionBase): + + TRANSFERS_FILES = False + + def run(self, tmp=None, task_vars=None): + if task_vars is None: + task_vars = dict() + + result = super(ActionModule, self).run(tmp, task_vars) + del tmp # tmp no longer has any effect + + facts = {} + cacheable = boolean(self._task.args.pop('cacheable', False)) + + if self._task.args: + for (k, v) in self._task.args.items(): + k = self._templar.template(k) + + if not isidentifier(k): + raise AnsibleActionFail("The variable name '%s' is not valid. Variables must start with a letter or underscore character, " + "and contain only letters, numbers and underscores." % k) + + # NOTE: this should really use BOOLEANS from convert_bool, but only in the k=v case, + # right now it converts matching explicit YAML strings also when 'jinja2_native' is disabled. + if not C.DEFAULT_JINJA2_NATIVE and isinstance(v, string_types) and v.lower() in ('true', 'false', 'yes', 'no'): + v = boolean(v, strict=False) + facts[k] = v + else: + raise AnsibleActionFail('No key/value pairs provided, at least one is required for this action to succeed') + + if facts: + # just as _facts actions, we don't set changed=true as we are not modifying the actual host + result['ansible_facts'] = facts + result['_ansible_facts_cacheable'] = cacheable + else: + # this should not happen, but JIC we get here + raise AnsibleActionFail('Unable to create any variables with provided arguments') + + return result diff --git a/lib/ansible/plugins/action/set_stats.py b/lib/ansible/plugins/action/set_stats.py new file mode 100644 index 0000000..9d429ce --- /dev/null +++ b/lib/ansible/plugins/action/set_stats.py @@ -0,0 +1,77 @@ +# Copyright 2016 Ansible (RedHat, Inc) +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible.module_utils.six import string_types +from ansible.module_utils.parsing.convert_bool import boolean +from ansible.plugins.action import ActionBase +from ansible.utils.vars import isidentifier + + +class ActionModule(ActionBase): + + TRANSFERS_FILES = False + _VALID_ARGS = frozenset(('aggregate', 'data', 'per_host')) + + # TODO: document this in non-empty set_stats.py module + def run(self, tmp=None, task_vars=None): + if task_vars is None: + task_vars = dict() + + result = super(ActionModule, self).run(tmp, task_vars) + del tmp # tmp no longer has any effect + + stats = {'data': {}, 'per_host': False, 'aggregate': True} + + if self._task.args: + data = self._task.args.get('data', {}) + + if not isinstance(data, dict): + data = self._templar.template(data, convert_bare=False, fail_on_undefined=True) + + if not isinstance(data, dict): + result['failed'] = True + result['msg'] = "The 'data' option needs to be a dictionary/hash" + return result + + # set boolean options, defaults are set above in stats init + for opt in ['per_host', 'aggregate']: + val = self._task.args.get(opt, None) + if val is not None: + if not isinstance(val, bool): + stats[opt] = boolean(self._templar.template(val), strict=False) + else: + stats[opt] = val + + for (k, v) in data.items(): + + k = self._templar.template(k) + + if not isidentifier(k): + result['failed'] = True + result['msg'] = ("The variable name '%s' is not valid. Variables must start with a letter or underscore character, and contain only " + "letters, numbers and underscores." % k) + return result + + stats['data'][k] = self._templar.template(v) + + result['changed'] = False + result['ansible_stats'] = stats + + return result diff --git a/lib/ansible/plugins/action/shell.py b/lib/ansible/plugins/action/shell.py new file mode 100644 index 0000000..617a373 --- /dev/null +++ b/lib/ansible/plugins/action/shell.py @@ -0,0 +1,27 @@ +# Copyright: (c) 2017, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible.plugins.action import ActionBase + + +class ActionModule(ActionBase): + + def run(self, tmp=None, task_vars=None): + del tmp # tmp no longer has any effect + + # Shell module is implemented via command with a special arg + self._task.args['_uses_shell'] = True + + command_action = self._shared_loader_obj.action_loader.get('ansible.legacy.command', + task=self._task, + connection=self._connection, + play_context=self._play_context, + loader=self._loader, + templar=self._templar, + shared_loader_obj=self._shared_loader_obj) + result = command_action.run(task_vars=task_vars) + + return result diff --git a/lib/ansible/plugins/action/template.py b/lib/ansible/plugins/action/template.py new file mode 100644 index 0000000..d2b3df9 --- /dev/null +++ b/lib/ansible/plugins/action/template.py @@ -0,0 +1,190 @@ +# Copyright: (c) 2015, Michael DeHaan <michael.dehaan@gmail.com> +# Copyright: (c) 2018, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import os +import shutil +import stat +import tempfile + +from ansible import constants as C +from ansible.config.manager import ensure_type +from ansible.errors import AnsibleError, AnsibleFileNotFound, AnsibleAction, AnsibleActionFail +from ansible.module_utils._text import to_bytes, to_text, to_native +from ansible.module_utils.parsing.convert_bool import boolean +from ansible.module_utils.six import string_types +from ansible.plugins.action import ActionBase +from ansible.template import generate_ansible_template_vars, AnsibleEnvironment + + +class ActionModule(ActionBase): + + TRANSFERS_FILES = True + DEFAULT_NEWLINE_SEQUENCE = "\n" + + def run(self, tmp=None, task_vars=None): + ''' handler for template operations ''' + + if task_vars is None: + task_vars = dict() + + result = super(ActionModule, self).run(tmp, task_vars) + del tmp # tmp no longer has any effect + + # Options type validation + # stings + for s_type in ('src', 'dest', 'state', 'newline_sequence', 'variable_start_string', 'variable_end_string', 'block_start_string', + 'block_end_string', 'comment_start_string', 'comment_end_string'): + if s_type in self._task.args: + value = ensure_type(self._task.args[s_type], 'string') + if value is not None and not isinstance(value, string_types): + raise AnsibleActionFail("%s is expected to be a string, but got %s instead" % (s_type, type(value))) + self._task.args[s_type] = value + + # booleans + try: + follow = boolean(self._task.args.get('follow', False), strict=False) + trim_blocks = boolean(self._task.args.get('trim_blocks', True), strict=False) + lstrip_blocks = boolean(self._task.args.get('lstrip_blocks', False), strict=False) + except TypeError as e: + raise AnsibleActionFail(to_native(e)) + + # assign to local vars for ease of use + source = self._task.args.get('src', None) + dest = self._task.args.get('dest', None) + state = self._task.args.get('state', None) + newline_sequence = self._task.args.get('newline_sequence', self.DEFAULT_NEWLINE_SEQUENCE) + variable_start_string = self._task.args.get('variable_start_string', None) + variable_end_string = self._task.args.get('variable_end_string', None) + block_start_string = self._task.args.get('block_start_string', None) + block_end_string = self._task.args.get('block_end_string', None) + comment_start_string = self._task.args.get('comment_start_string', None) + comment_end_string = self._task.args.get('comment_end_string', None) + output_encoding = self._task.args.get('output_encoding', 'utf-8') or 'utf-8' + + wrong_sequences = ["\\n", "\\r", "\\r\\n"] + allowed_sequences = ["\n", "\r", "\r\n"] + + # We need to convert unescaped sequences to proper escaped sequences for Jinja2 + if newline_sequence in wrong_sequences: + newline_sequence = allowed_sequences[wrong_sequences.index(newline_sequence)] + + try: + # logical validation + if state is not None: + raise AnsibleActionFail("'state' cannot be specified on a template") + elif source is None or dest is None: + raise AnsibleActionFail("src and dest are required") + elif newline_sequence not in allowed_sequences: + raise AnsibleActionFail("newline_sequence needs to be one of: \n, \r or \r\n") + else: + try: + source = self._find_needle('templates', source) + except AnsibleError as e: + raise AnsibleActionFail(to_text(e)) + + mode = self._task.args.get('mode', None) + if mode == 'preserve': + mode = '0%03o' % stat.S_IMODE(os.stat(source).st_mode) + + # Get vault decrypted tmp file + try: + tmp_source = self._loader.get_real_file(source) + except AnsibleFileNotFound as e: + raise AnsibleActionFail("could not find src=%s, %s" % (source, to_text(e))) + b_tmp_source = to_bytes(tmp_source, errors='surrogate_or_strict') + + # template the source data locally & get ready to transfer + try: + with open(b_tmp_source, 'rb') as f: + try: + template_data = to_text(f.read(), errors='surrogate_or_strict') + except UnicodeError: + raise AnsibleActionFail("Template source files must be utf-8 encoded") + + # set jinja2 internal search path for includes + searchpath = task_vars.get('ansible_search_path', []) + searchpath.extend([self._loader._basedir, os.path.dirname(source)]) + + # We want to search into the 'templates' subdir of each search path in + # addition to our original search paths. + newsearchpath = [] + for p in searchpath: + newsearchpath.append(os.path.join(p, 'templates')) + newsearchpath.append(p) + searchpath = newsearchpath + + # add ansible 'template' vars + temp_vars = task_vars.copy() + # NOTE in the case of ANSIBLE_DEBUG=1 task_vars is VarsWithSources(MutableMapping) + # so | operator cannot be used as it can be used only on dicts + # https://peps.python.org/pep-0584/#what-about-mapping-and-mutablemapping + temp_vars.update(generate_ansible_template_vars(self._task.args.get('src', None), source, dest)) + + # force templar to use AnsibleEnvironment to prevent issues with native types + # https://github.com/ansible/ansible/issues/46169 + templar = self._templar.copy_with_new_env(environment_class=AnsibleEnvironment, + searchpath=searchpath, + newline_sequence=newline_sequence, + block_start_string=block_start_string, + block_end_string=block_end_string, + variable_start_string=variable_start_string, + variable_end_string=variable_end_string, + comment_start_string=comment_start_string, + comment_end_string=comment_end_string, + trim_blocks=trim_blocks, + lstrip_blocks=lstrip_blocks, + available_variables=temp_vars) + resultant = templar.do_template(template_data, preserve_trailing_newlines=True, escape_backslashes=False) + except AnsibleAction: + raise + except Exception as e: + raise AnsibleActionFail("%s: %s" % (type(e).__name__, to_text(e))) + finally: + self._loader.cleanup_tmp_file(b_tmp_source) + + new_task = self._task.copy() + # mode is either the mode from task.args or the mode of the source file if the task.args + # mode == 'preserve' + new_task.args['mode'] = mode + + # remove 'template only' options: + for remove in ('newline_sequence', 'block_start_string', 'block_end_string', 'variable_start_string', 'variable_end_string', + 'comment_start_string', 'comment_end_string', 'trim_blocks', 'lstrip_blocks', 'output_encoding'): + new_task.args.pop(remove, None) + + local_tempdir = tempfile.mkdtemp(dir=C.DEFAULT_LOCAL_TMP) + + try: + result_file = os.path.join(local_tempdir, os.path.basename(source)) + with open(to_bytes(result_file, errors='surrogate_or_strict'), 'wb') as f: + f.write(to_bytes(resultant, encoding=output_encoding, errors='surrogate_or_strict')) + + new_task.args.update( + dict( + src=result_file, + dest=dest, + follow=follow, + ), + ) + # call with ansible.legacy prefix to eliminate collisions with collections while still allowing local override + copy_action = self._shared_loader_obj.action_loader.get('ansible.legacy.copy', + task=new_task, + connection=self._connection, + play_context=self._play_context, + loader=self._loader, + templar=self._templar, + shared_loader_obj=self._shared_loader_obj) + result.update(copy_action.run(task_vars=task_vars)) + finally: + shutil.rmtree(to_bytes(local_tempdir, errors='surrogate_or_strict')) + + except AnsibleAction as e: + result.update(e.result) + finally: + self._remove_tmp_path(self._connection._shell.tmpdir) + + return result diff --git a/lib/ansible/plugins/action/unarchive.py b/lib/ansible/plugins/action/unarchive.py new file mode 100644 index 0000000..4d188e3 --- /dev/null +++ b/lib/ansible/plugins/action/unarchive.py @@ -0,0 +1,111 @@ +# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com> +# (c) 2013, Dylan Martin <dmartin@seattlecentral.edu> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import os + +from ansible.errors import AnsibleError, AnsibleAction, AnsibleActionFail, AnsibleActionSkip +from ansible.module_utils._text import to_text +from ansible.module_utils.parsing.convert_bool import boolean +from ansible.plugins.action import ActionBase + + +class ActionModule(ActionBase): + + TRANSFERS_FILES = True + + def run(self, tmp=None, task_vars=None): + ''' handler for unarchive operations ''' + if task_vars is None: + task_vars = dict() + + result = super(ActionModule, self).run(tmp, task_vars) + del tmp # tmp no longer has any effect + + source = self._task.args.get('src', None) + dest = self._task.args.get('dest', None) + remote_src = boolean(self._task.args.get('remote_src', False), strict=False) + creates = self._task.args.get('creates', None) + decrypt = self._task.args.get('decrypt', True) + + try: + # "copy" is deprecated in favor of "remote_src". + if 'copy' in self._task.args: + # They are mutually exclusive. + if 'remote_src' in self._task.args: + raise AnsibleActionFail("parameters are mutually exclusive: ('copy', 'remote_src')") + # We will take the information from copy and store it in + # the remote_src var to use later in this file. + self._task.args['remote_src'] = remote_src = not boolean(self._task.args.pop('copy'), strict=False) + + if source is None or dest is None: + raise AnsibleActionFail("src (or content) and dest are required") + + if creates: + # do not run the command if the line contains creates=filename + # and the filename already exists. This allows idempotence + # of command executions. + creates = self._remote_expand_user(creates) + if self._remote_file_exists(creates): + raise AnsibleActionSkip("skipped, since %s exists" % creates) + + dest = self._remote_expand_user(dest) # CCTODO: Fix path for Windows hosts. + source = os.path.expanduser(source) + + if not remote_src: + try: + source = self._loader.get_real_file(self._find_needle('files', source), decrypt=decrypt) + except AnsibleError as e: + raise AnsibleActionFail(to_text(e)) + + try: + remote_stat = self._execute_remote_stat(dest, all_vars=task_vars, follow=True) + except AnsibleError as e: + raise AnsibleActionFail(to_text(e)) + + if not remote_stat['exists'] or not remote_stat['isdir']: + raise AnsibleActionFail("dest '%s' must be an existing dir" % dest) + + if not remote_src: + # transfer the file to a remote tmp location + tmp_src = self._connection._shell.join_path(self._connection._shell.tmpdir, 'source') + self._transfer_file(source, tmp_src) + + # handle diff mode client side + # handle check mode client side + + # remove action plugin only keys + new_module_args = self._task.args.copy() + for key in ('decrypt',): + if key in new_module_args: + del new_module_args[key] + + if not remote_src: + # fix file permissions when the copy is done as a different user + self._fixup_perms2((self._connection._shell.tmpdir, tmp_src)) + new_module_args['src'] = tmp_src + + # execute the unarchive module now, with the updated args (using ansible.legacy prefix to eliminate collections + # collisions with local override + result.update(self._execute_module(module_name='ansible.legacy.unarchive', module_args=new_module_args, task_vars=task_vars)) + except AnsibleAction as e: + result.update(e.result) + finally: + self._remove_tmp_path(self._connection._shell.tmpdir) + return result diff --git a/lib/ansible/plugins/action/uri.py b/lib/ansible/plugins/action/uri.py new file mode 100644 index 0000000..bbaf092 --- /dev/null +++ b/lib/ansible/plugins/action/uri.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- +# (c) 2015, Brian Coca <briancoca+dev@gmail.com> +# (c) 2018, Matt Martz <matt@sivel.net> +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import os + +from ansible.errors import AnsibleError, AnsibleAction, _AnsibleActionDone, AnsibleActionFail +from ansible.module_utils._text import to_native +from ansible.module_utils.common.collections import Mapping, MutableMapping +from ansible.module_utils.parsing.convert_bool import boolean +from ansible.module_utils.six import text_type +from ansible.plugins.action import ActionBase + + +class ActionModule(ActionBase): + + TRANSFERS_FILES = True + + def run(self, tmp=None, task_vars=None): + self._supports_async = True + + if task_vars is None: + task_vars = dict() + + result = super(ActionModule, self).run(tmp, task_vars) + del tmp # tmp no longer has any effect + + body_format = self._task.args.get('body_format', 'raw') + body = self._task.args.get('body') + src = self._task.args.get('src', None) + remote_src = boolean(self._task.args.get('remote_src', 'no'), strict=False) + + try: + if remote_src: + # everything is remote, so we just execute the module + # without changing any of the module arguments + # call with ansible.legacy prefix to prevent collections collisions while allowing local override + raise _AnsibleActionDone(result=self._execute_module(module_name='ansible.legacy.uri', + task_vars=task_vars, wrap_async=self._task.async_val)) + + kwargs = {} + + if src: + try: + src = self._find_needle('files', src) + except AnsibleError as e: + raise AnsibleActionFail(to_native(e)) + + tmp_src = self._connection._shell.join_path(self._connection._shell.tmpdir, os.path.basename(src)) + kwargs['src'] = tmp_src + self._transfer_file(src, tmp_src) + self._fixup_perms2((self._connection._shell.tmpdir, tmp_src)) + elif body_format == 'form-multipart': + if not isinstance(body, Mapping): + raise AnsibleActionFail( + 'body must be mapping, cannot be type %s' % body.__class__.__name__ + ) + for field, value in body.items(): + if not isinstance(value, MutableMapping): + continue + content = value.get('content') + filename = value.get('filename') + if not filename or content: + continue + + try: + filename = self._find_needle('files', filename) + except AnsibleError as e: + raise AnsibleActionFail(to_native(e)) + + tmp_src = self._connection._shell.join_path( + self._connection._shell.tmpdir, + os.path.basename(filename) + ) + value['filename'] = tmp_src + self._transfer_file(filename, tmp_src) + self._fixup_perms2((self._connection._shell.tmpdir, tmp_src)) + kwargs['body'] = body + + new_module_args = self._task.args | kwargs + + # call with ansible.legacy prefix to prevent collections collisions while allowing local override + result.update(self._execute_module('ansible.legacy.uri', module_args=new_module_args, task_vars=task_vars, wrap_async=self._task.async_val)) + except AnsibleAction as e: + result.update(e.result) + finally: + if not self._task.async_val: + self._remove_tmp_path(self._connection._shell.tmpdir) + return result diff --git a/lib/ansible/plugins/action/validate_argument_spec.py b/lib/ansible/plugins/action/validate_argument_spec.py new file mode 100644 index 0000000..dc7d6cb --- /dev/null +++ b/lib/ansible/plugins/action/validate_argument_spec.py @@ -0,0 +1,94 @@ +# Copyright 2021 Red Hat +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible.errors import AnsibleError +from ansible.plugins.action import ActionBase +from ansible.module_utils.six import string_types +from ansible.module_utils.common.arg_spec import ArgumentSpecValidator +from ansible.module_utils.errors import AnsibleValidationErrorMultiple +from ansible.utils.vars import combine_vars + + +class ActionModule(ActionBase): + ''' Validate an arg spec''' + + TRANSFERS_FILES = False + + def get_args_from_task_vars(self, argument_spec, task_vars): + ''' + Get any arguments that may come from `task_vars`. + + Expand templated variables so we can validate the actual values. + + :param argument_spec: A dict of the argument spec. + :param task_vars: A dict of task variables. + + :returns: A dict of values that can be validated against the arg spec. + ''' + args = {} + + for argument_name, argument_attrs in argument_spec.items(): + if argument_name in task_vars: + args[argument_name] = task_vars[argument_name] + args = self._templar.template(args) + return args + + def run(self, tmp=None, task_vars=None): + ''' + Validate an argument specification against a provided set of data. + + The `validate_argument_spec` module expects to receive the arguments: + - argument_spec: A dict whose keys are the valid argument names, and + whose values are dicts of the argument attributes (type, etc). + - provided_arguments: A dict whose keys are the argument names, and + whose values are the argument value. + + :param tmp: Deprecated. Do not use. + :param task_vars: A dict of task variables. + :return: An action result dict, including a 'argument_errors' key with a + list of validation errors found. + ''' + if task_vars is None: + task_vars = dict() + + result = super(ActionModule, self).run(tmp, task_vars) + del tmp # tmp no longer has any effect + + # This action can be called from anywhere, so pass in some info about what it is + # validating args for so the error results make some sense + result['validate_args_context'] = self._task.args.get('validate_args_context', {}) + + if 'argument_spec' not in self._task.args: + raise AnsibleError('"argument_spec" arg is required in args: %s' % self._task.args) + + # Get the task var called argument_spec. This will contain the arg spec + # data dict (for the proper entry point for a role). + argument_spec_data = self._task.args.get('argument_spec') + + # the values that were passed in and will be checked against argument_spec + provided_arguments = self._task.args.get('provided_arguments', {}) + + if not isinstance(argument_spec_data, dict): + raise AnsibleError('Incorrect type for argument_spec, expected dict and got %s' % type(argument_spec_data)) + + if not isinstance(provided_arguments, dict): + raise AnsibleError('Incorrect type for provided_arguments, expected dict and got %s' % type(provided_arguments)) + + args_from_vars = self.get_args_from_task_vars(argument_spec_data, task_vars) + validator = ArgumentSpecValidator(argument_spec_data) + validation_result = validator.validate(combine_vars(args_from_vars, provided_arguments), validate_role_argument_spec=True) + + if validation_result.error_messages: + result['failed'] = True + result['msg'] = 'Validation of arguments failed:\n%s' % '\n'.join(validation_result.error_messages) + result['argument_spec_data'] = argument_spec_data + result['argument_errors'] = validation_result.error_messages + return result + + result['changed'] = False + result['msg'] = 'The arg spec validation passed' + + return result diff --git a/lib/ansible/plugins/action/wait_for_connection.py b/lib/ansible/plugins/action/wait_for_connection.py new file mode 100644 index 0000000..8489c76 --- /dev/null +++ b/lib/ansible/plugins/action/wait_for_connection.py @@ -0,0 +1,120 @@ +# (c) 2017, Dag Wieers <dag@wieers.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# CI-required python3 boilerplate +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import time +from datetime import datetime, timedelta + +from ansible.module_utils._text import to_text +from ansible.plugins.action import ActionBase +from ansible.utils.display import Display + +display = Display() + + +class TimedOutException(Exception): + pass + + +class ActionModule(ActionBase): + TRANSFERS_FILES = False + _VALID_ARGS = frozenset(('connect_timeout', 'delay', 'sleep', 'timeout')) + + DEFAULT_CONNECT_TIMEOUT = 5 + DEFAULT_DELAY = 0 + DEFAULT_SLEEP = 1 + DEFAULT_TIMEOUT = 600 + + def do_until_success_or_timeout(self, what, timeout, connect_timeout, what_desc, sleep=1): + max_end_time = datetime.utcnow() + timedelta(seconds=timeout) + + e = None + while datetime.utcnow() < max_end_time: + try: + what(connect_timeout) + if what_desc: + display.debug("wait_for_connection: %s success" % what_desc) + return + except Exception as e: + error = e # PY3 compatibility to store exception for use outside of this block + if what_desc: + display.debug("wait_for_connection: %s fail (expected), retrying in %d seconds..." % (what_desc, sleep)) + time.sleep(sleep) + + raise TimedOutException("timed out waiting for %s: %s" % (what_desc, error)) + + def run(self, tmp=None, task_vars=None): + if task_vars is None: + task_vars = dict() + + connect_timeout = int(self._task.args.get('connect_timeout', self.DEFAULT_CONNECT_TIMEOUT)) + delay = int(self._task.args.get('delay', self.DEFAULT_DELAY)) + sleep = int(self._task.args.get('sleep', self.DEFAULT_SLEEP)) + timeout = int(self._task.args.get('timeout', self.DEFAULT_TIMEOUT)) + + if self._play_context.check_mode: + display.vvv("wait_for_connection: skipping for check_mode") + return dict(skipped=True) + + result = super(ActionModule, self).run(tmp, task_vars) + del tmp # tmp no longer has any effect + + def ping_module_test(connect_timeout): + ''' Test ping module, if available ''' + display.vvv("wait_for_connection: attempting ping module test") + # re-run interpreter discovery if we ran it in the first iteration + if self._discovered_interpreter_key: + task_vars['ansible_facts'].pop(self._discovered_interpreter_key, None) + # call connection reset between runs if it's there + try: + self._connection.reset() + except AttributeError: + pass + + ping_result = self._execute_module(module_name='ansible.legacy.ping', module_args=dict(), task_vars=task_vars) + + # Test module output + if ping_result['ping'] != 'pong': + raise Exception('ping test failed') + + start = datetime.now() + + if delay: + time.sleep(delay) + + try: + # If the connection has a transport_test method, use it first + if hasattr(self._connection, 'transport_test'): + self.do_until_success_or_timeout(self._connection.transport_test, timeout, connect_timeout, what_desc="connection port up", sleep=sleep) + + # Use the ping module test to determine end-to-end connectivity + self.do_until_success_or_timeout(ping_module_test, timeout, connect_timeout, what_desc="ping module test", sleep=sleep) + + except TimedOutException as e: + result['failed'] = True + result['msg'] = to_text(e) + + elapsed = datetime.now() - start + result['elapsed'] = elapsed.seconds + + # remove a temporary path we created + self._remove_tmp_path(self._connection._shell.tmpdir) + + return result diff --git a/lib/ansible/plugins/action/yum.py b/lib/ansible/plugins/action/yum.py new file mode 100644 index 0000000..d90a9e0 --- /dev/null +++ b/lib/ansible/plugins/action/yum.py @@ -0,0 +1,109 @@ +# (c) 2018, Ansible Project +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible.errors import AnsibleActionFail +from ansible.plugins.action import ActionBase +from ansible.utils.display import Display + +display = Display() + +VALID_BACKENDS = frozenset(('yum', 'yum4', 'dnf')) + + +class ActionModule(ActionBase): + + TRANSFERS_FILES = False + + def run(self, tmp=None, task_vars=None): + ''' + Action plugin handler for yum3 vs yum4(dnf) operations. + + Enables the yum module to use yum3 and/or yum4. Yum4 is a yum + command-line compatibility layer on top of dnf. Since the Ansible + modules for yum(aka yum3) and dnf(aka yum4) call each of yum3 and yum4's + python APIs natively on the backend, we need to handle this here and + pass off to the correct Ansible module to execute on the remote system. + ''' + + self._supports_check_mode = True + self._supports_async = True + + result = super(ActionModule, self).run(tmp, task_vars) + del tmp # tmp no longer has any effect + + # Carry-over concept from the package action plugin + if 'use' in self._task.args and 'use_backend' in self._task.args: + raise AnsibleActionFail("parameters are mutually exclusive: ('use', 'use_backend')") + + module = self._task.args.get('use', self._task.args.get('use_backend', 'auto')) + + if module == 'auto': + try: + if self._task.delegate_to: # if we delegate, we should use delegated host's facts + module = self._templar.template("{{hostvars['%s']['ansible_facts']['pkg_mgr']}}" % self._task.delegate_to) + else: + module = self._templar.template("{{ansible_facts.pkg_mgr}}") + except Exception: + pass # could not get it from template! + + if module not in VALID_BACKENDS: + facts = self._execute_module( + module_name="ansible.legacy.setup", module_args=dict(filter="ansible_pkg_mgr", gather_subset="!all"), + task_vars=task_vars) + display.debug("Facts %s" % facts) + module = facts.get("ansible_facts", {}).get("ansible_pkg_mgr", "auto") + if (not self._task.delegate_to or self._task.delegate_facts) and module != 'auto': + result['ansible_facts'] = {'pkg_mgr': module} + + if module not in VALID_BACKENDS: + result.update( + { + 'failed': True, + 'msg': ("Could not detect which major revision of yum is in use, which is required to determine module backend.", + "You should manually specify use_backend to tell the module whether to use the yum (yum3) or dnf (yum4) backend})"), + } + ) + + else: + if module == "yum4": + module = "dnf" + + # eliminate collisions with collections search while still allowing local override + module = 'ansible.legacy.' + module + + if not self._shared_loader_obj.module_loader.has_plugin(module): + result.update({'failed': True, 'msg': "Could not find a yum module backend for %s." % module}) + else: + # run either the yum (yum3) or dnf (yum4) backend module + new_module_args = self._task.args.copy() + if 'use_backend' in new_module_args: + del new_module_args['use_backend'] + if 'use' in new_module_args: + del new_module_args['use'] + + display.vvvv("Running %s as the backend for the yum action plugin" % module) + result.update(self._execute_module( + module_name=module, module_args=new_module_args, task_vars=task_vars, wrap_async=self._task.async_val)) + + # Cleanup + if not self._task.async_val: + # remove a temporary path we created + self._remove_tmp_path(self._connection._shell.tmpdir) + + return result diff --git a/lib/ansible/plugins/become/__init__.py b/lib/ansible/plugins/become/__init__.py new file mode 100644 index 0000000..9dacf22 --- /dev/null +++ b/lib/ansible/plugins/become/__init__.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- +# Copyright: (c) 2018, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import shlex + +from abc import abstractmethod +from random import choice +from string import ascii_lowercase +from gettext import dgettext + +from ansible.errors import AnsibleError +from ansible.module_utils._text import to_bytes +from ansible.plugins import AnsiblePlugin + + +def _gen_id(length=32): + ''' return random string used to identify the current privilege escalation ''' + return ''.join(choice(ascii_lowercase) for x in range(length)) + + +class BecomeBase(AnsiblePlugin): + + name = None # type: str | None + + # messages for detecting prompted password issues + fail = tuple() # type: tuple[str, ...] + missing = tuple() # type: tuple[str, ...] + + # many connection plugins cannot provide tty, set to True if your become + # plugin requires a tty, i.e su + require_tty = False + + # prompt to match + prompt = '' + + def __init__(self): + super(BecomeBase, self).__init__() + self._id = '' + self.success = '' + + def get_option(self, option, hostvars=None, playcontext=None): + """ Overrides the base get_option to provide a fallback to playcontext vars in case a 3rd party plugin did not + implement the base become options required in Ansible. """ + # TODO: add deprecation warning for ValueError in devel that removes the playcontext fallback + try: + return super(BecomeBase, self).get_option(option, hostvars=hostvars) + except KeyError: + pc_fallback = ['become_user', 'become_pass', 'become_flags', 'become_exe'] + if option not in pc_fallback: + raise + + return getattr(playcontext, option, None) + + def expect_prompt(self): + """This function assists connection plugins in determining if they need to wait for + a prompt. Both a prompt and a password are required. + """ + return self.prompt and self.get_option('become_pass') + + def _build_success_command(self, cmd, shell, noexe=False): + if not all((cmd, shell, self.success)): + return cmd + + try: + cmd = shlex.quote('%s %s %s %s' % (shell.ECHO, self.success, shell.COMMAND_SEP, cmd)) + except AttributeError: + # TODO: This should probably become some more robust functionality used to detect incompat + raise AnsibleError('The %s shell family is incompatible with the %s become plugin' % (shell.SHELL_FAMILY, self.name)) + exe = getattr(shell, 'executable', None) + if exe and not noexe: + cmd = '%s -c %s' % (exe, cmd) + return cmd + + @abstractmethod + def build_become_command(self, cmd, shell): + self._id = _gen_id() + self.success = 'BECOME-SUCCESS-%s' % self._id + + def check_success(self, b_output): + b_success = to_bytes(self.success) + return any(b_success in l.rstrip() for l in b_output.splitlines(True)) + + def check_password_prompt(self, b_output): + ''' checks if the expected password prompt exists in b_output ''' + if self.prompt: + b_prompt = to_bytes(self.prompt).strip() + return any(l.strip().startswith(b_prompt) for l in b_output.splitlines()) + return False + + def _check_password_error(self, b_out, msg): + ''' returns True/False if domain specific i18n version of msg is found in b_out ''' + b_fail = to_bytes(dgettext(self.name, msg)) + return b_fail and b_fail in b_out + + def check_incorrect_password(self, b_output): + for errstring in self.fail: + if self._check_password_error(b_output, errstring): + return True + return False + + def check_missing_password(self, b_output): + for errstring in self.missing: + if self._check_password_error(b_output, errstring): + return True + return False diff --git a/lib/ansible/plugins/become/runas.py b/lib/ansible/plugins/become/runas.py new file mode 100644 index 0000000..0b7d466 --- /dev/null +++ b/lib/ansible/plugins/become/runas.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +# Copyright: (c) 2018, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = """ + name: runas + short_description: Run As user + description: + - This become plugin allows your remote/login user to execute commands as another user via the windows runas facility. + author: ansible (@core) + version_added: "2.8" + options: + become_user: + description: User you 'become' to execute the task + ini: + - section: privilege_escalation + key: become_user + - section: runas_become_plugin + key: user + vars: + - name: ansible_become_user + - name: ansible_runas_user + env: + - name: ANSIBLE_BECOME_USER + - name: ANSIBLE_RUNAS_USER + keyword: + - name: become_user + required: True + become_flags: + description: Options to pass to runas, a space delimited list of k=v pairs + default: '' + ini: + - section: privilege_escalation + key: become_flags + - section: runas_become_plugin + key: flags + vars: + - name: ansible_become_flags + - name: ansible_runas_flags + env: + - name: ANSIBLE_BECOME_FLAGS + - name: ANSIBLE_RUNAS_FLAGS + keyword: + - name: become_flags + become_pass: + description: password + ini: + - section: runas_become_plugin + key: password + vars: + - name: ansible_become_password + - name: ansible_become_pass + - name: ansible_runas_pass + env: + - name: ANSIBLE_BECOME_PASS + - name: ANSIBLE_RUNAS_PASS + notes: + - runas is really implemented in the powershell module handler and as such can only be used with winrm connections. + - This plugin ignores the 'become_exe' setting as it uses an API and not an executable. + - The Secondary Logon service (seclogon) must be running to use runas +""" + +from ansible.plugins.become import BecomeBase + + +class BecomeModule(BecomeBase): + + name = 'runas' + + def build_become_command(self, cmd, shell): + # this is a noop, the 'real' runas is implemented + # inside the windows powershell execution subsystem + return cmd diff --git a/lib/ansible/plugins/become/su.py b/lib/ansible/plugins/become/su.py new file mode 100644 index 0000000..3a6fdea --- /dev/null +++ b/lib/ansible/plugins/become/su.py @@ -0,0 +1,168 @@ +# -*- coding: utf-8 -*- +# Copyright: (c) 2018, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = """ + name: su + short_description: Substitute User + description: + - This become plugin allows your remote/login user to execute commands as another user via the su utility. + author: ansible (@core) + version_added: "2.8" + options: + become_user: + description: User you 'become' to execute the task + default: root + ini: + - section: privilege_escalation + key: become_user + - section: su_become_plugin + key: user + vars: + - name: ansible_become_user + - name: ansible_su_user + env: + - name: ANSIBLE_BECOME_USER + - name: ANSIBLE_SU_USER + keyword: + - name: become_user + become_exe: + description: Su executable + default: su + ini: + - section: privilege_escalation + key: become_exe + - section: su_become_plugin + key: executable + vars: + - name: ansible_become_exe + - name: ansible_su_exe + env: + - name: ANSIBLE_BECOME_EXE + - name: ANSIBLE_SU_EXE + keyword: + - name: become_exe + become_flags: + description: Options to pass to su + default: '' + ini: + - section: privilege_escalation + key: become_flags + - section: su_become_plugin + key: flags + vars: + - name: ansible_become_flags + - name: ansible_su_flags + env: + - name: ANSIBLE_BECOME_FLAGS + - name: ANSIBLE_SU_FLAGS + keyword: + - name: become_flags + become_pass: + description: Password to pass to su + required: False + vars: + - name: ansible_become_password + - name: ansible_become_pass + - name: ansible_su_pass + env: + - name: ANSIBLE_BECOME_PASS + - name: ANSIBLE_SU_PASS + ini: + - section: su_become_plugin + key: password + prompt_l10n: + description: + - List of localized strings to match for prompt detection + - If empty we'll use the built in one + - Do NOT add a colon (:) to your custom entries. Ansible adds a colon at the end of each prompt; + if you add another one in your string, your prompt will fail with a "Timeout" error. + default: [] + type: list + elements: string + ini: + - section: su_become_plugin + key: localized_prompts + vars: + - name: ansible_su_prompt_l10n + env: + - name: ANSIBLE_SU_PROMPT_L10N +""" + +import re +import shlex + +from ansible.module_utils._text import to_bytes +from ansible.plugins.become import BecomeBase + + +class BecomeModule(BecomeBase): + + name = 'su' + + # messages for detecting prompted password issues + fail = ('Authentication failure',) + + SU_PROMPT_LOCALIZATIONS = [ + 'Password', + '암호', + 'パスワード', + 'Adgangskode', + 'Contraseña', + 'Contrasenya', + 'Hasło', + 'Heslo', + 'Jelszó', + 'Lösenord', + 'Mật khẩu', + 'Mot de passe', + 'Parola', + 'Parool', + 'Pasahitza', + 'Passord', + 'Passwort', + 'Salasana', + 'Sandi', + 'Senha', + 'Wachtwoord', + 'ססמה', + 'Лозинка', + 'Парола', + 'Пароль', + 'गुप्तशब्द', + 'शब्दकूट', + 'సంకేతపదము', + 'හස්පදය', + '密码', + '密碼', + '口令', + ] + + def check_password_prompt(self, b_output): + ''' checks if the expected password prompt exists in b_output ''' + + prompts = self.get_option('prompt_l10n') or self.SU_PROMPT_LOCALIZATIONS + b_password_string = b"|".join((br'(\w+\'s )?' + to_bytes(p)) for p in prompts) + # Colon or unicode fullwidth colon + b_password_string = b_password_string + to_bytes(u' ?(:|：) ?') + b_su_prompt_localizations_re = re.compile(b_password_string, flags=re.IGNORECASE) + return bool(b_su_prompt_localizations_re.match(b_output)) + + def build_become_command(self, cmd, shell): + super(BecomeModule, self).build_become_command(cmd, shell) + + # Prompt handling for ``su`` is more complicated, this + # is used to satisfy the connection plugin + self.prompt = True + + if not cmd: + return cmd + + exe = self.get_option('become_exe') or self.name + flags = self.get_option('become_flags') or '' + user = self.get_option('become_user') or '' + success_cmd = self._build_success_command(cmd, shell) + + return "%s %s %s -c %s" % (exe, flags, user, shlex.quote(success_cmd)) diff --git a/lib/ansible/plugins/become/sudo.py b/lib/ansible/plugins/become/sudo.py new file mode 100644 index 0000000..fb285f0 --- /dev/null +++ b/lib/ansible/plugins/become/sudo.py @@ -0,0 +1,121 @@ +# -*- coding: utf-8 -*- +# Copyright: (c) 2018, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = """ + name: sudo + short_description: Substitute User DO + description: + - This become plugin allows your remote/login user to execute commands as another user via the sudo utility. + author: ansible (@core) + version_added: "2.8" + options: + become_user: + description: User you 'become' to execute the task + default: root + ini: + - section: privilege_escalation + key: become_user + - section: sudo_become_plugin + key: user + vars: + - name: ansible_become_user + - name: ansible_sudo_user + env: + - name: ANSIBLE_BECOME_USER + - name: ANSIBLE_SUDO_USER + keyword: + - name: become_user + become_exe: + description: Sudo executable + default: sudo + ini: + - section: privilege_escalation + key: become_exe + - section: sudo_become_plugin + key: executable + vars: + - name: ansible_become_exe + - name: ansible_sudo_exe + env: + - name: ANSIBLE_BECOME_EXE + - name: ANSIBLE_SUDO_EXE + keyword: + - name: become_exe + become_flags: + description: Options to pass to sudo + default: -H -S -n + ini: + - section: privilege_escalation + key: become_flags + - section: sudo_become_plugin + key: flags + vars: + - name: ansible_become_flags + - name: ansible_sudo_flags + env: + - name: ANSIBLE_BECOME_FLAGS + - name: ANSIBLE_SUDO_FLAGS + keyword: + - name: become_flags + become_pass: + description: Password to pass to sudo + required: False + vars: + - name: ansible_become_password + - name: ansible_become_pass + - name: ansible_sudo_pass + env: + - name: ANSIBLE_BECOME_PASS + - name: ANSIBLE_SUDO_PASS + ini: + - section: sudo_become_plugin + key: password +""" + +import re +import shlex + +from ansible.plugins.become import BecomeBase + + +class BecomeModule(BecomeBase): + + name = 'sudo' + + # messages for detecting prompted password issues + fail = ('Sorry, try again.',) + missing = ('Sorry, a password is required to run sudo', 'sudo: a password is required') + + def build_become_command(self, cmd, shell): + super(BecomeModule, self).build_become_command(cmd, shell) + + if not cmd: + return cmd + + becomecmd = self.get_option('become_exe') or self.name + + flags = self.get_option('become_flags') or '' + prompt = '' + if self.get_option('become_pass'): + self.prompt = '[sudo via ansible, key=%s] password:' % self._id + if flags: # this could be simplified, but kept as is for now for backwards string matching + reflag = [] + for flag in shlex.split(flags): + if flag in ('-n', '--non-interactive'): + continue + elif not flag.startswith('--'): + # handle -XnxxX flags only + flag = re.sub(r'^(-\w*)n(\w*.*)', r'\1\2', flag) + reflag.append(flag) + flags = shlex.join(reflag) + + prompt = '-p "%s"' % (self.prompt) + + user = self.get_option('become_user') or '' + if user: + user = '-u %s' % (user) + + return ' '.join([becomecmd, flags, prompt, user, self._build_success_command(cmd, shell)]) diff --git a/lib/ansible/plugins/cache/__init__.py b/lib/ansible/plugins/cache/__init__.py new file mode 100644 index 0000000..3fb0d9b --- /dev/null +++ b/lib/ansible/plugins/cache/__init__.py @@ -0,0 +1,375 @@ +# (c) 2014, Michael DeHaan <michael.dehaan@gmail.com> +# (c) 2018, Ansible Project +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import copy +import errno +import os +import tempfile +import time + +from abc import abstractmethod +from collections.abc import MutableMapping + +from ansible import constants as C +from ansible.errors import AnsibleError +from ansible.module_utils._text import to_bytes, to_text +from ansible.plugins import AnsiblePlugin +from ansible.plugins.loader import cache_loader +from ansible.utils.collection_loader import resource_from_fqcr +from ansible.utils.display import Display + +display = Display() + + +class BaseCacheModule(AnsiblePlugin): + + # Backwards compat only. Just import the global display instead + _display = display + + def __init__(self, *args, **kwargs): + super(BaseCacheModule, self).__init__() + self.set_options(var_options=args, direct=kwargs) + + @abstractmethod + def get(self, key): + pass + + @abstractmethod + def set(self, key, value): + pass + + @abstractmethod + def keys(self): + pass + + @abstractmethod + def contains(self, key): + pass + + @abstractmethod + def delete(self, key): + pass + + @abstractmethod + def flush(self): + pass + + @abstractmethod + def copy(self): + pass + + +class BaseFileCacheModule(BaseCacheModule): + """ + A caching module backed by file based storage. + """ + def __init__(self, *args, **kwargs): + + try: + super(BaseFileCacheModule, self).__init__(*args, **kwargs) + self._cache_dir = self._get_cache_connection(self.get_option('_uri')) + self._timeout = float(self.get_option('_timeout')) + except KeyError: + self._cache_dir = self._get_cache_connection(C.CACHE_PLUGIN_CONNECTION) + self._timeout = float(C.CACHE_PLUGIN_TIMEOUT) + self.plugin_name = resource_from_fqcr(self.__module__) + self._cache = {} + self.validate_cache_connection() + + def _get_cache_connection(self, source): + if source: + try: + return os.path.expanduser(os.path.expandvars(source)) + except TypeError: + pass + + def validate_cache_connection(self): + if not self._cache_dir: + raise AnsibleError("error, '%s' cache plugin requires the 'fact_caching_connection' config option " + "to be set (to a writeable directory path)" % self.plugin_name) + + if not os.path.exists(self._cache_dir): + try: + os.makedirs(self._cache_dir) + except (OSError, IOError) as e: + raise AnsibleError("error in '%s' cache plugin while trying to create cache dir %s : %s" % (self.plugin_name, self._cache_dir, to_bytes(e))) + else: + for x in (os.R_OK, os.W_OK, os.X_OK): + if not os.access(self._cache_dir, x): + raise AnsibleError("error in '%s' cache, configured path (%s) does not have necessary permissions (rwx), disabling plugin" % ( + self.plugin_name, self._cache_dir)) + + def _get_cache_file_name(self, key): + prefix = self.get_option('_prefix') + if prefix: + cachefile = "%s/%s%s" % (self._cache_dir, prefix, key) + else: + cachefile = "%s/%s" % (self._cache_dir, key) + return cachefile + + def get(self, key): + """ This checks the in memory cache first as the fact was not expired at 'gather time' + and it would be problematic if the key did expire after some long running tasks and + user gets 'undefined' error in the same play """ + + if key not in self._cache: + + if self.has_expired(key) or key == "": + raise KeyError + + cachefile = self._get_cache_file_name(key) + try: + value = self._load(cachefile) + self._cache[key] = value + except ValueError as e: + display.warning("error in '%s' cache plugin while trying to read %s : %s. " + "Most likely a corrupt file, so erasing and failing." % (self.plugin_name, cachefile, to_bytes(e))) + self.delete(key) + raise AnsibleError("The cache file %s was corrupt, or did not otherwise contain valid data. " + "It has been removed, so you can re-run your command now." % cachefile) + except (OSError, IOError) as e: + display.warning("error in '%s' cache plugin while trying to read %s : %s" % (self.plugin_name, cachefile, to_bytes(e))) + raise KeyError + except Exception as e: + raise AnsibleError("Error while decoding the cache file %s: %s" % (cachefile, to_bytes(e))) + + return self._cache.get(key) + + def set(self, key, value): + + self._cache[key] = value + + cachefile = self._get_cache_file_name(key) + tmpfile_handle, tmpfile_path = tempfile.mkstemp(dir=os.path.dirname(cachefile)) + try: + try: + self._dump(value, tmpfile_path) + except (OSError, IOError) as e: + display.warning("error in '%s' cache plugin while trying to write to '%s' : %s" % (self.plugin_name, tmpfile_path, to_bytes(e))) + try: + os.rename(tmpfile_path, cachefile) + except (OSError, IOError) as e: + display.warning("error in '%s' cache plugin while trying to move '%s' to '%s' : %s" % (self.plugin_name, tmpfile_path, cachefile, to_bytes(e))) + finally: + try: + os.unlink(tmpfile_path) + except OSError: + pass + + def has_expired(self, key): + + if self._timeout == 0: + return False + + cachefile = self._get_cache_file_name(key) + try: + st = os.stat(cachefile) + except (OSError, IOError) as e: + if e.errno == errno.ENOENT: + return False + else: + display.warning("error in '%s' cache plugin while trying to stat %s : %s" % (self.plugin_name, cachefile, to_bytes(e))) + return False + + if time.time() - st.st_mtime <= self._timeout: + return False + + if key in self._cache: + del self._cache[key] + return True + + def keys(self): + # When using a prefix we must remove it from the key name before + # checking the expiry and returning it to the caller. Keys that do not + # share the same prefix cannot be fetched from the cache. + prefix = self.get_option('_prefix') + prefix_length = len(prefix) + keys = [] + for k in os.listdir(self._cache_dir): + if k.startswith('.') or not k.startswith(prefix): + continue + + k = k[prefix_length:] + if not self.has_expired(k): + keys.append(k) + + return keys + + def contains(self, key): + cachefile = self._get_cache_file_name(key) + + if key in self._cache: + return True + + if self.has_expired(key): + return False + try: + os.stat(cachefile) + return True + except (OSError, IOError) as e: + if e.errno == errno.ENOENT: + return False + else: + display.warning("error in '%s' cache plugin while trying to stat %s : %s" % (self.plugin_name, cachefile, to_bytes(e))) + + def delete(self, key): + try: + del self._cache[key] + except KeyError: + pass + try: + os.remove(self._get_cache_file_name(key)) + except (OSError, IOError): + pass # TODO: only pass on non existing? + + def flush(self): + self._cache = {} + for key in self.keys(): + self.delete(key) + + def copy(self): + ret = dict() + for key in self.keys(): + ret[key] = self.get(key) + return ret + + @abstractmethod + def _load(self, filepath): + """ + Read data from a filepath and return it as a value + + :arg filepath: The filepath to read from. + :returns: The value stored in the filepath + + This method reads from the file on disk and takes care of any parsing + and transformation of the data before returning it. The value + returned should be what Ansible would expect if it were uncached data. + + .. note:: Filehandles have advantages but calling code doesn't know + whether this file is text or binary, should be decoded, or accessed via + a library function. Therefore the API uses a filepath and opens + the file inside of the method. + """ + pass + + @abstractmethod + def _dump(self, value, filepath): + """ + Write data to a filepath + + :arg value: The value to store + :arg filepath: The filepath to store it at + """ + pass + + +class CachePluginAdjudicator(MutableMapping): + """ + Intermediary between a cache dictionary and a CacheModule + """ + def __init__(self, plugin_name='memory', **kwargs): + self._cache = {} + self._retrieved = {} + + self._plugin = cache_loader.get(plugin_name, **kwargs) + if not self._plugin: + raise AnsibleError('Unable to load the cache plugin (%s).' % plugin_name) + + self._plugin_name = plugin_name + + def update_cache_if_changed(self): + if self._retrieved != self._cache: + self.set_cache() + + def set_cache(self): + for top_level_cache_key in self._cache.keys(): + self._plugin.set(top_level_cache_key, self._cache[top_level_cache_key]) + self._retrieved = copy.deepcopy(self._cache) + + def load_whole_cache(self): + for key in self._plugin.keys(): + self._cache[key] = self._plugin.get(key) + + def __repr__(self): + return to_text(self._cache) + + def __iter__(self): + return iter(self.keys()) + + def __len__(self): + return len(self.keys()) + + def _do_load_key(self, key): + load = False + if all([ + key not in self._cache, + key not in self._retrieved, + self._plugin_name != 'memory', + self._plugin.contains(key), + ]): + load = True + return load + + def __getitem__(self, key): + if self._do_load_key(key): + try: + self._cache[key] = self._plugin.get(key) + except KeyError: + pass + else: + self._retrieved[key] = self._cache[key] + return self._cache[key] + + def get(self, key, default=None): + if self._do_load_key(key): + try: + self._cache[key] = self._plugin.get(key) + except KeyError as e: + pass + else: + self._retrieved[key] = self._cache[key] + return self._cache.get(key, default) + + def items(self): + return self._cache.items() + + def values(self): + return self._cache.values() + + def keys(self): + return self._cache.keys() + + def pop(self, key, *args): + if args: + return self._cache.pop(key, args[0]) + return self._cache.pop(key) + + def __delitem__(self, key): + del self._cache[key] + + def __setitem__(self, key, value): + self._cache[key] = value + + def flush(self): + self._plugin.flush() + self._cache = {} + + def update(self, value): + self._cache.update(value) diff --git a/lib/ansible/plugins/cache/base.py b/lib/ansible/plugins/cache/base.py new file mode 100644 index 0000000..692b1b3 --- /dev/null +++ b/lib/ansible/plugins/cache/base.py @@ -0,0 +1,21 @@ +# (c) 2017, ansible by Red Hat +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +# moved actual classes to __init__ kept here for backward compat with 3rd parties +from ansible.plugins.cache import BaseCacheModule, BaseFileCacheModule diff --git a/lib/ansible/plugins/cache/jsonfile.py b/lib/ansible/plugins/cache/jsonfile.py new file mode 100644 index 0000000..a26828a --- /dev/null +++ b/lib/ansible/plugins/cache/jsonfile.py @@ -0,0 +1,64 @@ +# (c) 2014, Brian Coca, Josh Drake, et al +# (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' + name: jsonfile + short_description: JSON formatted files. + description: + - This cache uses JSON formatted, per host, files saved to the filesystem. + version_added: "1.9" + author: Ansible Core (@ansible-core) + options: + _uri: + required: True + description: + - Path in which the cache plugin will save the JSON files + env: + - name: ANSIBLE_CACHE_PLUGIN_CONNECTION + ini: + - key: fact_caching_connection + section: defaults + type: path + _prefix: + description: User defined prefix to use when creating the JSON files + env: + - name: ANSIBLE_CACHE_PLUGIN_PREFIX + ini: + - key: fact_caching_prefix + section: defaults + _timeout: + default: 86400 + description: Expiration timeout for the cache plugin data + env: + - name: ANSIBLE_CACHE_PLUGIN_TIMEOUT + ini: + - key: fact_caching_timeout + section: defaults + type: integer +''' + +import codecs +import json + +from ansible.parsing.ajson import AnsibleJSONEncoder, AnsibleJSONDecoder +from ansible.plugins.cache import BaseFileCacheModule + + +class CacheModule(BaseFileCacheModule): + """ + A caching module backed by json files. + """ + + def _load(self, filepath): + # Valid JSON is always UTF-8 encoded. + with codecs.open(filepath, 'r', encoding='utf-8') as f: + return json.load(f, cls=AnsibleJSONDecoder) + + def _dump(self, value, filepath): + with codecs.open(filepath, 'w', encoding='utf-8') as f: + f.write(json.dumps(value, cls=AnsibleJSONEncoder, sort_keys=True, indent=4)) diff --git a/lib/ansible/plugins/cache/memory.py b/lib/ansible/plugins/cache/memory.py new file mode 100644 index 0000000..59f97b6 --- /dev/null +++ b/lib/ansible/plugins/cache/memory.py @@ -0,0 +1,53 @@ +# (c) 2014, Brian Coca, Josh Drake, et al +# (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' + name: memory + short_description: RAM backed, non persistent + description: + - RAM backed cache that is not persistent. + - This is the default used if no other plugin is specified. + - There are no options to configure. + version_added: historical + author: core team (@ansible-core) +''' + +from ansible.plugins.cache import BaseCacheModule + + +class CacheModule(BaseCacheModule): + + def __init__(self, *args, **kwargs): + self._cache = {} + + def get(self, key): + return self._cache.get(key) + + def set(self, key, value): + self._cache[key] = value + + def keys(self): + return self._cache.keys() + + def contains(self, key): + return key in self._cache + + def delete(self, key): + del self._cache[key] + + def flush(self): + self._cache = {} + + def copy(self): + return self._cache.copy() + + def __getstate__(self): + return self.copy() + + def __setstate__(self, data): + self._cache = data diff --git a/lib/ansible/plugins/callback/__init__.py b/lib/ansible/plugins/callback/__init__.py new file mode 100644 index 0000000..d4fc347 --- /dev/null +++ b/lib/ansible/plugins/callback/__init__.py @@ -0,0 +1,610 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import difflib +import json +import re +import sys +import textwrap + +from collections import OrderedDict +from collections.abc import MutableMapping +from copy import deepcopy + +from ansible import constants as C +from ansible.module_utils.common.text.converters import to_text +from ansible.module_utils.six import text_type +from ansible.parsing.ajson import AnsibleJSONEncoder +from ansible.parsing.yaml.dumper import AnsibleDumper +from ansible.parsing.yaml.objects import AnsibleUnicode +from ansible.plugins import AnsiblePlugin +from ansible.utils.color import stringc +from ansible.utils.display import Display +from ansible.utils.unsafe_proxy import AnsibleUnsafeText, NativeJinjaUnsafeText +from ansible.vars.clean import strip_internal_keys, module_response_deepcopy + +import yaml + +global_display = Display() + + +__all__ = ["CallbackBase"] + + +_DEBUG_ALLOWED_KEYS = frozenset(('msg', 'exception', 'warnings', 'deprecations')) +_YAML_TEXT_TYPES = (text_type, AnsibleUnicode, AnsibleUnsafeText, NativeJinjaUnsafeText) +# Characters that libyaml/pyyaml consider breaks +_YAML_BREAK_CHARS = '\n\x85\u2028\u2029' # NL, NEL, LS, PS +# regex representation of libyaml/pyyaml of a space followed by a break character +_SPACE_BREAK_RE = re.compile(fr' +([{_YAML_BREAK_CHARS}])') + + +class _AnsibleCallbackDumper(AnsibleDumper): + def __init__(self, lossy=False): + self._lossy = lossy + + def __call__(self, *args, **kwargs): + # pyyaml expects that we are passing an object that can be instantiated, but to + # smuggle the ``lossy`` configuration, we do that in ``__init__`` and then + # define this ``__call__`` that will mimic the ability for pyyaml to instantiate class + super().__init__(*args, **kwargs) + return self + + +def _should_use_block(scalar): + """Returns true if string should be in block format based on the existence of various newline separators""" + # This method of searching is faster than using a regex + for ch in _YAML_BREAK_CHARS: + if ch in scalar: + return True + return False + + +class _SpecialCharacterTranslator: + def __getitem__(self, ch): + # "special character" logic from pyyaml yaml.emitter.Emitter.analyze_scalar, translated to decimal + # for perf w/ str.translate + if (ch == 10 or + 32 <= ch <= 126 or + ch == 133 or + 160 <= ch <= 55295 or + 57344 <= ch <= 65533 or + 65536 <= ch < 1114111)\ + and ch != 65279: + return ch + return None + + +def _filter_yaml_special(scalar): + """Filter a string removing any character that libyaml/pyyaml declare as special""" + return scalar.translate(_SpecialCharacterTranslator()) + + +def _munge_data_for_lossy_yaml(scalar): + """Modify a string so that analyze_scalar in libyaml/pyyaml will allow block formatting""" + # we care more about readability than accuracy, so... + # ...libyaml/pyyaml does not permit trailing spaces for block scalars + scalar = scalar.rstrip() + # ...libyaml/pyyaml does not permit tabs for block scalars + scalar = scalar.expandtabs() + # ...libyaml/pyyaml only permits special characters for double quoted scalars + scalar = _filter_yaml_special(scalar) + # ...libyaml/pyyaml only permits spaces followed by breaks for double quoted scalars + return _SPACE_BREAK_RE.sub(r'\1', scalar) + + +def _pretty_represent_str(self, data): + """Uses block style for multi-line strings""" + data = text_type(data) + if _should_use_block(data): + style = '|' + if self._lossy: + data = _munge_data_for_lossy_yaml(data) + else: + style = self.default_style + + node = yaml.representer.ScalarNode('tag:yaml.org,2002:str', data, style=style) + if self.alias_key is not None: + self.represented_objects[self.alias_key] = node + return node + + +for data_type in _YAML_TEXT_TYPES: + _AnsibleCallbackDumper.add_representer( + data_type, + _pretty_represent_str + ) + + +class CallbackBase(AnsiblePlugin): + + ''' + This is a base ansible callback class that does nothing. New callbacks should + use this class as a base and override any callback methods they wish to execute + custom actions. + ''' + + def __init__(self, display=None, options=None): + if display: + self._display = display + else: + self._display = global_display + + if self._display.verbosity >= 4: + name = getattr(self, 'CALLBACK_NAME', 'unnamed') + ctype = getattr(self, 'CALLBACK_TYPE', 'old') + version = getattr(self, 'CALLBACK_VERSION', '1.0') + self._display.vvvv('Loading callback plugin %s of type %s, v%s from %s' % (name, ctype, version, sys.modules[self.__module__].__file__)) + + self.disabled = False + self.wants_implicit_tasks = False + + self._plugin_options = {} + if options is not None: + self.set_options(options) + + self._hide_in_debug = ('changed', 'failed', 'skipped', 'invocation', 'skip_reason') + + ''' helper for callbacks, so they don't all have to include deepcopy ''' + _copy_result = deepcopy + + def set_option(self, k, v): + self._plugin_options[k] = v + + def get_option(self, k): + return self._plugin_options[k] + + def set_options(self, task_keys=None, var_options=None, direct=None): + ''' This is different than the normal plugin method as callbacks get called early and really don't accept keywords. + Also _options was already taken for CLI args and callbacks use _plugin_options instead. + ''' + + # load from config + self._plugin_options = C.config.get_plugin_options(self.plugin_type, self._load_name, keys=task_keys, variables=var_options, direct=direct) + + @staticmethod + def host_label(result): + """Return label for the hostname (& delegated hostname) of a task + result. + """ + label = "%s" % result._host.get_name() + if result._task.delegate_to and result._task.delegate_to != result._host.get_name(): + # show delegated host + label += " -> %s" % result._task.delegate_to + # in case we have 'extra resolution' + ahost = result._result.get('_ansible_delegated_vars', {}).get('ansible_host', result._task.delegate_to) + if result._task.delegate_to != ahost: + label += "(%s)" % ahost + return label + + def _run_is_verbose(self, result, verbosity=0): + return ((self._display.verbosity > verbosity or result._result.get('_ansible_verbose_always', False) is True) + and result._result.get('_ansible_verbose_override', False) is False) + + def _dump_results(self, result, indent=None, sort_keys=True, keep_invocation=False, serialize=True): + try: + result_format = self.get_option('result_format') + except KeyError: + # Callback does not declare result_format nor extend result_format_callback + result_format = 'json' + + try: + pretty_results = self.get_option('pretty_results') + except KeyError: + # Callback does not declare pretty_results nor extend result_format_callback + pretty_results = None + + indent_conditions = ( + result.get('_ansible_verbose_always'), + pretty_results is None and result_format != 'json', + pretty_results is True, + self._display.verbosity > 2, + ) + + if not indent and any(indent_conditions): + indent = 4 + if pretty_results is False: + # pretty_results=False overrides any specified indentation + indent = None + + # All result keys stating with _ansible_ are internal, so remove them from the result before we output anything. + abridged_result = strip_internal_keys(module_response_deepcopy(result)) + + # remove invocation unless specifically wanting it + if not keep_invocation and self._display.verbosity < 3 and 'invocation' in result: + del abridged_result['invocation'] + + # remove diff information from screen output + if self._display.verbosity < 3 and 'diff' in result: + del abridged_result['diff'] + + # remove exception from screen output + if 'exception' in abridged_result: + del abridged_result['exception'] + + if not serialize: + # Just return ``abridged_result`` without going through serialization + # to permit callbacks to take advantage of ``_dump_results`` + # that want to further modify the result, or use custom serialization + return abridged_result + + if result_format == 'json': + try: + return json.dumps(abridged_result, cls=AnsibleJSONEncoder, indent=indent, ensure_ascii=False, sort_keys=sort_keys) + except TypeError: + # Python3 bug: throws an exception when keys are non-homogenous types: + # https://bugs.python.org/issue25457 + # sort into an OrderedDict and then json.dumps() that instead + if not OrderedDict: + raise + return json.dumps(OrderedDict(sorted(abridged_result.items(), key=to_text)), + cls=AnsibleJSONEncoder, indent=indent, + ensure_ascii=False, sort_keys=False) + elif result_format == 'yaml': + # None is a sentinel in this case that indicates default behavior + # default behavior for yaml is to prettify results + lossy = pretty_results in (None, True) + if lossy: + # if we already have stdout, we don't need stdout_lines + if 'stdout' in abridged_result and 'stdout_lines' in abridged_result: + abridged_result['stdout_lines'] = '<omitted>' + + # if we already have stderr, we don't need stderr_lines + if 'stderr' in abridged_result and 'stderr_lines' in abridged_result: + abridged_result['stderr_lines'] = '<omitted>' + + return '\n%s' % textwrap.indent( + yaml.dump( + abridged_result, + allow_unicode=True, + Dumper=_AnsibleCallbackDumper(lossy=lossy), + default_flow_style=False, + indent=indent, + # sort_keys=sort_keys # This requires PyYAML>=5.1 + ), + ' ' * (indent or 4) + ) + + def _handle_warnings(self, res): + ''' display warnings, if enabled and any exist in the result ''' + if C.ACTION_WARNINGS: + if 'warnings' in res and res['warnings']: + for warning in res['warnings']: + self._display.warning(warning) + del res['warnings'] + if 'deprecations' in res and res['deprecations']: + for warning in res['deprecations']: + self._display.deprecated(**warning) + del res['deprecations'] + + def _handle_exception(self, result, use_stderr=False): + + if 'exception' in result: + msg = "An exception occurred during task execution. " + exception_str = to_text(result['exception']) + if self._display.verbosity < 3: + # extract just the actual error message from the exception text + error = exception_str.strip().split('\n')[-1] + msg += "To see the full traceback, use -vvv. The error was: %s" % error + else: + msg = "The full traceback is:\n" + exception_str + del result['exception'] + + self._display.display(msg, color=C.COLOR_ERROR, stderr=use_stderr) + + def _serialize_diff(self, diff): + try: + result_format = self.get_option('result_format') + except KeyError: + # Callback does not declare result_format nor extend result_format_callback + result_format = 'json' + + try: + pretty_results = self.get_option('pretty_results') + except KeyError: + # Callback does not declare pretty_results nor extend result_format_callback + pretty_results = None + + if result_format == 'json': + return json.dumps(diff, sort_keys=True, indent=4, separators=(u',', u': ')) + u'\n' + elif result_format == 'yaml': + # None is a sentinel in this case that indicates default behavior + # default behavior for yaml is to prettify results + lossy = pretty_results in (None, True) + return '%s\n' % textwrap.indent( + yaml.dump( + diff, + allow_unicode=True, + Dumper=_AnsibleCallbackDumper(lossy=lossy), + default_flow_style=False, + indent=4, + # sort_keys=sort_keys # This requires PyYAML>=5.1 + ), + ' ' + ) + + def _get_diff(self, difflist): + + if not isinstance(difflist, list): + difflist = [difflist] + + ret = [] + for diff in difflist: + if 'dst_binary' in diff: + ret.append(u"diff skipped: destination file appears to be binary\n") + if 'src_binary' in diff: + ret.append(u"diff skipped: source file appears to be binary\n") + if 'dst_larger' in diff: + ret.append(u"diff skipped: destination file size is greater than %d\n" % diff['dst_larger']) + if 'src_larger' in diff: + ret.append(u"diff skipped: source file size is greater than %d\n" % diff['src_larger']) + if 'before' in diff and 'after' in diff: + # format complex structures into 'files' + for x in ['before', 'after']: + if isinstance(diff[x], MutableMapping): + diff[x] = self._serialize_diff(diff[x]) + elif diff[x] is None: + diff[x] = '' + if 'before_header' in diff: + before_header = u"before: %s" % diff['before_header'] + else: + before_header = u'before' + if 'after_header' in diff: + after_header = u"after: %s" % diff['after_header'] + else: + after_header = u'after' + before_lines = diff['before'].splitlines(True) + after_lines = diff['after'].splitlines(True) + if before_lines and not before_lines[-1].endswith(u'\n'): + before_lines[-1] += u'\n\\ No newline at end of file\n' + if after_lines and not after_lines[-1].endswith('\n'): + after_lines[-1] += u'\n\\ No newline at end of file\n' + differ = difflib.unified_diff(before_lines, + after_lines, + fromfile=before_header, + tofile=after_header, + fromfiledate=u'', + tofiledate=u'', + n=C.DIFF_CONTEXT) + difflines = list(differ) + has_diff = False + for line in difflines: + has_diff = True + if line.startswith(u'+'): + line = stringc(line, C.COLOR_DIFF_ADD) + elif line.startswith(u'-'): + line = stringc(line, C.COLOR_DIFF_REMOVE) + elif line.startswith(u'@@'): + line = stringc(line, C.COLOR_DIFF_LINES) + ret.append(line) + if has_diff: + ret.append('\n') + if 'prepared' in diff: + ret.append(diff['prepared']) + return u''.join(ret) + + def _get_item_label(self, result): + ''' retrieves the value to be displayed as a label for an item entry from a result object''' + if result.get('_ansible_no_log', False): + item = "(censored due to no_log)" + else: + item = result.get('_ansible_item_label', result.get('item')) + return item + + def _process_items(self, result): + # just remove them as now they get handled by individual callbacks + del result._result['results'] + + def _clean_results(self, result, task_name): + ''' removes data from results for display ''' + + # mostly controls that debug only outputs what it was meant to + if task_name in C._ACTION_DEBUG: + if 'msg' in result: + # msg should be alone + for key in list(result.keys()): + if key not in _DEBUG_ALLOWED_KEYS and not key.startswith('_'): + result.pop(key) + else: + # 'var' value as field, so eliminate others and what is left should be varname + for hidme in self._hide_in_debug: + result.pop(hidme, None) + + def _print_task_path(self, task, color=C.COLOR_DEBUG): + path = task.get_path() + if path: + self._display.display(u"task path: %s" % path, color=color) + + def set_play_context(self, play_context): + pass + + def on_any(self, *args, **kwargs): + pass + + def runner_on_failed(self, host, res, ignore_errors=False): + pass + + def runner_on_ok(self, host, res): + pass + + def runner_on_skipped(self, host, item=None): + pass + + def runner_on_unreachable(self, host, res): + pass + + def runner_on_no_hosts(self): + pass + + def runner_on_async_poll(self, host, res, jid, clock): + pass + + def runner_on_async_ok(self, host, res, jid): + pass + + def runner_on_async_failed(self, host, res, jid): + pass + + def playbook_on_start(self): + pass + + def playbook_on_notify(self, host, handler): + pass + + def playbook_on_no_hosts_matched(self): + pass + + def playbook_on_no_hosts_remaining(self): + pass + + def playbook_on_task_start(self, name, is_conditional): + pass + + def playbook_on_vars_prompt(self, varname, private=True, prompt=None, encrypt=None, confirm=False, salt_size=None, salt=None, default=None, unsafe=None): + pass + + def playbook_on_setup(self): + pass + + def playbook_on_import_for_host(self, host, imported_file): + pass + + def playbook_on_not_import_for_host(self, host, missing_file): + pass + + def playbook_on_play_start(self, name): + pass + + def playbook_on_stats(self, stats): + pass + + def on_file_diff(self, host, diff): + pass + + # V2 METHODS, by default they call v1 counterparts if possible + def v2_on_any(self, *args, **kwargs): + self.on_any(args, kwargs) + + def v2_runner_on_failed(self, result, ignore_errors=False): + host = result._host.get_name() + self.runner_on_failed(host, result._result, ignore_errors) + + def v2_runner_on_ok(self, result): + host = result._host.get_name() + self.runner_on_ok(host, result._result) + + def v2_runner_on_skipped(self, result): + if C.DISPLAY_SKIPPED_HOSTS: + host = result._host.get_name() + self.runner_on_skipped(host, self._get_item_label(getattr(result._result, 'results', {}))) + + def v2_runner_on_unreachable(self, result): + host = result._host.get_name() + self.runner_on_unreachable(host, result._result) + + def v2_runner_on_async_poll(self, result): + host = result._host.get_name() + jid = result._result.get('ansible_job_id') + # FIXME, get real clock + clock = 0 + self.runner_on_async_poll(host, result._result, jid, clock) + + def v2_runner_on_async_ok(self, result): + host = result._host.get_name() + jid = result._result.get('ansible_job_id') + self.runner_on_async_ok(host, result._result, jid) + + def v2_runner_on_async_failed(self, result): + host = result._host.get_name() + # Attempt to get the async job ID. If the job does not finish before the + # async timeout value, the ID may be within the unparsed 'async_result' dict. + jid = result._result.get('ansible_job_id') + if not jid and 'async_result' in result._result: + jid = result._result['async_result'].get('ansible_job_id') + self.runner_on_async_failed(host, result._result, jid) + + def v2_playbook_on_start(self, playbook): + self.playbook_on_start() + + def v2_playbook_on_notify(self, handler, host): + self.playbook_on_notify(host, handler) + + def v2_playbook_on_no_hosts_matched(self): + self.playbook_on_no_hosts_matched() + + def v2_playbook_on_no_hosts_remaining(self): + self.playbook_on_no_hosts_remaining() + + def v2_playbook_on_task_start(self, task, is_conditional): + self.playbook_on_task_start(task.name, is_conditional) + + # FIXME: not called + def v2_playbook_on_cleanup_task_start(self, task): + pass # no v1 correspondence + + def v2_playbook_on_handler_task_start(self, task): + pass # no v1 correspondence + + def v2_playbook_on_vars_prompt(self, varname, private=True, prompt=None, encrypt=None, confirm=False, salt_size=None, salt=None, default=None, unsafe=None): + self.playbook_on_vars_prompt(varname, private, prompt, encrypt, confirm, salt_size, salt, default, unsafe) + + # FIXME: not called + def v2_playbook_on_import_for_host(self, result, imported_file): + host = result._host.get_name() + self.playbook_on_import_for_host(host, imported_file) + + # FIXME: not called + def v2_playbook_on_not_import_for_host(self, result, missing_file): + host = result._host.get_name() + self.playbook_on_not_import_for_host(host, missing_file) + + def v2_playbook_on_play_start(self, play): + self.playbook_on_play_start(play.name) + + def v2_playbook_on_stats(self, stats): + self.playbook_on_stats(stats) + + def v2_on_file_diff(self, result): + if 'diff' in result._result: + host = result._host.get_name() + self.on_file_diff(host, result._result['diff']) + + def v2_playbook_on_include(self, included_file): + pass # no v1 correspondence + + def v2_runner_item_on_ok(self, result): + pass + + def v2_runner_item_on_failed(self, result): + pass + + def v2_runner_item_on_skipped(self, result): + pass + + def v2_runner_retry(self, result): + pass + + def v2_runner_on_start(self, host, task): + """Event used when host begins execution of a task + + .. versionadded:: 2.8 + """ + pass diff --git a/lib/ansible/plugins/callback/default.py b/lib/ansible/plugins/callback/default.py new file mode 100644 index 0000000..54ef452 --- /dev/null +++ b/lib/ansible/plugins/callback/default.py @@ -0,0 +1,409 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' + name: default + type: stdout + short_description: default Ansible screen output + version_added: historical + description: + - This is the default output callback for ansible-playbook. + extends_documentation_fragment: + - default_callback + - result_format_callback + requirements: + - set as stdout in configuration +''' + + +from ansible import constants as C +from ansible import context +from ansible.playbook.task_include import TaskInclude +from ansible.plugins.callback import CallbackBase +from ansible.utils.color import colorize, hostcolor +from ansible.utils.fqcn import add_internal_fqcns + + +class CallbackModule(CallbackBase): + + ''' + This is the default callback interface, which simply prints messages + to stdout when new callback events are received. + ''' + + CALLBACK_VERSION = 2.0 + CALLBACK_TYPE = 'stdout' + CALLBACK_NAME = 'default' + + def __init__(self): + + self._play = None + self._last_task_banner = None + self._last_task_name = None + self._task_type_cache = {} + super(CallbackModule, self).__init__() + + def v2_runner_on_failed(self, result, ignore_errors=False): + + host_label = self.host_label(result) + self._clean_results(result._result, result._task.action) + + if self._last_task_banner != result._task._uuid: + self._print_task_banner(result._task) + + self._handle_exception(result._result, use_stderr=self.get_option('display_failed_stderr')) + self._handle_warnings(result._result) + + if result._task.loop and 'results' in result._result: + self._process_items(result) + + else: + if self._display.verbosity < 2 and self.get_option('show_task_path_on_failure'): + self._print_task_path(result._task) + msg = "fatal: [%s]: FAILED! => %s" % (host_label, self._dump_results(result._result)) + self._display.display(msg, color=C.COLOR_ERROR, stderr=self.get_option('display_failed_stderr')) + + if ignore_errors: + self._display.display("...ignoring", color=C.COLOR_SKIP) + + def v2_runner_on_ok(self, result): + + host_label = self.host_label(result) + + if isinstance(result._task, TaskInclude): + if self._last_task_banner != result._task._uuid: + self._print_task_banner(result._task) + return + elif result._result.get('changed', False): + if self._last_task_banner != result._task._uuid: + self._print_task_banner(result._task) + + msg = "changed: [%s]" % (host_label,) + color = C.COLOR_CHANGED + else: + if not self.get_option('display_ok_hosts'): + return + + if self._last_task_banner != result._task._uuid: + self._print_task_banner(result._task) + + msg = "ok: [%s]" % (host_label,) + color = C.COLOR_OK + + self._handle_warnings(result._result) + + if result._task.loop and 'results' in result._result: + self._process_items(result) + else: + self._clean_results(result._result, result._task.action) + + if self._run_is_verbose(result): + msg += " => %s" % (self._dump_results(result._result),) + self._display.display(msg, color=color) + + def v2_runner_on_skipped(self, result): + + if self.get_option('display_skipped_hosts'): + + self._clean_results(result._result, result._task.action) + + if self._last_task_banner != result._task._uuid: + self._print_task_banner(result._task) + + if result._task.loop is not None and 'results' in result._result: + self._process_items(result) + + msg = "skipping: [%s]" % result._host.get_name() + if self._run_is_verbose(result): + msg += " => %s" % self._dump_results(result._result) + self._display.display(msg, color=C.COLOR_SKIP) + + def v2_runner_on_unreachable(self, result): + if self._last_task_banner != result._task._uuid: + self._print_task_banner(result._task) + + host_label = self.host_label(result) + msg = "fatal: [%s]: UNREACHABLE! => %s" % (host_label, self._dump_results(result._result)) + self._display.display(msg, color=C.COLOR_UNREACHABLE, stderr=self.get_option('display_failed_stderr')) + + if result._task.ignore_unreachable: + self._display.display("...ignoring", color=C.COLOR_SKIP) + + def v2_playbook_on_no_hosts_matched(self): + self._display.display("skipping: no hosts matched", color=C.COLOR_SKIP) + + def v2_playbook_on_no_hosts_remaining(self): + self._display.banner("NO MORE HOSTS LEFT") + + def v2_playbook_on_task_start(self, task, is_conditional): + self._task_start(task, prefix='TASK') + + def _task_start(self, task, prefix=None): + # Cache output prefix for task if provided + # This is needed to properly display 'RUNNING HANDLER' and similar + # when hiding skipped/ok task results + if prefix is not None: + self._task_type_cache[task._uuid] = prefix + + # Preserve task name, as all vars may not be available for templating + # when we need it later + if self._play.strategy in add_internal_fqcns(('free', 'host_pinned')): + # Explicitly set to None for strategy free/host_pinned to account for any cached + # task title from a previous non-free play + self._last_task_name = None + else: + self._last_task_name = task.get_name().strip() + + # Display the task banner immediately if we're not doing any filtering based on task result + if self.get_option('display_skipped_hosts') and self.get_option('display_ok_hosts'): + self._print_task_banner(task) + + def _print_task_banner(self, task): + # args can be specified as no_log in several places: in the task or in + # the argument spec. We can check whether the task is no_log but the + # argument spec can't be because that is only run on the target + # machine and we haven't run it thereyet at this time. + # + # So we give people a config option to affect display of the args so + # that they can secure this if they feel that their stdout is insecure + # (shoulder surfing, logging stdout straight to a file, etc). + args = '' + if not task.no_log and C.DISPLAY_ARGS_TO_STDOUT: + args = u', '.join(u'%s=%s' % a for a in task.args.items()) + args = u' %s' % args + + prefix = self._task_type_cache.get(task._uuid, 'TASK') + + # Use cached task name + task_name = self._last_task_name + if task_name is None: + task_name = task.get_name().strip() + + if task.check_mode and self.get_option('check_mode_markers'): + checkmsg = " [CHECK MODE]" + else: + checkmsg = "" + self._display.banner(u"%s [%s%s]%s" % (prefix, task_name, args, checkmsg)) + + if self._display.verbosity >= 2: + self._print_task_path(task) + + self._last_task_banner = task._uuid + + def v2_playbook_on_cleanup_task_start(self, task): + self._task_start(task, prefix='CLEANUP TASK') + + def v2_playbook_on_handler_task_start(self, task): + self._task_start(task, prefix='RUNNING HANDLER') + + def v2_runner_on_start(self, host, task): + if self.get_option('show_per_host_start'): + self._display.display(" [started %s on %s]" % (task, host), color=C.COLOR_OK) + + def v2_playbook_on_play_start(self, play): + name = play.get_name().strip() + if play.check_mode and self.get_option('check_mode_markers'): + checkmsg = " [CHECK MODE]" + else: + checkmsg = "" + if not name: + msg = u"PLAY%s" % checkmsg + else: + msg = u"PLAY [%s]%s" % (name, checkmsg) + + self._play = play + + self._display.banner(msg) + + def v2_on_file_diff(self, result): + if result._task.loop and 'results' in result._result: + for res in result._result['results']: + if 'diff' in res and res['diff'] and res.get('changed', False): + diff = self._get_diff(res['diff']) + if diff: + if self._last_task_banner != result._task._uuid: + self._print_task_banner(result._task) + self._display.display(diff) + elif 'diff' in result._result and result._result['diff'] and result._result.get('changed', False): + diff = self._get_diff(result._result['diff']) + if diff: + if self._last_task_banner != result._task._uuid: + self._print_task_banner(result._task) + self._display.display(diff) + + def v2_runner_item_on_ok(self, result): + + host_label = self.host_label(result) + if isinstance(result._task, TaskInclude): + return + elif result._result.get('changed', False): + if self._last_task_banner != result._task._uuid: + self._print_task_banner(result._task) + + msg = 'changed' + color = C.COLOR_CHANGED + else: + if not self.get_option('display_ok_hosts'): + return + + if self._last_task_banner != result._task._uuid: + self._print_task_banner(result._task) + + msg = 'ok' + color = C.COLOR_OK + + msg = "%s: [%s] => (item=%s)" % (msg, host_label, self._get_item_label(result._result)) + self._clean_results(result._result, result._task.action) + if self._run_is_verbose(result): + msg += " => %s" % self._dump_results(result._result) + self._display.display(msg, color=color) + + def v2_runner_item_on_failed(self, result): + if self._last_task_banner != result._task._uuid: + self._print_task_banner(result._task) + + host_label = self.host_label(result) + self._clean_results(result._result, result._task.action) + self._handle_exception(result._result, use_stderr=self.get_option('display_failed_stderr')) + + msg = "failed: [%s]" % (host_label,) + self._handle_warnings(result._result) + self._display.display( + msg + " (item=%s) => %s" % (self._get_item_label(result._result), self._dump_results(result._result)), + color=C.COLOR_ERROR, + stderr=self.get_option('display_failed_stderr') + ) + + def v2_runner_item_on_skipped(self, result): + if self.get_option('display_skipped_hosts'): + if self._last_task_banner != result._task._uuid: + self._print_task_banner(result._task) + + self._clean_results(result._result, result._task.action) + msg = "skipping: [%s] => (item=%s) " % (result._host.get_name(), self._get_item_label(result._result)) + if self._run_is_verbose(result): + msg += " => %s" % self._dump_results(result._result) + self._display.display(msg, color=C.COLOR_SKIP) + + def v2_playbook_on_include(self, included_file): + msg = 'included: %s for %s' % (included_file._filename, ", ".join([h.name for h in included_file._hosts])) + label = self._get_item_label(included_file._vars) + if label: + msg += " => (item=%s)" % label + self._display.display(msg, color=C.COLOR_SKIP) + + def v2_playbook_on_stats(self, stats): + self._display.banner("PLAY RECAP") + + hosts = sorted(stats.processed.keys()) + for h in hosts: + t = stats.summarize(h) + + self._display.display( + u"%s : %s %s %s %s %s %s %s" % ( + hostcolor(h, t), + colorize(u'ok', t['ok'], C.COLOR_OK), + colorize(u'changed', t['changed'], C.COLOR_CHANGED), + colorize(u'unreachable', t['unreachable'], C.COLOR_UNREACHABLE), + colorize(u'failed', t['failures'], C.COLOR_ERROR), + colorize(u'skipped', t['skipped'], C.COLOR_SKIP), + colorize(u'rescued', t['rescued'], C.COLOR_OK), + colorize(u'ignored', t['ignored'], C.COLOR_WARN), + ), + screen_only=True + ) + + self._display.display( + u"%s : %s %s %s %s %s %s %s" % ( + hostcolor(h, t, False), + colorize(u'ok', t['ok'], None), + colorize(u'changed', t['changed'], None), + colorize(u'unreachable', t['unreachable'], None), + colorize(u'failed', t['failures'], None), + colorize(u'skipped', t['skipped'], None), + colorize(u'rescued', t['rescued'], None), + colorize(u'ignored', t['ignored'], None), + ), + log_only=True + ) + + self._display.display("", screen_only=True) + + # print custom stats if required + if stats.custom and self.get_option('show_custom_stats'): + self._display.banner("CUSTOM STATS: ") + # per host + # TODO: come up with 'pretty format' + for k in sorted(stats.custom.keys()): + if k == '_run': + continue + self._display.display('\t%s: %s' % (k, self._dump_results(stats.custom[k], indent=1).replace('\n', ''))) + + # print per run custom stats + if '_run' in stats.custom: + self._display.display("", screen_only=True) + self._display.display('\tRUN: %s' % self._dump_results(stats.custom['_run'], indent=1).replace('\n', '')) + self._display.display("", screen_only=True) + + if context.CLIARGS['check'] and self.get_option('check_mode_markers'): + self._display.banner("DRY RUN") + + def v2_playbook_on_start(self, playbook): + if self._display.verbosity > 1: + from os.path import basename + self._display.banner("PLAYBOOK: %s" % basename(playbook._file_name)) + + # show CLI arguments + if self._display.verbosity > 3: + if context.CLIARGS.get('args'): + self._display.display('Positional arguments: %s' % ' '.join(context.CLIARGS['args']), + color=C.COLOR_VERBOSE, screen_only=True) + + for argument in (a for a in context.CLIARGS if a != 'args'): + val = context.CLIARGS[argument] + if val: + self._display.display('%s: %s' % (argument, val), color=C.COLOR_VERBOSE, screen_only=True) + + if context.CLIARGS['check'] and self.get_option('check_mode_markers'): + self._display.banner("DRY RUN") + + def v2_runner_retry(self, result): + task_name = result.task_name or result._task + host_label = self.host_label(result) + msg = "FAILED - RETRYING: [%s]: %s (%d retries left)." % (host_label, task_name, result._result['retries'] - result._result['attempts']) + if self._run_is_verbose(result, verbosity=2): + msg += "Result was: %s" % self._dump_results(result._result) + self._display.display(msg, color=C.COLOR_DEBUG) + + def v2_runner_on_async_poll(self, result): + host = result._host.get_name() + jid = result._result.get('ansible_job_id') + started = result._result.get('started') + finished = result._result.get('finished') + self._display.display( + 'ASYNC POLL on %s: jid=%s started=%s finished=%s' % (host, jid, started, finished), + color=C.COLOR_DEBUG + ) + + def v2_runner_on_async_ok(self, result): + host = result._host.get_name() + jid = result._result.get('ansible_job_id') + self._display.display("ASYNC OK on %s: jid=%s" % (host, jid), color=C.COLOR_DEBUG) + + def v2_runner_on_async_failed(self, result): + host = result._host.get_name() + + # Attempt to get the async job ID. If the job does not finish before the + # async timeout value, the ID may be within the unparsed 'async_result' dict. + jid = result._result.get('ansible_job_id') + if not jid and 'async_result' in result._result: + jid = result._result['async_result'].get('ansible_job_id') + self._display.display("ASYNC FAILED on %s: jid=%s" % (host, jid), color=C.COLOR_DEBUG) + + def v2_playbook_on_notify(self, handler, host): + if self._display.verbosity > 1: + self._display.display("NOTIFIED HANDLER %s for %s" % (handler.get_name(), host), color=C.COLOR_VERBOSE, screen_only=True) diff --git a/lib/ansible/plugins/callback/junit.py b/lib/ansible/plugins/callback/junit.py new file mode 100644 index 0000000..75cdbc7 --- /dev/null +++ b/lib/ansible/plugins/callback/junit.py @@ -0,0 +1,364 @@ +# (c) 2016 Matt Clay <matt@mystile.com> +# (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' + name: junit + type: aggregate + short_description: write playbook output to a JUnit file. + version_added: historical + description: + - This callback writes playbook output to a JUnit formatted XML file. + - "Tasks show up in the report as follows: + 'ok': pass + 'failed' with 'EXPECTED FAILURE' in the task name: pass + 'failed' with 'TOGGLE RESULT' in the task name: pass + 'ok' with 'TOGGLE RESULT' in the task name: failure + 'failed' due to an exception: error + 'failed' for other reasons: failure + 'skipped': skipped" + options: + output_dir: + name: JUnit output dir + default: ~/.ansible.log + description: Directory to write XML files to. + env: + - name: JUNIT_OUTPUT_DIR + task_class: + name: JUnit Task class + default: False + description: Configure the output to be one class per yaml file + env: + - name: JUNIT_TASK_CLASS + task_relative_path: + name: JUnit Task relative path + default: none + description: Configure the output to use relative paths to given directory + version_added: "2.8" + env: + - name: JUNIT_TASK_RELATIVE_PATH + replace_out_of_tree_path: + name: Replace out of tree path + default: none + description: Replace the directory portion of an out-of-tree relative task path with the given placeholder + version_added: "2.12.3" + env: + - name: JUNIT_REPLACE_OUT_OF_TREE_PATH + fail_on_change: + name: JUnit fail on change + default: False + description: Consider any tasks reporting "changed" as a junit test failure + env: + - name: JUNIT_FAIL_ON_CHANGE + fail_on_ignore: + name: JUnit fail on ignore + default: False + description: Consider failed tasks as a junit test failure even if ignore_on_error is set + env: + - name: JUNIT_FAIL_ON_IGNORE + include_setup_tasks_in_report: + name: JUnit include setup tasks in report + default: True + description: Should the setup tasks be included in the final report + env: + - name: JUNIT_INCLUDE_SETUP_TASKS_IN_REPORT + hide_task_arguments: + name: Hide the arguments for a task + default: False + description: Hide the arguments for a task + version_added: "2.8" + env: + - name: JUNIT_HIDE_TASK_ARGUMENTS + test_case_prefix: + name: Prefix to find actual test cases + default: <empty> + description: Consider a task only as test case if it has this value as prefix. Additionally failing tasks are recorded as failed test cases. + version_added: "2.8" + env: + - name: JUNIT_TEST_CASE_PREFIX + requirements: + - enable in configuration +''' + +import os +import time +import re + +from ansible import constants as C +from ansible.module_utils._text import to_bytes, to_text +from ansible.plugins.callback import CallbackBase +from ansible.utils._junit_xml import ( + TestCase, + TestError, + TestFailure, + TestSuite, + TestSuites, +) + + +class CallbackModule(CallbackBase): + """ + This callback writes playbook output to a JUnit formatted XML file. + + Tasks show up in the report as follows: + 'ok': pass + 'failed' with 'EXPECTED FAILURE' in the task name: pass + 'failed' with 'TOGGLE RESULT' in the task name: pass + 'ok' with 'TOGGLE RESULT' in the task name: failure + 'failed' due to an exception: error + 'failed' for other reasons: failure + 'skipped': skipped + + This plugin makes use of the following environment variables: + JUNIT_OUTPUT_DIR (optional): Directory to write XML files to. + Default: ~/.ansible.log + JUNIT_TASK_CLASS (optional): Configure the output to be one class per yaml file + Default: False + JUNIT_TASK_RELATIVE_PATH (optional): Configure the output to use relative paths to given directory + Default: none + JUNIT_FAIL_ON_CHANGE (optional): Consider any tasks reporting "changed" as a junit test failure + Default: False + JUNIT_FAIL_ON_IGNORE (optional): Consider failed tasks as a junit test failure even if ignore_on_error is set + Default: False + JUNIT_INCLUDE_SETUP_TASKS_IN_REPORT (optional): Should the setup tasks be included in the final report + Default: True + JUNIT_HIDE_TASK_ARGUMENTS (optional): Hide the arguments for a task + Default: False + JUNIT_TEST_CASE_PREFIX (optional): Consider a task only as test case if it has this value as prefix. Additionally failing tasks are recorded as failed + test cases. + Default: <empty> + """ + + CALLBACK_VERSION = 2.0 + CALLBACK_TYPE = 'aggregate' + CALLBACK_NAME = 'junit' + CALLBACK_NEEDS_ENABLED = True + + def __init__(self): + super(CallbackModule, self).__init__() + + self._output_dir = os.getenv('JUNIT_OUTPUT_DIR', os.path.expanduser('~/.ansible.log')) + self._task_class = os.getenv('JUNIT_TASK_CLASS', 'False').lower() + self._task_relative_path = os.getenv('JUNIT_TASK_RELATIVE_PATH', '') + self._fail_on_change = os.getenv('JUNIT_FAIL_ON_CHANGE', 'False').lower() + self._fail_on_ignore = os.getenv('JUNIT_FAIL_ON_IGNORE', 'False').lower() + self._include_setup_tasks_in_report = os.getenv('JUNIT_INCLUDE_SETUP_TASKS_IN_REPORT', 'True').lower() + self._hide_task_arguments = os.getenv('JUNIT_HIDE_TASK_ARGUMENTS', 'False').lower() + self._test_case_prefix = os.getenv('JUNIT_TEST_CASE_PREFIX', '') + self._replace_out_of_tree_path = os.getenv('JUNIT_REPLACE_OUT_OF_TREE_PATH', None) + self._playbook_path = None + self._playbook_name = None + self._play_name = None + self._task_data = None + + self.disabled = False + + self._task_data = {} + + if self._replace_out_of_tree_path is not None: + self._replace_out_of_tree_path = to_text(self._replace_out_of_tree_path) + + if not os.path.exists(self._output_dir): + os.makedirs(self._output_dir) + + def _start_task(self, task): + """ record the start of a task for one or more hosts """ + + uuid = task._uuid + + if uuid in self._task_data: + return + + play = self._play_name + name = task.get_name().strip() + path = task.get_path() + action = task.action + + if not task.no_log and self._hide_task_arguments == 'false': + args = ', '.join(('%s=%s' % a for a in task.args.items())) + if args: + name += ' ' + args + + self._task_data[uuid] = TaskData(uuid, name, path, play, action) + + def _finish_task(self, status, result): + """ record the results of a task for a single host """ + + task_uuid = result._task._uuid + + if hasattr(result, '_host'): + host_uuid = result._host._uuid + host_name = result._host.name + else: + host_uuid = 'include' + host_name = 'include' + + task_data = self._task_data[task_uuid] + + if self._fail_on_change == 'true' and status == 'ok' and result._result.get('changed', False): + status = 'failed' + + # ignore failure if expected and toggle result if asked for + if status == 'failed' and 'EXPECTED FAILURE' in task_data.name: + status = 'ok' + elif 'TOGGLE RESULT' in task_data.name: + if status == 'failed': + status = 'ok' + elif status == 'ok': + status = 'failed' + + if task_data.name.startswith(self._test_case_prefix) or status == 'failed': + task_data.add_host(HostData(host_uuid, host_name, status, result)) + + def _build_test_case(self, task_data, host_data): + """ build a TestCase from the given TaskData and HostData """ + + name = '[%s] %s: %s' % (host_data.name, task_data.play, task_data.name) + duration = host_data.finish - task_data.start + + if self._task_relative_path and task_data.path: + junit_classname = to_text(os.path.relpath(to_bytes(task_data.path), to_bytes(self._task_relative_path))) + else: + junit_classname = task_data.path + + if self._replace_out_of_tree_path is not None and junit_classname.startswith('../'): + junit_classname = self._replace_out_of_tree_path + to_text(os.path.basename(to_bytes(junit_classname))) + + if self._task_class == 'true': + junit_classname = re.sub(r'\.yml:[0-9]+$', '', junit_classname) + + if host_data.status == 'included': + return TestCase(name=name, classname=junit_classname, time=duration, system_out=str(host_data.result)) + + res = host_data.result._result + rc = res.get('rc', 0) + dump = self._dump_results(res, indent=0) + dump = self._cleanse_string(dump) + + if host_data.status == 'ok': + return TestCase(name=name, classname=junit_classname, time=duration, system_out=dump) + + test_case = TestCase(name=name, classname=junit_classname, time=duration) + + if host_data.status == 'failed': + if 'exception' in res: + message = res['exception'].strip().split('\n')[-1] + output = res['exception'] + test_case.errors.append(TestError(message=message, output=output)) + elif 'msg' in res: + message = res['msg'] + test_case.failures.append(TestFailure(message=message, output=dump)) + else: + test_case.failures.append(TestFailure(message='rc=%s' % rc, output=dump)) + elif host_data.status == 'skipped': + if 'skip_reason' in res: + message = res['skip_reason'] + else: + message = 'skipped' + test_case.skipped = message + + return test_case + + def _cleanse_string(self, value): + """ convert surrogate escapes to the unicode replacement character to avoid XML encoding errors """ + return to_text(to_bytes(value, errors='surrogateescape'), errors='replace') + + def _generate_report(self): + """ generate a TestSuite report from the collected TaskData and HostData """ + + test_cases = [] + + for task_uuid, task_data in self._task_data.items(): + if task_data.action in C._ACTION_SETUP and self._include_setup_tasks_in_report == 'false': + continue + + for host_uuid, host_data in task_data.host_data.items(): + test_cases.append(self._build_test_case(task_data, host_data)) + + test_suite = TestSuite(name=self._playbook_name, cases=test_cases) + test_suites = TestSuites(suites=[test_suite]) + report = test_suites.to_pretty_xml() + + output_file = os.path.join(self._output_dir, '%s-%s.xml' % (self._playbook_name, time.time())) + + with open(output_file, 'wb') as xml: + xml.write(to_bytes(report, errors='surrogate_or_strict')) + + def v2_playbook_on_start(self, playbook): + self._playbook_path = playbook._file_name + self._playbook_name = os.path.splitext(os.path.basename(self._playbook_path))[0] + + def v2_playbook_on_play_start(self, play): + self._play_name = play.get_name() + + def v2_runner_on_no_hosts(self, task): + self._start_task(task) + + def v2_playbook_on_task_start(self, task, is_conditional): + self._start_task(task) + + def v2_playbook_on_cleanup_task_start(self, task): + self._start_task(task) + + def v2_playbook_on_handler_task_start(self, task): + self._start_task(task) + + def v2_runner_on_failed(self, result, ignore_errors=False): + if ignore_errors and self._fail_on_ignore != 'true': + self._finish_task('ok', result) + else: + self._finish_task('failed', result) + + def v2_runner_on_ok(self, result): + self._finish_task('ok', result) + + def v2_runner_on_skipped(self, result): + self._finish_task('skipped', result) + + def v2_playbook_on_include(self, included_file): + self._finish_task('included', included_file) + + def v2_playbook_on_stats(self, stats): + self._generate_report() + + +class TaskData: + """ + Data about an individual task. + """ + + def __init__(self, uuid, name, path, play, action): + self.uuid = uuid + self.name = name + self.path = path + self.play = play + self.start = None + self.host_data = {} + self.start = time.time() + self.action = action + + def add_host(self, host): + if host.uuid in self.host_data: + if host.status == 'included': + # concatenate task include output from multiple items + host.result = '%s\n%s' % (self.host_data[host.uuid].result, host.result) + else: + raise Exception('%s: %s: %s: duplicate host callback: %s' % (self.path, self.play, self.name, host.name)) + + self.host_data[host.uuid] = host + + +class HostData: + """ + Data about an individual host. + """ + + def __init__(self, uuid, name, status, result): + self.uuid = uuid + self.name = name + self.status = status + self.result = result + self.finish = time.time() diff --git a/lib/ansible/plugins/callback/minimal.py b/lib/ansible/plugins/callback/minimal.py new file mode 100644 index 0000000..c4d713f --- /dev/null +++ b/lib/ansible/plugins/callback/minimal.py @@ -0,0 +1,80 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' + name: minimal + type: stdout + short_description: minimal Ansible screen output + version_added: historical + description: + - This is the default output callback used by the ansible command (ad-hoc) + extends_documentation_fragment: + - result_format_callback +''' + +from ansible.plugins.callback import CallbackBase +from ansible import constants as C + + +class CallbackModule(CallbackBase): + + ''' + This is the default callback interface, which simply prints messages + to stdout when new callback events are received. + ''' + + CALLBACK_VERSION = 2.0 + CALLBACK_TYPE = 'stdout' + CALLBACK_NAME = 'minimal' + + def _command_generic_msg(self, host, result, caption): + ''' output the result of a command run ''' + + buf = "%s | %s | rc=%s >>\n" % (host, caption, result.get('rc', -1)) + buf += result.get('stdout', '') + buf += result.get('stderr', '') + buf += result.get('msg', '') + + return buf + "\n" + + def v2_runner_on_failed(self, result, ignore_errors=False): + + self._handle_exception(result._result) + self._handle_warnings(result._result) + + if result._task.action in C.MODULE_NO_JSON and 'module_stderr' not in result._result: + self._display.display(self._command_generic_msg(result._host.get_name(), result._result, "FAILED"), color=C.COLOR_ERROR) + else: + self._display.display("%s | FAILED! => %s" % (result._host.get_name(), self._dump_results(result._result, indent=4)), color=C.COLOR_ERROR) + + def v2_runner_on_ok(self, result): + self._clean_results(result._result, result._task.action) + + self._handle_warnings(result._result) + + if result._result.get('changed', False): + color = C.COLOR_CHANGED + state = 'CHANGED' + else: + color = C.COLOR_OK + state = 'SUCCESS' + + if result._task.action in C.MODULE_NO_JSON and 'ansible_job_id' not in result._result: + self._display.display(self._command_generic_msg(result._host.get_name(), result._result, state), color=color) + else: + self._display.display("%s | %s => %s" % (result._host.get_name(), state, self._dump_results(result._result, indent=4)), color=color) + + def v2_runner_on_skipped(self, result): + self._display.display("%s | SKIPPED" % (result._host.get_name()), color=C.COLOR_SKIP) + + def v2_runner_on_unreachable(self, result): + self._display.display("%s | UNREACHABLE! => %s" % (result._host.get_name(), self._dump_results(result._result, indent=4)), color=C.COLOR_UNREACHABLE) + + def v2_on_file_diff(self, result): + if 'diff' in result._result and result._result['diff']: + self._display.display(self._get_diff(result._result['diff'])) diff --git a/lib/ansible/plugins/callback/oneline.py b/lib/ansible/plugins/callback/oneline.py new file mode 100644 index 0000000..fd51b27 --- /dev/null +++ b/lib/ansible/plugins/callback/oneline.py @@ -0,0 +1,77 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' + name: oneline + type: stdout + short_description: oneline Ansible screen output + version_added: historical + description: + - This is the output callback used by the -o/--one-line command line option. +''' + +from ansible.plugins.callback import CallbackBase +from ansible import constants as C + + +class CallbackModule(CallbackBase): + + ''' + This is the default callback interface, which simply prints messages + to stdout when new callback events are received. + ''' + + CALLBACK_VERSION = 2.0 + CALLBACK_TYPE = 'stdout' + CALLBACK_NAME = 'oneline' + + def _command_generic_msg(self, hostname, result, caption): + stdout = result.get('stdout', '').replace('\n', '\\n').replace('\r', '\\r') + if 'stderr' in result and result['stderr']: + stderr = result.get('stderr', '').replace('\n', '\\n').replace('\r', '\\r') + return "%s | %s | rc=%s | (stdout) %s (stderr) %s" % (hostname, caption, result.get('rc', -1), stdout, stderr) + else: + return "%s | %s | rc=%s | (stdout) %s" % (hostname, caption, result.get('rc', -1), stdout) + + def v2_runner_on_failed(self, result, ignore_errors=False): + if 'exception' in result._result: + if self._display.verbosity < 3: + # extract just the actual error message from the exception text + error = result._result['exception'].strip().split('\n')[-1] + msg = "An exception occurred during task execution. To see the full traceback, use -vvv. The error was: %s" % error + else: + msg = "An exception occurred during task execution. The full traceback is:\n" + result._result['exception'].replace('\n', '') + + if result._task.action in C.MODULE_NO_JSON and 'module_stderr' not in result._result: + self._display.display(self._command_generic_msg(result._host.get_name(), result._result, 'FAILED'), color=C.COLOR_ERROR) + else: + self._display.display(msg, color=C.COLOR_ERROR) + + self._display.display("%s | FAILED! => %s" % (result._host.get_name(), self._dump_results(result._result, indent=0).replace('\n', '')), + color=C.COLOR_ERROR) + + def v2_runner_on_ok(self, result): + + if result._result.get('changed', False): + color = C.COLOR_CHANGED + state = 'CHANGED' + else: + color = C.COLOR_OK + state = 'SUCCESS' + + if result._task.action in C.MODULE_NO_JSON and 'ansible_job_id' not in result._result: + self._display.display(self._command_generic_msg(result._host.get_name(), result._result, state), color=color) + else: + self._display.display("%s | %s => %s" % (result._host.get_name(), state, self._dump_results(result._result, indent=0).replace('\n', '')), + color=color) + + def v2_runner_on_unreachable(self, result): + self._display.display("%s | UNREACHABLE!: %s" % (result._host.get_name(), result._result.get('msg', '')), color=C.COLOR_UNREACHABLE) + + def v2_runner_on_skipped(self, result): + self._display.display("%s | SKIPPED" % (result._host.get_name()), color=C.COLOR_SKIP) diff --git a/lib/ansible/plugins/callback/tree.py b/lib/ansible/plugins/callback/tree.py new file mode 100644 index 0000000..a9f65d2 --- /dev/null +++ b/lib/ansible/plugins/callback/tree.py @@ -0,0 +1,86 @@ +# (c) 2012-2014, Ansible, Inc +# (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' + name: tree + type: notification + requirements: + - invoked in the command line + short_description: Save host events to files + version_added: "2.0" + options: + directory: + version_added: '2.11' + description: directory that will contain the per host JSON files. Also set by the C(--tree) option when using adhoc. + ini: + - section: callback_tree + key: directory + env: + - name: ANSIBLE_CALLBACK_TREE_DIR + default: "~/.ansible/tree" + type: path + description: + - "This callback is used by the Ansible (adhoc) command line option C(-t|--tree)." + - This produces a JSON dump of events in a directory, a file for each host, the directory used MUST be passed as a command line option. +''' + +import os + +from ansible.constants import TREE_DIR +from ansible.module_utils._text import to_bytes, to_text +from ansible.plugins.callback import CallbackBase +from ansible.utils.path import makedirs_safe, unfrackpath + + +class CallbackModule(CallbackBase): + ''' + This callback puts results into a host specific file in a directory in json format. + ''' + + CALLBACK_VERSION = 2.0 + CALLBACK_TYPE = 'aggregate' + CALLBACK_NAME = 'tree' + CALLBACK_NEEDS_ENABLED = True + + def set_options(self, task_keys=None, var_options=None, direct=None): + ''' override to set self.tree ''' + + super(CallbackModule, self).set_options(task_keys=task_keys, var_options=var_options, direct=direct) + + if TREE_DIR: + # TREE_DIR comes from the CLI option --tree, only available for adhoc + self.tree = unfrackpath(TREE_DIR) + else: + self.tree = self.get_option('directory') + + def write_tree_file(self, hostname, buf): + ''' write something into treedir/hostname ''' + + buf = to_bytes(buf) + try: + makedirs_safe(self.tree) + except (OSError, IOError) as e: + self._display.warning(u"Unable to access or create the configured directory (%s): %s" % (to_text(self.tree), to_text(e))) + + try: + path = to_bytes(os.path.join(self.tree, hostname)) + with open(path, 'wb+') as fd: + fd.write(buf) + except (OSError, IOError) as e: + self._display.warning(u"Unable to write to %s's file: %s" % (hostname, to_text(e))) + + def result_to_tree(self, result): + self.write_tree_file(result._host.get_name(), self._dump_results(result._result)) + + def v2_runner_on_ok(self, result): + self.result_to_tree(result) + + def v2_runner_on_failed(self, result, ignore_errors=False): + self.result_to_tree(result) + + def v2_runner_on_unreachable(self, result): + self.result_to_tree(result) diff --git a/lib/ansible/plugins/cliconf/__init__.py b/lib/ansible/plugins/cliconf/__init__.py new file mode 100644 index 0000000..be0f23e --- /dev/null +++ b/lib/ansible/plugins/cliconf/__init__.py @@ -0,0 +1,477 @@ +# +# (c) 2017 Red Hat Inc. +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. +# +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from abc import abstractmethod +from functools import wraps + +from ansible.plugins import AnsiblePlugin +from ansible.errors import AnsibleError, AnsibleConnectionFailure +from ansible.module_utils._text import to_bytes, to_text + +try: + from scp import SCPClient + HAS_SCP = True +except ImportError: + HAS_SCP = False + + +def enable_mode(func): + @wraps(func) + def wrapped(self, *args, **kwargs): + prompt = self._connection.get_prompt() + if not to_text(prompt, errors='surrogate_or_strict').strip().endswith('#'): + raise AnsibleError('operation requires privilege escalation') + return func(self, *args, **kwargs) + return wrapped + + +class CliconfBase(AnsiblePlugin): + """ + A base class for implementing cli connections + + .. note:: String inputs to :meth:`send_command` will be cast to byte strings + within this method and as such are not required to be made byte strings + beforehand. Please avoid using literal byte strings (``b'string'``) in + :class:`CliConfBase` plugins as this can lead to unexpected errors when + running on Python 3 + + List of supported rpc's: + :get_config: Retrieves the specified configuration from the device + :edit_config: Loads the specified commands into the remote device + :get: Execute specified command on remote device + :get_capabilities: Retrieves device information and supported rpc methods + :commit: Load configuration from candidate to running + :discard_changes: Discard changes to candidate datastore + + Note: List of supported rpc's for remote device can be extracted from + output of get_capabilities() + + :returns: Returns output received from remote device as byte string + + Usage: + from ansible.module_utils.connection import Connection + + conn = Connection() + conn.get('show lldp neighbors detail') + conn.get_config('running') + conn.edit_config(['hostname test', 'netconf ssh']) + """ + + __rpc__ = ['get_config', 'edit_config', 'get_capabilities', 'get', 'enable_response_logging', 'disable_response_logging'] + + def __init__(self, connection): + super(CliconfBase, self).__init__() + self._connection = connection + self.history = list() + self.response_logging = False + + def _alarm_handler(self, signum, frame): + """Alarm handler raised in case of command timeout """ + self._connection.queue_message('log', 'closing shell due to command timeout (%s seconds).' % self._connection._play_context.timeout) + self.close() + + def send_command(self, command=None, prompt=None, answer=None, sendonly=False, newline=True, prompt_retry_check=False, check_all=False): + """Executes a command over the device connection + + This method will execute a command over the device connection and + return the results to the caller. This method will also perform + logging of any commands based on the `nolog` argument. + + :param command: The command to send over the connection to the device + :param prompt: A single regex pattern or a sequence of patterns to evaluate the expected prompt from the command + :param answer: The answer to respond with if the prompt is matched. + :param sendonly: Bool value that will send the command but not wait for a result. + :param newline: Bool value that will append the newline character to the command + :param prompt_retry_check: Bool value for trying to detect more prompts + :param check_all: Bool value to indicate if all the values in prompt sequence should be matched or any one of + given prompt. + :returns: The output from the device after executing the command + """ + kwargs = { + 'command': to_bytes(command), + 'sendonly': sendonly, + 'newline': newline, + 'prompt_retry_check': prompt_retry_check, + 'check_all': check_all + } + + if prompt is not None: + if isinstance(prompt, list): + kwargs['prompt'] = [to_bytes(p) for p in prompt] + else: + kwargs['prompt'] = to_bytes(prompt) + if answer is not None: + if isinstance(answer, list): + kwargs['answer'] = [to_bytes(p) for p in answer] + else: + kwargs['answer'] = to_bytes(answer) + + resp = self._connection.send(**kwargs) + + if not self.response_logging: + self.history.append(('*****', '*****')) + else: + self.history.append((kwargs['command'], resp)) + + return resp + + def get_base_rpc(self): + """Returns list of base rpc method supported by remote device""" + return self.__rpc__ + + def get_history(self): + """ Returns the history file for all commands + + This will return a log of all the commands that have been sent to + the device and all of the output received. By default, all commands + and output will be redacted unless explicitly configured otherwise. + + :return: An ordered list of command, output pairs + """ + return self.history + + def reset_history(self): + """ Resets the history of run commands + :return: None + """ + self.history = list() + + def enable_response_logging(self): + """Enable logging command response""" + self.response_logging = True + + def disable_response_logging(self): + """Disable logging command response""" + self.response_logging = False + + @abstractmethod + def get_config(self, source='running', flags=None, format=None): + """Retrieves the specified configuration from the device + + This method will retrieve the configuration specified by source and + return it to the caller as a string. Subsequent calls to this method + will retrieve a new configuration from the device + + :param source: The configuration source to return from the device. + This argument accepts either `running` or `startup` as valid values. + + :param flags: For devices that support configuration filtering, this + keyword argument is used to filter the returned configuration. + The use of this keyword argument is device dependent and will be + silently ignored on devices that do not support it. + + :param format: For devices that support fetching different configuration + format, this keyword argument is used to specify the format in which + configuration is to be retrieved. + + :return: The device configuration as specified by the source argument. + """ + pass + + @abstractmethod + def edit_config(self, candidate=None, commit=True, replace=None, diff=False, comment=None): + """Loads the candidate configuration into the network device + + This method will load the specified candidate config into the device + and merge with the current configuration unless replace is set to + True. If the device does not support config replace an errors + is returned. + + :param candidate: The configuration to load into the device and merge + with the current running configuration + + :param commit: Boolean value that indicates if the device candidate + configuration should be pushed in the running configuration or discarded. + + :param replace: If the value is True/False it indicates if running configuration should be completely + replace by candidate configuration. If can also take configuration file path as value, + the file in this case should be present on the remote host in the mentioned path as a + prerequisite. + :param comment: Commit comment provided it is supported by remote host + :return: Returns a json string with contains configuration applied on remote host, the returned + response on executing configuration commands and platform relevant data. + { + "diff": "", + "response": [], + "request": [] + } + + """ + pass + + @abstractmethod + def get(self, command=None, prompt=None, answer=None, sendonly=False, newline=True, output=None, check_all=False): + """Execute specified command on remote device + This method will retrieve the specified data and + return it to the caller as a string. + :param command: command in string format to be executed on remote device + :param prompt: the expected prompt generated by executing command, this can + be a string or a list of strings + :param answer: the string to respond to the prompt with + :param sendonly: bool to disable waiting for response, default is false + :param newline: bool to indicate if newline should be added at end of answer or not + :param output: For devices that support fetching command output in different + format, this keyword argument is used to specify the output in which + response is to be retrieved. + :param check_all: Bool value to indicate if all the values in prompt sequence should be matched or any one of + given prompt. + :return: The output from the device after executing the command + """ + pass + + @abstractmethod + def get_capabilities(self): + """Returns the basic capabilities of the network device + This method will provide some basic facts about the device and + what capabilities it has to modify the configuration. The minimum + return from this method takes the following format. + eg: + { + + 'rpc': [list of supported rpcs], + 'network_api': <str>, # the name of the transport + 'device_info': { + 'network_os': <str>, + 'network_os_version': <str>, + 'network_os_model': <str>, + 'network_os_hostname': <str>, + 'network_os_image': <str>, + 'network_os_platform': <str>, + }, + 'device_operations': { + 'supports_diff_replace': <bool>, # identify if config should be merged or replaced is supported + 'supports_commit': <bool>, # identify if commit is supported by device or not + 'supports_rollback': <bool>, # identify if rollback is supported or not + 'supports_defaults': <bool>, # identify if fetching running config with default is supported + 'supports_commit_comment': <bool>, # identify if adding comment to commit is supported of not + 'supports_onbox_diff': <bool>, # identify if on box diff capability is supported or not + 'supports_generate_diff': <bool>, # identify if diff capability is supported within plugin + 'supports_multiline_delimiter': <bool>, # identify if multiline demiliter is supported within config + 'supports_diff_match': <bool>, # identify if match is supported + 'supports_diff_ignore_lines': <bool>, # identify if ignore line in diff is supported + 'supports_config_replace': <bool>, # identify if running config replace with candidate config is supported + 'supports_admin': <bool>, # identify if admin configure mode is supported or not + 'supports_commit_label': <bool>, # identify if commit label is supported or not + } + 'format': [list of supported configuration format], + 'diff_match': [list of supported match values], + 'diff_replace': [list of supported replace values], + 'output': [list of supported command output format] + } + :return: capability as json string + """ + result = {} + result['rpc'] = self.get_base_rpc() + result['device_info'] = self.get_device_info() + result['network_api'] = 'cliconf' + return result + + @abstractmethod + def get_device_info(self): + """Returns basic information about the network device. + + This method will provide basic information about the device such as OS version and model + name. This data is expected to be used to fill the 'device_info' key in get_capabilities() + above. + + :return: dictionary of device information + """ + pass + + def commit(self, comment=None): + """Commit configuration changes + + This method will perform the commit operation on a previously loaded + candidate configuration that was loaded using `edit_config()`. If + there is a candidate configuration, it will be committed to the + active configuration. If there is not a candidate configuration, this + method should just silently return. + + :return: None + """ + return self._connection.method_not_found("commit is not supported by network_os %s" % self._play_context.network_os) + + def discard_changes(self): + """Discard candidate configuration + + This method will discard the current candidate configuration if one + is present. If there is no candidate configuration currently loaded, + then this method should just silently return + + :returns: None + """ + return self._connection.method_not_found("discard_changes is not supported by network_os %s" % self._play_context.network_os) + + def rollback(self, rollback_id, commit=True): + """ + + :param rollback_id: The commit id to which configuration should be rollbacked + :param commit: Flag to indicate if changes should be committed or not + :return: Returns diff between before and after change. + """ + pass + + def copy_file(self, source=None, destination=None, proto='scp', timeout=30): + """Copies file over scp/sftp to remote device + + :param source: Source file path + :param destination: Destination file path on remote device + :param proto: Protocol to be used for file transfer, + supported protocol: scp and sftp + :param timeout: Specifies the wait time to receive response from + remote host before triggering timeout exception + :return: None + """ + ssh = self._connection.paramiko_conn._connect_uncached() + if proto == 'scp': + if not HAS_SCP: + raise AnsibleError("Required library scp is not installed. Please install it using `pip install scp`") + with SCPClient(ssh.get_transport(), socket_timeout=timeout) as scp: + out = scp.put(source, destination) + elif proto == 'sftp': + with ssh.open_sftp() as sftp: + sftp.put(source, destination) + + def get_file(self, source=None, destination=None, proto='scp', timeout=30): + """Fetch file over scp/sftp from remote device + :param source: Source file path + :param destination: Destination file path + :param proto: Protocol to be used for file transfer, + supported protocol: scp and sftp + :param timeout: Specifies the wait time to receive response from + remote host before triggering timeout exception + :return: None + """ + """Fetch file over scp/sftp from remote device""" + ssh = self._connection.paramiko_conn._connect_uncached() + if proto == 'scp': + if not HAS_SCP: + raise AnsibleError("Required library scp is not installed. Please install it using `pip install scp`") + try: + with SCPClient(ssh.get_transport(), socket_timeout=timeout) as scp: + scp.get(source, destination) + except EOFError: + # This appears to be benign. + pass + elif proto == 'sftp': + with ssh.open_sftp() as sftp: + sftp.get(source, destination) + + def get_diff(self, candidate=None, running=None, diff_match=None, diff_ignore_lines=None, path=None, diff_replace=None): + """ + Generate diff between candidate and running configuration. If the + remote host supports onbox diff capabilities ie. supports_onbox_diff in that case + candidate and running configurations are not required to be passed as argument. + In case if onbox diff capability is not supported candidate argument is mandatory + and running argument is optional. + :param candidate: The configuration which is expected to be present on remote host. + :param running: The base configuration which is used to generate diff. + :param diff_match: Instructs how to match the candidate configuration with current device configuration + Valid values are 'line', 'strict', 'exact', 'none'. + 'line' - commands are matched line by line + 'strict' - command lines are matched with respect to position + 'exact' - command lines must be an equal match + 'none' - will not compare the candidate configuration with the running configuration + :param diff_ignore_lines: Use this argument to specify one or more lines that should be + ignored during the diff. This is used for lines in the configuration + that are automatically updated by the system. This argument takes + a list of regular expressions or exact line matches. + :param path: The ordered set of parents that uniquely identify the section or hierarchy + the commands should be checked against. If the parents argument + is omitted, the commands are checked against the set of top + level or global commands. + :param diff_replace: Instructs on the way to perform the configuration on the device. + If the replace argument is set to I(line) then the modified lines are + pushed to the device in configuration mode. If the replace argument is + set to I(block) then the entire command block is pushed to the device in + configuration mode if any line is not correct. + :return: Configuration and/or banner diff in json format. + { + 'config_diff': '' + } + + """ + pass + + def run_commands(self, commands=None, check_rc=True): + """ + Execute a list of commands on remote host and return the list of response + :param commands: The list of command that needs to be executed on remote host. + The individual command in list can either be a command string or command dict. + If the command is dict the valid keys are + { + 'command': <command to be executed> + 'prompt': <expected prompt on executing the command>, + 'answer': <answer for the prompt>, + 'output': <the format in which command output should be rendered eg: 'json', 'text'>, + 'sendonly': <Boolean flag to indicate if it command execution response should be ignored or not> + } + :param check_rc: Boolean flag to check if returned response should be checked for error or not. + If check_rc is False the error output is appended in return response list, else if the + value is True an exception is raised. + :return: List of returned response + """ + pass + + def check_edit_config_capability(self, operations, candidate=None, commit=True, replace=None, comment=None): + + if not candidate and not replace: + raise ValueError("must provide a candidate or replace to load configuration") + + if commit not in (True, False): + raise ValueError("'commit' must be a bool, got %s" % commit) + + if replace and not operations['supports_replace']: + raise ValueError("configuration replace is not supported") + + if comment and not operations.get('supports_commit_comment', False): + raise ValueError("commit comment is not supported") + + if replace and not operations.get('supports_replace', False): + raise ValueError("configuration replace is not supported") + + def set_cli_prompt_context(self): + """ + Ensure the command prompt on device is in right mode + :return: None + """ + pass + + def _update_cli_prompt_context(self, config_context=None, exit_command='exit'): + """ + Update the cli prompt context to ensure it is in operational mode + :param config_context: It is string value to identify if the current cli prompt ends with config mode prompt + :param exit_command: Command to execute to exit the config mode + :return: None + """ + out = self._connection.get_prompt() + if out is None: + raise AnsibleConnectionFailure(message=u'cli prompt is not identified from the last received' + u' response window: %s' % self._connection._last_recv_window) + + while True: + out = to_text(out, errors='surrogate_then_replace').strip() + if config_context and out.endswith(config_context): + self._connection.queue_message('vvvv', 'wrong context, sending exit to device') + self.send_command(exit_command) + out = self._connection.get_prompt() + else: + break diff --git a/lib/ansible/plugins/connection/__init__.py b/lib/ansible/plugins/connection/__init__.py new file mode 100644 index 0000000..daa683c --- /dev/null +++ b/lib/ansible/plugins/connection/__init__.py @@ -0,0 +1,382 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# (c) 2015 Toshio Kuratomi <tkuratomi@ansible.com> +# (c) 2017, Peter Sprygada <psprygad@redhat.com> +# (c) 2017 Ansible Project +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import fcntl +import os +import shlex +import typing as t + +from abc import abstractmethod +from functools import wraps + +from ansible import constants as C +from ansible.module_utils._text import to_bytes, to_text +from ansible.plugins import AnsiblePlugin +from ansible.utils.display import Display +from ansible.plugins.loader import connection_loader, get_shell_plugin +from ansible.utils.path import unfrackpath + +display = Display() + + +__all__ = ['ConnectionBase', 'ensure_connect'] + +BUFSIZE = 65536 + + +def ensure_connect(func): + @wraps(func) + def wrapped(self, *args, **kwargs): + if not self._connected: + self._connect() + return func(self, *args, **kwargs) + return wrapped + + +class ConnectionBase(AnsiblePlugin): + ''' + A base class for connections to contain common code. + ''' + + has_pipelining = False + has_native_async = False # eg, winrm + always_pipeline_modules = False # eg, winrm + has_tty = True # for interacting with become plugins + # When running over this connection type, prefer modules written in a certain language + # as discovered by the specified file extension. An empty string as the + # language means any language. + module_implementation_preferences = ('',) # type: t.Iterable[str] + allow_executable = True + + # the following control whether or not the connection supports the + # persistent connection framework or not + supports_persistence = False + force_persistence = False + + default_user = None + + def __init__(self, play_context, new_stdin, shell=None, *args, **kwargs): + + super(ConnectionBase, self).__init__() + + # All these hasattrs allow subclasses to override these parameters + if not hasattr(self, '_play_context'): + # Backwards compat: self._play_context isn't really needed, using set_options/get_option + self._play_context = play_context + if not hasattr(self, '_new_stdin'): + self._new_stdin = new_stdin + if not hasattr(self, '_display'): + # Backwards compat: self._display isn't really needed, just import the global display and use that. + self._display = display + if not hasattr(self, '_connected'): + self._connected = False + + self.success_key = None + self.prompt = None + self._connected = False + self._socket_path = None + + # helper plugins + self._shell = shell + + # we always must have shell + if not self._shell: + shell_type = play_context.shell if play_context.shell else getattr(self, '_shell_type', None) + self._shell = get_shell_plugin(shell_type=shell_type, executable=self._play_context.executable) + + self.become = None + + def set_become_plugin(self, plugin): + self.become = plugin + + @property + def connected(self): + '''Read-only property holding whether the connection to the remote host is active or closed.''' + return self._connected + + @property + def socket_path(self): + '''Read-only property holding the connection socket path for this remote host''' + return self._socket_path + + @staticmethod + def _split_ssh_args(argstring): + """ + Takes a string like '-o Foo=1 -o Bar="foo bar"' and returns a + list ['-o', 'Foo=1', '-o', 'Bar=foo bar'] that can be added to + the argument list. The list will not contain any empty elements. + """ + # In Python3, shlex.split doesn't work on a byte string. + return [to_text(x.strip()) for x in shlex.split(argstring) if x.strip()] + + @property + @abstractmethod + def transport(self): + """String used to identify this Connection class from other classes""" + pass + + @abstractmethod + def _connect(self): + """Connect to the host we've been initialized with""" + + @ensure_connect + @abstractmethod + def exec_command(self, cmd, in_data=None, sudoable=True): + """Run a command on the remote host. + + :arg cmd: byte string containing the command + :kwarg in_data: If set, this data is passed to the command's stdin. + This is used to implement pipelining. Currently not all + connection plugins implement pipelining. + :kwarg sudoable: Tell the connection plugin if we're executing + a command via a privilege escalation mechanism. This may affect + how the connection plugin returns data. Note that not all + connections can handle privilege escalation. + :returns: a tuple of (return code, stdout, stderr) The return code is + an int while stdout and stderr are both byte strings. + + When a command is executed, it goes through multiple commands to get + there. It looks approximately like this:: + + [LocalShell] ConnectionCommand [UsersLoginShell (*)] ANSIBLE_SHELL_EXECUTABLE [(BecomeCommand ANSIBLE_SHELL_EXECUTABLE)] Command + :LocalShell: Is optional. It is run locally to invoke the + ``Connection Command``. In most instances, the + ``ConnectionCommand`` can be invoked directly instead. The ssh + connection plugin which can have values that need expanding + locally specified via ssh_args is the sole known exception to + this. Shell metacharacters in the command itself should be + processed on the remote machine, not on the local machine so no + shell is needed on the local machine. (Example, ``/bin/sh``) + :ConnectionCommand: This is the command that connects us to the remote + machine to run the rest of the command. ``ansible_user``, + ``ansible_ssh_host`` and so forth are fed to this piece of the + command to connect to the correct host (Examples ``ssh``, + ``chroot``) + :UsersLoginShell: This shell may or may not be created depending on + the ConnectionCommand used by the connection plugin. This is the + shell that the ``ansible_user`` has configured as their login + shell. In traditional UNIX parlance, this is the last field of + a user's ``/etc/passwd`` entry We do not specifically try to run + the ``UsersLoginShell`` when we connect. Instead it is implicit + in the actions that the ``ConnectionCommand`` takes when it + connects to a remote machine. ``ansible_shell_type`` may be set + to inform ansible of differences in how the ``UsersLoginShell`` + handles things like quoting if a shell has different semantics + than the Bourne shell. + :ANSIBLE_SHELL_EXECUTABLE: This is the shell set via the inventory var + ``ansible_shell_executable`` or via + ``constants.DEFAULT_EXECUTABLE`` if the inventory var is not set. + We explicitly invoke this shell so that we have predictable + quoting rules at this point. ``ANSIBLE_SHELL_EXECUTABLE`` is only + settable by the user because some sudo setups may only allow + invoking a specific shell. (For instance, ``/bin/bash`` may be + allowed but ``/bin/sh``, our default, may not). We invoke this + twice, once after the ``ConnectionCommand`` and once after the + ``BecomeCommand``. After the ConnectionCommand, this is run by + the ``UsersLoginShell``. After the ``BecomeCommand`` we specify + that the ``ANSIBLE_SHELL_EXECUTABLE`` is being invoked directly. + :BecomeComand ANSIBLE_SHELL_EXECUTABLE: Is the command that performs + privilege escalation. Setting this up is performed by the action + plugin prior to running ``exec_command``. So we just get passed + :param:`cmd` which has the BecomeCommand already added. + (Examples: sudo, su) If we have a BecomeCommand then we will + invoke a ANSIBLE_SHELL_EXECUTABLE shell inside of it so that we + have a consistent view of quoting. + :Command: Is the command we're actually trying to run remotely. + (Examples: mkdir -p $HOME/.ansible, python $HOME/.ansible/tmp-script-file) + """ + pass + + @ensure_connect + @abstractmethod + def put_file(self, in_path, out_path): + """Transfer a file from local to remote""" + pass + + @ensure_connect + @abstractmethod + def fetch_file(self, in_path, out_path): + """Fetch a file from remote to local; callers are expected to have pre-created the directory chain for out_path""" + pass + + @abstractmethod + def close(self): + """Terminate the connection""" + pass + + def connection_lock(self): + f = self._play_context.connection_lockfd + display.vvvv('CONNECTION: pid %d waiting for lock on %d' % (os.getpid(), f), host=self._play_context.remote_addr) + fcntl.lockf(f, fcntl.LOCK_EX) + display.vvvv('CONNECTION: pid %d acquired lock on %d' % (os.getpid(), f), host=self._play_context.remote_addr) + + def connection_unlock(self): + f = self._play_context.connection_lockfd + fcntl.lockf(f, fcntl.LOCK_UN) + display.vvvv('CONNECTION: pid %d released lock on %d' % (os.getpid(), f), host=self._play_context.remote_addr) + + def reset(self): + display.warning("Reset is not implemented for this connection") + + def update_vars(self, variables): + ''' + Adds 'magic' variables relating to connections to the variable dictionary provided. + In case users need to access from the play, this is a legacy from runner. + ''' + for varname in C.COMMON_CONNECTION_VARS: + value = None + if varname in variables: + # dont update existing + continue + elif 'password' in varname or 'passwd' in varname: + # no secrets! + continue + elif varname == 'ansible_connection': + # its me mom! + value = self._load_name + elif varname == 'ansible_shell_type': + # its my cousin ... + value = self._shell._load_name + else: + # deal with generic options if the plugin supports em (for exmaple not all connections have a remote user) + options = C.config.get_plugin_options_from_var('connection', self._load_name, varname) + if options: + value = self.get_option(options[0]) # for these variables there should be only one option + elif 'become' not in varname: + # fallback to play_context, unles becoem related TODO: in the end should come from task/play and not pc + for prop, var_list in C.MAGIC_VARIABLE_MAPPING.items(): + if varname in var_list: + try: + value = getattr(self._play_context, prop) + break + except AttributeError: + # it was not defined, fine to ignore + continue + + if value is not None: + display.debug('Set connection var {0} to {1}'.format(varname, value)) + variables[varname] = value + + +class NetworkConnectionBase(ConnectionBase): + """ + A base class for network-style connections. + """ + + force_persistence = True + # Do not use _remote_is_local in other connections + _remote_is_local = True + + def __init__(self, play_context, new_stdin, *args, **kwargs): + super(NetworkConnectionBase, self).__init__(play_context, new_stdin, *args, **kwargs) + self._messages = [] + self._conn_closed = False + + self._network_os = self._play_context.network_os + + self._local = connection_loader.get('local', play_context, '/dev/null') + self._local.set_options() + + self._sub_plugin = {} + self._cached_variables = (None, None, None) + + # reconstruct the socket_path and set instance values accordingly + self._ansible_playbook_pid = kwargs.get('ansible_playbook_pid') + self._update_connection_state() + + def __getattr__(self, name): + try: + return self.__dict__[name] + except KeyError: + if not name.startswith('_'): + plugin = self._sub_plugin.get('obj') + if plugin: + method = getattr(plugin, name, None) + if method is not None: + return method + raise AttributeError("'%s' object has no attribute '%s'" % (self.__class__.__name__, name)) + + def exec_command(self, cmd, in_data=None, sudoable=True): + return self._local.exec_command(cmd, in_data, sudoable) + + def queue_message(self, level, message): + """ + Adds a message to the queue of messages waiting to be pushed back to the controller process. + + :arg level: A string which can either be the name of a method in display, or 'log'. When + the messages are returned to task_executor, a value of log will correspond to + ``display.display(message, log_only=True)``, while another value will call ``display.[level](message)`` + """ + self._messages.append((level, message)) + + def pop_messages(self): + messages, self._messages = self._messages, [] + return messages + + def put_file(self, in_path, out_path): + """Transfer a file from local to remote""" + return self._local.put_file(in_path, out_path) + + def fetch_file(self, in_path, out_path): + """Fetch a file from remote to local""" + return self._local.fetch_file(in_path, out_path) + + def reset(self): + ''' + Reset the connection + ''' + if self._socket_path: + self.queue_message('vvvv', 'resetting persistent connection for socket_path %s' % self._socket_path) + self.close() + self.queue_message('vvvv', 'reset call on connection instance') + + def close(self): + self._conn_closed = True + if self._connected: + self._connected = False + + def set_options(self, task_keys=None, var_options=None, direct=None): + super(NetworkConnectionBase, self).set_options(task_keys=task_keys, var_options=var_options, direct=direct) + if self.get_option('persistent_log_messages'): + warning = "Persistent connection logging is enabled for %s. This will log ALL interactions" % self._play_context.remote_addr + logpath = getattr(C, 'DEFAULT_LOG_PATH') + if logpath is not None: + warning += " to %s" % logpath + self.queue_message('warning', "%s and WILL NOT redact sensitive configuration like passwords. USE WITH CAUTION!" % warning) + + if self._sub_plugin.get('obj') and self._sub_plugin.get('type') != 'external': + try: + self._sub_plugin['obj'].set_options(task_keys=task_keys, var_options=var_options, direct=direct) + except AttributeError: + pass + + def _update_connection_state(self): + ''' + Reconstruct the connection socket_path and check if it exists + + If the socket path exists then the connection is active and set + both the _socket_path value to the path and the _connected value + to True. If the socket path doesn't exist, leave the socket path + value to None and the _connected value to False + ''' + ssh = connection_loader.get('ssh', class_only=True) + control_path = ssh._create_control_path( + self._play_context.remote_addr, self._play_context.port, + self._play_context.remote_user, self._play_context.connection, + self._ansible_playbook_pid + ) + + tmp_path = unfrackpath(C.PERSISTENT_CONTROL_PATH_DIR) + socket_path = unfrackpath(control_path % dict(directory=tmp_path)) + + if os.path.exists(socket_path): + self._connected = True + self._socket_path = socket_path + + def _log_messages(self, message): + if self.get_option('persistent_log_messages'): + self.queue_message('log', message) diff --git a/lib/ansible/plugins/connection/local.py b/lib/ansible/plugins/connection/local.py new file mode 100644 index 0000000..27afd10 --- /dev/null +++ b/lib/ansible/plugins/connection/local.py @@ -0,0 +1,194 @@ +# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com> +# (c) 2015, 2017 Toshio Kuratomi <tkuratomi@ansible.com> +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' + name: local + short_description: execute on controller + description: + - This connection plugin allows ansible to execute tasks on the Ansible 'controller' instead of on a remote host. + author: ansible (@core) + version_added: historical + extends_documentation_fragment: + - connection_pipelining + notes: + - The remote user is ignored, the user with which the ansible CLI was executed is used instead. +''' + +import fcntl +import getpass +import os +import pty +import shutil +import subprocess + +import ansible.constants as C +from ansible.errors import AnsibleError, AnsibleFileNotFound +from ansible.module_utils.compat import selectors +from ansible.module_utils.six import text_type, binary_type +from ansible.module_utils._text import to_bytes, to_native, to_text +from ansible.plugins.connection import ConnectionBase +from ansible.utils.display import Display +from ansible.utils.path import unfrackpath + +display = Display() + + +class Connection(ConnectionBase): + ''' Local based connections ''' + + transport = 'local' + has_pipelining = True + + def __init__(self, *args, **kwargs): + + super(Connection, self).__init__(*args, **kwargs) + self.cwd = None + try: + self.default_user = getpass.getuser() + except KeyError: + display.vv("Current user (uid=%s) does not seem to exist on this system, leaving user empty." % os.getuid()) + self.default_user = "" + + def _connect(self): + ''' connect to the local host; nothing to do here ''' + + # Because we haven't made any remote connection we're running as + # the local user, rather than as whatever is configured in remote_user. + self._play_context.remote_user = self.default_user + + if not self._connected: + display.vvv(u"ESTABLISH LOCAL CONNECTION FOR USER: {0}".format(self._play_context.remote_user), host=self._play_context.remote_addr) + self._connected = True + return self + + def exec_command(self, cmd, in_data=None, sudoable=True): + ''' run a command on the local host ''' + + super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable) + + display.debug("in local.exec_command()") + + executable = C.DEFAULT_EXECUTABLE.split()[0] if C.DEFAULT_EXECUTABLE else None + + if not os.path.exists(to_bytes(executable, errors='surrogate_or_strict')): + raise AnsibleError("failed to find the executable specified %s." + " Please verify if the executable exists and re-try." % executable) + + display.vvv(u"EXEC {0}".format(to_text(cmd)), host=self._play_context.remote_addr) + display.debug("opening command with Popen()") + + if isinstance(cmd, (text_type, binary_type)): + cmd = to_bytes(cmd) + else: + cmd = map(to_bytes, cmd) + + master = None + stdin = subprocess.PIPE + if sudoable and self.become and self.become.expect_prompt() and not self.get_option('pipelining'): + # Create a pty if sudoable for privlege escalation that needs it. + # Falls back to using a standard pipe if this fails, which may + # cause the command to fail in certain situations where we are escalating + # privileges or the command otherwise needs a pty. + try: + master, stdin = pty.openpty() + except (IOError, OSError) as e: + display.debug("Unable to open pty: %s" % to_native(e)) + + p = subprocess.Popen( + cmd, + shell=isinstance(cmd, (text_type, binary_type)), + executable=executable, + cwd=self.cwd, + stdin=stdin, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + # if we created a master, we can close the other half of the pty now, otherwise master is stdin + if master is not None: + os.close(stdin) + + display.debug("done running command with Popen()") + + if self.become and self.become.expect_prompt() and sudoable: + fcntl.fcntl(p.stdout, fcntl.F_SETFL, fcntl.fcntl(p.stdout, fcntl.F_GETFL) | os.O_NONBLOCK) + fcntl.fcntl(p.stderr, fcntl.F_SETFL, fcntl.fcntl(p.stderr, fcntl.F_GETFL) | os.O_NONBLOCK) + selector = selectors.DefaultSelector() + selector.register(p.stdout, selectors.EVENT_READ) + selector.register(p.stderr, selectors.EVENT_READ) + + become_output = b'' + try: + while not self.become.check_success(become_output) and not self.become.check_password_prompt(become_output): + events = selector.select(self._play_context.timeout) + if not events: + stdout, stderr = p.communicate() + raise AnsibleError('timeout waiting for privilege escalation password prompt:\n' + to_native(become_output)) + + for key, event in events: + if key.fileobj == p.stdout: + chunk = p.stdout.read() + elif key.fileobj == p.stderr: + chunk = p.stderr.read() + + if not chunk: + stdout, stderr = p.communicate() + raise AnsibleError('privilege output closed while waiting for password prompt:\n' + to_native(become_output)) + become_output += chunk + finally: + selector.close() + + if not self.become.check_success(become_output): + become_pass = self.become.get_option('become_pass', playcontext=self._play_context) + if master is None: + p.stdin.write(to_bytes(become_pass, errors='surrogate_or_strict') + b'\n') + else: + os.write(master, to_bytes(become_pass, errors='surrogate_or_strict') + b'\n') + + fcntl.fcntl(p.stdout, fcntl.F_SETFL, fcntl.fcntl(p.stdout, fcntl.F_GETFL) & ~os.O_NONBLOCK) + fcntl.fcntl(p.stderr, fcntl.F_SETFL, fcntl.fcntl(p.stderr, fcntl.F_GETFL) & ~os.O_NONBLOCK) + + display.debug("getting output with communicate()") + stdout, stderr = p.communicate(in_data) + display.debug("done communicating") + + # finally, close the other half of the pty, if it was created + if master: + os.close(master) + + display.debug("done with local.exec_command()") + return (p.returncode, stdout, stderr) + + def put_file(self, in_path, out_path): + ''' transfer a file from local to local ''' + + super(Connection, self).put_file(in_path, out_path) + + in_path = unfrackpath(in_path, basedir=self.cwd) + out_path = unfrackpath(out_path, basedir=self.cwd) + + display.vvv(u"PUT {0} TO {1}".format(in_path, out_path), host=self._play_context.remote_addr) + if not os.path.exists(to_bytes(in_path, errors='surrogate_or_strict')): + raise AnsibleFileNotFound("file or module does not exist: {0}".format(to_native(in_path))) + try: + shutil.copyfile(to_bytes(in_path, errors='surrogate_or_strict'), to_bytes(out_path, errors='surrogate_or_strict')) + except shutil.Error: + raise AnsibleError("failed to copy: {0} and {1} are the same".format(to_native(in_path), to_native(out_path))) + except IOError as e: + raise AnsibleError("failed to transfer file to {0}: {1}".format(to_native(out_path), to_native(e))) + + def fetch_file(self, in_path, out_path): + ''' fetch a file from local to local -- for compatibility ''' + + super(Connection, self).fetch_file(in_path, out_path) + + display.vvv(u"FETCH {0} TO {1}".format(in_path, out_path), host=self._play_context.remote_addr) + self.put_file(in_path, out_path) + + def close(self): + ''' terminate the connection; nothing to do here ''' + self._connected = False diff --git a/lib/ansible/plugins/connection/paramiko_ssh.py b/lib/ansible/plugins/connection/paramiko_ssh.py new file mode 100644 index 0000000..b9fd898 --- /dev/null +++ b/lib/ansible/plugins/connection/paramiko_ssh.py @@ -0,0 +1,695 @@ +# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com> +# (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = """ + author: Ansible Core Team + name: paramiko + short_description: Run tasks via python ssh (paramiko) + description: + - Use the python ssh implementation (Paramiko) to connect to targets + - The paramiko transport is provided because many distributions, in particular EL6 and before do not support ControlPersist + in their SSH implementations. + - This is needed on the Ansible control machine to be reasonably efficient with connections. + Thus paramiko is faster for most users on these platforms. + Users with ControlPersist capability can consider using -c ssh or configuring the transport in the configuration file. + - This plugin also borrows a lot of settings from the ssh plugin as they both cover the same protocol. + version_added: "0.1" + options: + remote_addr: + description: + - Address of the remote target + default: inventory_hostname + vars: + - name: inventory_hostname + - name: ansible_host + - name: ansible_ssh_host + - name: ansible_paramiko_host + remote_user: + description: + - User to login/authenticate as + - Can be set from the CLI via the C(--user) or C(-u) options. + vars: + - name: ansible_user + - name: ansible_ssh_user + - name: ansible_paramiko_user + env: + - name: ANSIBLE_REMOTE_USER + - name: ANSIBLE_PARAMIKO_REMOTE_USER + version_added: '2.5' + ini: + - section: defaults + key: remote_user + - section: paramiko_connection + key: remote_user + version_added: '2.5' + keyword: + - name: remote_user + password: + description: + - Secret used to either login the ssh server or as a passphrase for ssh keys that require it + - Can be set from the CLI via the C(--ask-pass) option. + vars: + - name: ansible_password + - name: ansible_ssh_pass + - name: ansible_ssh_password + - name: ansible_paramiko_pass + - name: ansible_paramiko_password + version_added: '2.5' + use_rsa_sha2_algorithms: + description: + - Whether or not to enable RSA SHA2 algorithms for pubkeys and hostkeys + - On paramiko versions older than 2.9, this only affects hostkeys + - For behavior matching paramiko<2.9 set this to C(False) + vars: + - name: ansible_paramiko_use_rsa_sha2_algorithms + ini: + - {key: use_rsa_sha2_algorithms, section: paramiko_connection} + env: + - {name: ANSIBLE_PARAMIKO_USE_RSA_SHA2_ALGORITHMS} + default: True + type: boolean + version_added: '2.14' + host_key_auto_add: + description: 'Automatically add host keys' + env: [{name: ANSIBLE_PARAMIKO_HOST_KEY_AUTO_ADD}] + ini: + - {key: host_key_auto_add, section: paramiko_connection} + type: boolean + look_for_keys: + default: True + description: 'False to disable searching for private key files in ~/.ssh/' + env: [{name: ANSIBLE_PARAMIKO_LOOK_FOR_KEYS}] + ini: + - {key: look_for_keys, section: paramiko_connection} + type: boolean + proxy_command: + default: '' + description: + - Proxy information for running the connection via a jumphost + - Also this plugin will scan 'ssh_args', 'ssh_extra_args' and 'ssh_common_args' from the 'ssh' plugin settings for proxy information if set. + env: [{name: ANSIBLE_PARAMIKO_PROXY_COMMAND}] + ini: + - {key: proxy_command, section: paramiko_connection} + ssh_args: + description: Only used in parsing ProxyCommand for use in this plugin. + default: '' + ini: + - section: 'ssh_connection' + key: 'ssh_args' + env: + - name: ANSIBLE_SSH_ARGS + vars: + - name: ansible_ssh_args + version_added: '2.7' + ssh_common_args: + description: Only used in parsing ProxyCommand for use in this plugin. + ini: + - section: 'ssh_connection' + key: 'ssh_common_args' + version_added: '2.7' + env: + - name: ANSIBLE_SSH_COMMON_ARGS + version_added: '2.7' + vars: + - name: ansible_ssh_common_args + cli: + - name: ssh_common_args + default: '' + ssh_extra_args: + description: Only used in parsing ProxyCommand for use in this plugin. + vars: + - name: ansible_ssh_extra_args + env: + - name: ANSIBLE_SSH_EXTRA_ARGS + version_added: '2.7' + ini: + - key: ssh_extra_args + section: ssh_connection + version_added: '2.7' + cli: + - name: ssh_extra_args + default: '' + pty: + default: True + description: 'SUDO usually requires a PTY, True to give a PTY and False to not give a PTY.' + env: + - name: ANSIBLE_PARAMIKO_PTY + ini: + - section: paramiko_connection + key: pty + type: boolean + record_host_keys: + default: True + description: 'Save the host keys to a file' + env: [{name: ANSIBLE_PARAMIKO_RECORD_HOST_KEYS}] + ini: + - section: paramiko_connection + key: record_host_keys + type: boolean + host_key_checking: + description: 'Set this to "False" if you want to avoid host key checking by the underlying tools Ansible uses to connect to the host' + type: boolean + default: True + env: + - name: ANSIBLE_HOST_KEY_CHECKING + - name: ANSIBLE_SSH_HOST_KEY_CHECKING + version_added: '2.5' + - name: ANSIBLE_PARAMIKO_HOST_KEY_CHECKING + version_added: '2.5' + ini: + - section: defaults + key: host_key_checking + - section: paramiko_connection + key: host_key_checking + version_added: '2.5' + vars: + - name: ansible_host_key_checking + version_added: '2.5' + - name: ansible_ssh_host_key_checking + version_added: '2.5' + - name: ansible_paramiko_host_key_checking + version_added: '2.5' + use_persistent_connections: + description: 'Toggles the use of persistence for connections' + type: boolean + default: False + env: + - name: ANSIBLE_USE_PERSISTENT_CONNECTIONS + ini: + - section: defaults + key: use_persistent_connections + banner_timeout: + type: float + default: 30 + version_added: '2.14' + description: + - Configures, in seconds, the amount of time to wait for the SSH + banner to be presented. This option is supported by paramiko + version 1.15.0 or newer. + ini: + - section: paramiko_connection + key: banner_timeout + env: + - name: ANSIBLE_PARAMIKO_BANNER_TIMEOUT +# TODO: +#timeout=self._play_context.timeout, +""" + +import os +import socket +import tempfile +import traceback +import fcntl +import sys +import re + +from termios import tcflush, TCIFLUSH +from ansible.module_utils.compat.version import LooseVersion +from binascii import hexlify + +from ansible.errors import ( + AnsibleAuthenticationFailure, + AnsibleConnectionFailure, + AnsibleError, + AnsibleFileNotFound, +) +from ansible.module_utils.compat.paramiko import PARAMIKO_IMPORT_ERR, paramiko +from ansible.plugins.connection import ConnectionBase +from ansible.utils.display import Display +from ansible.utils.path import makedirs_safe +from ansible.module_utils._text import to_bytes, to_native, to_text + +display = Display() + + +AUTHENTICITY_MSG = """ +paramiko: The authenticity of host '%s' can't be established. +The %s key fingerprint is %s. +Are you sure you want to continue connecting (yes/no)? +""" + +# SSH Options Regex +SETTINGS_REGEX = re.compile(r'(\w+)(?:\s*=\s*|\s+)(.+)') + + +class MyAddPolicy(object): + """ + Based on AutoAddPolicy in paramiko so we can determine when keys are added + + and also prompt for input. + + Policy for automatically adding the hostname and new host key to the + local L{HostKeys} object, and saving it. This is used by L{SSHClient}. + """ + + def __init__(self, new_stdin, connection): + self._new_stdin = new_stdin + self.connection = connection + self._options = connection._options + + def missing_host_key(self, client, hostname, key): + + if all((self._options['host_key_checking'], not self._options['host_key_auto_add'])): + + fingerprint = hexlify(key.get_fingerprint()) + ktype = key.get_name() + + if self.connection.get_option('use_persistent_connections') or self.connection.force_persistence: + # don't print the prompt string since the user cannot respond + # to the question anyway + raise AnsibleError(AUTHENTICITY_MSG[1:92] % (hostname, ktype, fingerprint)) + + self.connection.connection_lock() + + old_stdin = sys.stdin + sys.stdin = self._new_stdin + + # clear out any premature input on sys.stdin + tcflush(sys.stdin, TCIFLUSH) + + inp = input(AUTHENTICITY_MSG % (hostname, ktype, fingerprint)) + sys.stdin = old_stdin + + self.connection.connection_unlock() + + if inp not in ['yes', 'y', '']: + raise AnsibleError("host connection rejected by user") + + key._added_by_ansible_this_time = True + + # existing implementation below: + client._host_keys.add(hostname, key.get_name(), key) + + # host keys are actually saved in close() function below + # in order to control ordering. + + +# keep connection objects on a per host basis to avoid repeated attempts to reconnect + +SSH_CONNECTION_CACHE = {} # type: dict[str, paramiko.client.SSHClient] +SFTP_CONNECTION_CACHE = {} # type: dict[str, paramiko.sftp_client.SFTPClient] + + +class Connection(ConnectionBase): + ''' SSH based connections with Paramiko ''' + + transport = 'paramiko' + _log_channel = None + + def _cache_key(self): + return "%s__%s__" % (self._play_context.remote_addr, self._play_context.remote_user) + + def _connect(self): + cache_key = self._cache_key() + if cache_key in SSH_CONNECTION_CACHE: + self.ssh = SSH_CONNECTION_CACHE[cache_key] + else: + self.ssh = SSH_CONNECTION_CACHE[cache_key] = self._connect_uncached() + + self._connected = True + return self + + def _set_log_channel(self, name): + '''Mimic paramiko.SSHClient.set_log_channel''' + self._log_channel = name + + def _parse_proxy_command(self, port=22): + proxy_command = None + # Parse ansible_ssh_common_args, specifically looking for ProxyCommand + ssh_args = [ + self.get_option('ssh_extra_args'), + self.get_option('ssh_common_args'), + self.get_option('ssh_args', ''), + ] + + args = self._split_ssh_args(' '.join(ssh_args)) + for i, arg in enumerate(args): + if arg.lower() == 'proxycommand': + # _split_ssh_args split ProxyCommand from the command itself + proxy_command = args[i + 1] + else: + # ProxyCommand and the command itself are a single string + match = SETTINGS_REGEX.match(arg) + if match: + if match.group(1).lower() == 'proxycommand': + proxy_command = match.group(2) + + if proxy_command: + break + + proxy_command = self.get_option('proxy_command') or proxy_command + + sock_kwarg = {} + if proxy_command: + replacers = { + '%h': self._play_context.remote_addr, + '%p': port, + '%r': self._play_context.remote_user + } + for find, replace in replacers.items(): + proxy_command = proxy_command.replace(find, str(replace)) + try: + sock_kwarg = {'sock': paramiko.ProxyCommand(proxy_command)} + display.vvv("CONFIGURE PROXY COMMAND FOR CONNECTION: %s" % proxy_command, host=self._play_context.remote_addr) + except AttributeError: + display.warning('Paramiko ProxyCommand support unavailable. ' + 'Please upgrade to Paramiko 1.9.0 or newer. ' + 'Not using configured ProxyCommand') + + return sock_kwarg + + def _connect_uncached(self): + ''' activates the connection object ''' + + if paramiko is None: + raise AnsibleError("paramiko is not installed: %s" % to_native(PARAMIKO_IMPORT_ERR)) + + port = self._play_context.port or 22 + display.vvv("ESTABLISH PARAMIKO SSH CONNECTION FOR USER: %s on PORT %s TO %s" % (self._play_context.remote_user, port, self._play_context.remote_addr), + host=self._play_context.remote_addr) + + ssh = paramiko.SSHClient() + + # Set pubkey and hostkey algorithms to disable, the only manipulation allowed currently + # is keeping or omitting rsa-sha2 algorithms + paramiko_preferred_pubkeys = getattr(paramiko.Transport, '_preferred_pubkeys', ()) + paramiko_preferred_hostkeys = getattr(paramiko.Transport, '_preferred_keys', ()) + use_rsa_sha2_algorithms = self.get_option('use_rsa_sha2_algorithms') + disabled_algorithms = {} + if not use_rsa_sha2_algorithms: + if paramiko_preferred_pubkeys: + disabled_algorithms['pubkeys'] = tuple(a for a in paramiko_preferred_pubkeys if 'rsa-sha2' in a) + if paramiko_preferred_hostkeys: + disabled_algorithms['keys'] = tuple(a for a in paramiko_preferred_hostkeys if 'rsa-sha2' in a) + + # override paramiko's default logger name + if self._log_channel is not None: + ssh.set_log_channel(self._log_channel) + + self.keyfile = os.path.expanduser("~/.ssh/known_hosts") + + if self.get_option('host_key_checking'): + for ssh_known_hosts in ("/etc/ssh/ssh_known_hosts", "/etc/openssh/ssh_known_hosts"): + try: + # TODO: check if we need to look at several possible locations, possible for loop + ssh.load_system_host_keys(ssh_known_hosts) + break + except IOError: + pass # file was not found, but not required to function + ssh.load_system_host_keys() + + ssh_connect_kwargs = self._parse_proxy_command(port) + + ssh.set_missing_host_key_policy(MyAddPolicy(self._new_stdin, self)) + + conn_password = self.get_option('password') or self._play_context.password + + allow_agent = True + + if conn_password is not None: + allow_agent = False + + try: + key_filename = None + if self._play_context.private_key_file: + key_filename = os.path.expanduser(self._play_context.private_key_file) + + # paramiko 2.2 introduced auth_timeout parameter + if LooseVersion(paramiko.__version__) >= LooseVersion('2.2.0'): + ssh_connect_kwargs['auth_timeout'] = self._play_context.timeout + + # paramiko 1.15 introduced banner timeout parameter + if LooseVersion(paramiko.__version__) >= LooseVersion('1.15.0'): + ssh_connect_kwargs['banner_timeout'] = self.get_option('banner_timeout') + + ssh.connect( + self._play_context.remote_addr.lower(), + username=self._play_context.remote_user, + allow_agent=allow_agent, + look_for_keys=self.get_option('look_for_keys'), + key_filename=key_filename, + password=conn_password, + timeout=self._play_context.timeout, + port=port, + disabled_algorithms=disabled_algorithms, + **ssh_connect_kwargs, + ) + except paramiko.ssh_exception.BadHostKeyException as e: + raise AnsibleConnectionFailure('host key mismatch for %s' % e.hostname) + except paramiko.ssh_exception.AuthenticationException as e: + msg = 'Failed to authenticate: {0}'.format(to_text(e)) + raise AnsibleAuthenticationFailure(msg) + except Exception as e: + msg = to_text(e) + if u"PID check failed" in msg: + raise AnsibleError("paramiko version issue, please upgrade paramiko on the machine running ansible") + elif u"Private key file is encrypted" in msg: + msg = 'ssh %s@%s:%s : %s\nTo connect as a different user, use -u <username>.' % ( + self._play_context.remote_user, self._play_context.remote_addr, port, msg) + raise AnsibleConnectionFailure(msg) + else: + raise AnsibleConnectionFailure(msg) + + return ssh + + def exec_command(self, cmd, in_data=None, sudoable=True): + ''' run a command on the remote host ''' + + super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable) + + if in_data: + raise AnsibleError("Internal Error: this module does not support optimized module pipelining") + + bufsize = 4096 + + try: + self.ssh.get_transport().set_keepalive(5) + chan = self.ssh.get_transport().open_session() + except Exception as e: + text_e = to_text(e) + msg = u"Failed to open session" + if text_e: + msg += u": %s" % text_e + raise AnsibleConnectionFailure(to_native(msg)) + + # sudo usually requires a PTY (cf. requiretty option), therefore + # we give it one by default (pty=True in ansible.cfg), and we try + # to initialise from the calling environment when sudoable is enabled + if self.get_option('pty') and sudoable: + chan.get_pty(term=os.getenv('TERM', 'vt100'), width=int(os.getenv('COLUMNS', 0)), height=int(os.getenv('LINES', 0))) + + display.vvv("EXEC %s" % cmd, host=self._play_context.remote_addr) + + cmd = to_bytes(cmd, errors='surrogate_or_strict') + + no_prompt_out = b'' + no_prompt_err = b'' + become_output = b'' + + try: + chan.exec_command(cmd) + if self.become and self.become.expect_prompt(): + passprompt = False + become_sucess = False + while not (become_sucess or passprompt): + display.debug('Waiting for Privilege Escalation input') + + chunk = chan.recv(bufsize) + display.debug("chunk is: %s" % chunk) + if not chunk: + if b'unknown user' in become_output: + n_become_user = to_native(self.become.get_option('become_user', + playcontext=self._play_context)) + raise AnsibleError('user %s does not exist' % n_become_user) + else: + break + # raise AnsibleError('ssh connection closed waiting for password prompt') + become_output += chunk + + # need to check every line because we might get lectured + # and we might get the middle of a line in a chunk + for l in become_output.splitlines(True): + if self.become.check_success(l): + become_sucess = True + break + elif self.become.check_password_prompt(l): + passprompt = True + break + + if passprompt: + if self.become: + become_pass = self.become.get_option('become_pass', playcontext=self._play_context) + chan.sendall(to_bytes(become_pass, errors='surrogate_or_strict') + b'\n') + else: + raise AnsibleError("A password is required but none was supplied") + else: + no_prompt_out += become_output + no_prompt_err += become_output + except socket.timeout: + raise AnsibleError('ssh timed out waiting for privilege escalation.\n' + become_output) + + stdout = b''.join(chan.makefile('rb', bufsize)) + stderr = b''.join(chan.makefile_stderr('rb', bufsize)) + + return (chan.recv_exit_status(), no_prompt_out + stdout, no_prompt_out + stderr) + + def put_file(self, in_path, out_path): + ''' transfer a file from local to remote ''' + + super(Connection, self).put_file(in_path, out_path) + + display.vvv("PUT %s TO %s" % (in_path, out_path), host=self._play_context.remote_addr) + + if not os.path.exists(to_bytes(in_path, errors='surrogate_or_strict')): + raise AnsibleFileNotFound("file or module does not exist: %s" % in_path) + + try: + self.sftp = self.ssh.open_sftp() + except Exception as e: + raise AnsibleError("failed to open a SFTP connection (%s)" % e) + + try: + self.sftp.put(to_bytes(in_path, errors='surrogate_or_strict'), to_bytes(out_path, errors='surrogate_or_strict')) + except IOError: + raise AnsibleError("failed to transfer file to %s" % out_path) + + def _connect_sftp(self): + + cache_key = "%s__%s__" % (self._play_context.remote_addr, self._play_context.remote_user) + if cache_key in SFTP_CONNECTION_CACHE: + return SFTP_CONNECTION_CACHE[cache_key] + else: + result = SFTP_CONNECTION_CACHE[cache_key] = self._connect().ssh.open_sftp() + return result + + def fetch_file(self, in_path, out_path): + ''' save a remote file to the specified path ''' + + super(Connection, self).fetch_file(in_path, out_path) + + display.vvv("FETCH %s TO %s" % (in_path, out_path), host=self._play_context.remote_addr) + + try: + self.sftp = self._connect_sftp() + except Exception as e: + raise AnsibleError("failed to open a SFTP connection (%s)" % to_native(e)) + + try: + self.sftp.get(to_bytes(in_path, errors='surrogate_or_strict'), to_bytes(out_path, errors='surrogate_or_strict')) + except IOError: + raise AnsibleError("failed to transfer file from %s" % in_path) + + def _any_keys_added(self): + + for hostname, keys in self.ssh._host_keys.items(): + for keytype, key in keys.items(): + added_this_time = getattr(key, '_added_by_ansible_this_time', False) + if added_this_time: + return True + return False + + def _save_ssh_host_keys(self, filename): + ''' + not using the paramiko save_ssh_host_keys function as we want to add new SSH keys at the bottom so folks + don't complain about it :) + ''' + + if not self._any_keys_added(): + return False + + path = os.path.expanduser("~/.ssh") + makedirs_safe(path) + + with open(filename, 'w') as f: + + for hostname, keys in self.ssh._host_keys.items(): + + for keytype, key in keys.items(): + + # was f.write + added_this_time = getattr(key, '_added_by_ansible_this_time', False) + if not added_this_time: + f.write("%s %s %s\n" % (hostname, keytype, key.get_base64())) + + for hostname, keys in self.ssh._host_keys.items(): + + for keytype, key in keys.items(): + added_this_time = getattr(key, '_added_by_ansible_this_time', False) + if added_this_time: + f.write("%s %s %s\n" % (hostname, keytype, key.get_base64())) + + def reset(self): + if not self._connected: + return + self.close() + self._connect() + + def close(self): + ''' terminate the connection ''' + + cache_key = self._cache_key() + SSH_CONNECTION_CACHE.pop(cache_key, None) + SFTP_CONNECTION_CACHE.pop(cache_key, None) + + if hasattr(self, 'sftp'): + if self.sftp is not None: + self.sftp.close() + + if self.get_option('host_key_checking') and self.get_option('record_host_keys') and self._any_keys_added(): + + # add any new SSH host keys -- warning -- this could be slow + # (This doesn't acquire the connection lock because it needs + # to exclude only other known_hosts writers, not connections + # that are starting up.) + lockfile = self.keyfile.replace("known_hosts", ".known_hosts.lock") + dirname = os.path.dirname(self.keyfile) + makedirs_safe(dirname) + + KEY_LOCK = open(lockfile, 'w') + fcntl.lockf(KEY_LOCK, fcntl.LOCK_EX) + + try: + # just in case any were added recently + + self.ssh.load_system_host_keys() + self.ssh._host_keys.update(self.ssh._system_host_keys) + + # gather information about the current key file, so + # we can ensure the new file has the correct mode/owner + + key_dir = os.path.dirname(self.keyfile) + if os.path.exists(self.keyfile): + key_stat = os.stat(self.keyfile) + mode = key_stat.st_mode + uid = key_stat.st_uid + gid = key_stat.st_gid + else: + mode = 33188 + uid = os.getuid() + gid = os.getgid() + + # Save the new keys to a temporary file and move it into place + # rather than rewriting the file. We set delete=False because + # the file will be moved into place rather than cleaned up. + + tmp_keyfile = tempfile.NamedTemporaryFile(dir=key_dir, delete=False) + os.chmod(tmp_keyfile.name, mode & 0o7777) + os.chown(tmp_keyfile.name, uid, gid) + + self._save_ssh_host_keys(tmp_keyfile.name) + tmp_keyfile.close() + + os.rename(tmp_keyfile.name, self.keyfile) + + except Exception: + + # unable to save keys, including scenario when key was invalid + # and caught earlier + traceback.print_exc() + fcntl.lockf(KEY_LOCK, fcntl.LOCK_UN) + + self.ssh.close() + self._connected = False diff --git a/lib/ansible/plugins/connection/psrp.py b/lib/ansible/plugins/connection/psrp.py new file mode 100644 index 0000000..dfcf0e5 --- /dev/null +++ b/lib/ansible/plugins/connection/psrp.py @@ -0,0 +1,898 @@ +# Copyright (c) 2018 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = """ +author: Ansible Core Team +name: psrp +short_description: Run tasks over Microsoft PowerShell Remoting Protocol +description: +- Run commands or put/fetch on a target via PSRP (WinRM plugin) +- This is similar to the I(winrm) connection plugin which uses the same + underlying transport but instead runs in a PowerShell interpreter. +version_added: "2.7" +requirements: +- pypsrp>=0.4.0 (Python library) +extends_documentation_fragment: + - connection_pipelining +options: + # transport options + remote_addr: + description: + - The hostname or IP address of the remote host. + default: inventory_hostname + type: str + vars: + - name: inventory_hostname + - name: ansible_host + - name: ansible_psrp_host + remote_user: + description: + - The user to log in as. + type: str + vars: + - name: ansible_user + - name: ansible_psrp_user + keyword: + - name: remote_user + remote_password: + description: Authentication password for the C(remote_user). Can be supplied as CLI option. + type: str + vars: + - name: ansible_password + - name: ansible_winrm_pass + - name: ansible_winrm_password + aliases: + - password # Needed for --ask-pass to come through on delegation + port: + description: + - The port for PSRP to connect on the remote target. + - Default is C(5986) if I(protocol) is not defined or is C(https), + otherwise the port is C(5985). + type: int + vars: + - name: ansible_port + - name: ansible_psrp_port + keyword: + - name: port + protocol: + description: + - Set the protocol to use for the connection. + - Default is C(https) if I(port) is not defined or I(port) is not C(5985). + choices: + - http + - https + type: str + vars: + - name: ansible_psrp_protocol + path: + description: + - The URI path to connect to. + type: str + vars: + - name: ansible_psrp_path + default: 'wsman' + auth: + description: + - The authentication protocol to use when authenticating the remote user. + - The default, C(negotiate), will attempt to use C(Kerberos) if it is + available and fall back to C(NTLM) if it isn't. + type: str + vars: + - name: ansible_psrp_auth + choices: + - basic + - certificate + - negotiate + - kerberos + - ntlm + - credssp + default: negotiate + cert_validation: + description: + - Whether to validate the remote server's certificate or not. + - Set to C(ignore) to not validate any certificates. + - I(ca_cert) can be set to the path of a PEM certificate chain to + use in the validation. + choices: + - validate + - ignore + default: validate + type: str + vars: + - name: ansible_psrp_cert_validation + ca_cert: + description: + - The path to a PEM certificate chain to use when validating the server's + certificate. + - This value is ignored if I(cert_validation) is set to C(ignore). + type: path + vars: + - name: ansible_psrp_cert_trust_path + - name: ansible_psrp_ca_cert + aliases: [ cert_trust_path ] + connection_timeout: + description: + - The connection timeout for making the request to the remote host. + - This is measured in seconds. + type: int + vars: + - name: ansible_psrp_connection_timeout + default: 30 + read_timeout: + description: + - The read timeout for receiving data from the remote host. + - This value must always be greater than I(operation_timeout). + - This option requires pypsrp >= 0.3. + - This is measured in seconds. + type: int + vars: + - name: ansible_psrp_read_timeout + default: 30 + version_added: '2.8' + reconnection_retries: + description: + - The number of retries on connection errors. + type: int + vars: + - name: ansible_psrp_reconnection_retries + default: 0 + version_added: '2.8' + reconnection_backoff: + description: + - The backoff time to use in between reconnection attempts. + (First sleeps X, then sleeps 2*X, then sleeps 4*X, ...) + - This is measured in seconds. + - The C(ansible_psrp_reconnection_backoff) variable was added in Ansible + 2.9. + type: int + vars: + - name: ansible_psrp_connection_backoff + - name: ansible_psrp_reconnection_backoff + default: 2 + version_added: '2.8' + message_encryption: + description: + - Controls the message encryption settings, this is different from TLS + encryption when I(ansible_psrp_protocol) is C(https). + - Only the auth protocols C(negotiate), C(kerberos), C(ntlm), and + C(credssp) can do message encryption. The other authentication protocols + only support encryption when C(protocol) is set to C(https). + - C(auto) means means message encryption is only used when not using + TLS/HTTPS. + - C(always) is the same as C(auto) but message encryption is always used + even when running over TLS/HTTPS. + - C(never) disables any encryption checks that are in place when running + over HTTP and disables any authentication encryption processes. + type: str + vars: + - name: ansible_psrp_message_encryption + choices: + - auto + - always + - never + default: auto + proxy: + description: + - Set the proxy URL to use when connecting to the remote host. + vars: + - name: ansible_psrp_proxy + type: str + ignore_proxy: + description: + - Will disable any environment proxy settings and connect directly to the + remote host. + - This option is ignored if C(proxy) is set. + vars: + - name: ansible_psrp_ignore_proxy + type: bool + default: 'no' + + # auth options + certificate_key_pem: + description: + - The local path to an X509 certificate key to use with certificate auth. + type: path + vars: + - name: ansible_psrp_certificate_key_pem + certificate_pem: + description: + - The local path to an X509 certificate to use with certificate auth. + type: path + vars: + - name: ansible_psrp_certificate_pem + credssp_auth_mechanism: + description: + - The sub authentication mechanism to use with CredSSP auth. + - When C(auto), both Kerberos and NTLM is attempted with kerberos being + preferred. + type: str + choices: + - auto + - kerberos + - ntlm + default: auto + vars: + - name: ansible_psrp_credssp_auth_mechanism + credssp_disable_tlsv1_2: + description: + - Disables the use of TLSv1.2 on the CredSSP authentication channel. + - This should not be set to C(yes) unless dealing with a host that does not + have TLSv1.2. + default: no + type: bool + vars: + - name: ansible_psrp_credssp_disable_tlsv1_2 + credssp_minimum_version: + description: + - The minimum CredSSP server authentication version that will be accepted. + - Set to C(5) to ensure the server has been patched and is not vulnerable + to CVE 2018-0886. + default: 2 + type: int + vars: + - name: ansible_psrp_credssp_minimum_version + negotiate_delegate: + description: + - Allow the remote user the ability to delegate it's credentials to another + server, i.e. credential delegation. + - Only valid when Kerberos was the negotiated auth or was explicitly set as + the authentication. + - Ignored when NTLM was the negotiated auth. + type: bool + vars: + - name: ansible_psrp_negotiate_delegate + negotiate_hostname_override: + description: + - Override the remote hostname when searching for the host in the Kerberos + lookup. + - This allows Ansible to connect over IP but authenticate with the remote + server using it's DNS name. + - Only valid when Kerberos was the negotiated auth or was explicitly set as + the authentication. + - Ignored when NTLM was the negotiated auth. + type: str + vars: + - name: ansible_psrp_negotiate_hostname_override + negotiate_send_cbt: + description: + - Send the Channel Binding Token (CBT) structure when authenticating. + - CBT is used to provide extra protection against Man in the Middle C(MitM) + attacks by binding the outer transport channel to the auth channel. + - CBT is not used when using just C(HTTP), only C(HTTPS). + default: yes + type: bool + vars: + - name: ansible_psrp_negotiate_send_cbt + negotiate_service: + description: + - Override the service part of the SPN used during Kerberos authentication. + - Only valid when Kerberos was the negotiated auth or was explicitly set as + the authentication. + - Ignored when NTLM was the negotiated auth. + default: WSMAN + type: str + vars: + - name: ansible_psrp_negotiate_service + + # protocol options + operation_timeout: + description: + - Sets the WSMan timeout for each operation. + - This is measured in seconds. + - This should not exceed the value for C(connection_timeout). + type: int + vars: + - name: ansible_psrp_operation_timeout + default: 20 + max_envelope_size: + description: + - Sets the maximum size of each WSMan message sent to the remote host. + - This is measured in bytes. + - Defaults to C(150KiB) for compatibility with older hosts. + type: int + vars: + - name: ansible_psrp_max_envelope_size + default: 153600 + configuration_name: + description: + - The name of the PowerShell configuration endpoint to connect to. + type: str + vars: + - name: ansible_psrp_configuration_name + default: Microsoft.PowerShell +""" + +import base64 +import json +import logging +import os + +from ansible import constants as C +from ansible.errors import AnsibleConnectionFailure, AnsibleError +from ansible.errors import AnsibleFileNotFound +from ansible.module_utils.parsing.convert_bool import boolean +from ansible.module_utils._text import to_bytes, to_native, to_text +from ansible.plugins.connection import ConnectionBase +from ansible.plugins.shell.powershell import _common_args +from ansible.utils.display import Display +from ansible.utils.hashing import sha1 + +HAS_PYPSRP = True +PYPSRP_IMP_ERR = None +try: + import pypsrp + from pypsrp.complex_objects import GenericComplexObject, PSInvocationState, RunspacePoolState + from pypsrp.exceptions import AuthenticationError, WinRMError + from pypsrp.host import PSHost, PSHostUserInterface + from pypsrp.powershell import PowerShell, RunspacePool + from pypsrp.wsman import WSMan, AUTH_KWARGS + from requests.exceptions import ConnectionError, ConnectTimeout +except ImportError as err: + HAS_PYPSRP = False + PYPSRP_IMP_ERR = err + +display = Display() + + +class Connection(ConnectionBase): + + transport = 'psrp' + module_implementation_preferences = ('.ps1', '.exe', '') + allow_executable = False + has_pipelining = True + allow_extras = True + + def __init__(self, *args, **kwargs): + self.always_pipeline_modules = True + self.has_native_async = True + + self.runspace = None + self.host = None + self._last_pipeline = False + + self._shell_type = 'powershell' + super(Connection, self).__init__(*args, **kwargs) + + if not C.DEFAULT_DEBUG: + logging.getLogger('pypsrp').setLevel(logging.WARNING) + logging.getLogger('requests_credssp').setLevel(logging.INFO) + logging.getLogger('urllib3').setLevel(logging.INFO) + + def _connect(self): + if not HAS_PYPSRP: + raise AnsibleError("pypsrp or dependencies are not installed: %s" + % to_native(PYPSRP_IMP_ERR)) + super(Connection, self)._connect() + self._build_kwargs() + display.vvv("ESTABLISH PSRP CONNECTION FOR USER: %s ON PORT %s TO %s" % + (self._psrp_user, self._psrp_port, self._psrp_host), + host=self._psrp_host) + + if not self.runspace: + connection = WSMan(**self._psrp_conn_kwargs) + + # create our pseudo host to capture the exit code and host output + host_ui = PSHostUserInterface() + self.host = PSHost(None, None, False, "Ansible PSRP Host", None, + host_ui, None) + + self.runspace = RunspacePool( + connection, host=self.host, + configuration_name=self._psrp_configuration_name + ) + display.vvvvv( + "PSRP OPEN RUNSPACE: auth=%s configuration=%s endpoint=%s" % + (self._psrp_auth, self._psrp_configuration_name, + connection.transport.endpoint), host=self._psrp_host + ) + try: + self.runspace.open() + except AuthenticationError as e: + raise AnsibleConnectionFailure("failed to authenticate with " + "the server: %s" % to_native(e)) + except WinRMError as e: + raise AnsibleConnectionFailure( + "psrp connection failure during runspace open: %s" + % to_native(e) + ) + except (ConnectionError, ConnectTimeout) as e: + raise AnsibleConnectionFailure( + "Failed to connect to the host via PSRP: %s" + % to_native(e) + ) + + self._connected = True + self._last_pipeline = None + return self + + def reset(self): + if not self._connected: + self.runspace = None + return + + # Try out best to ensure the runspace is closed to free up server side resources + try: + self.close() + except Exception as e: + # There's a good chance the connection was already closed so just log the error and move on + display.debug("PSRP reset - failed to closed runspace: %s" % to_text(e)) + + display.vvvvv("PSRP: Reset Connection", host=self._psrp_host) + self.runspace = None + self._connect() + + def exec_command(self, cmd, in_data=None, sudoable=True): + super(Connection, self).exec_command(cmd, in_data=in_data, + sudoable=sudoable) + + if cmd.startswith(" ".join(_common_args) + " -EncodedCommand"): + # This is a PowerShell script encoded by the shell plugin, we will + # decode the script and execute it in the runspace instead of + # starting a new interpreter to save on time + b_command = base64.b64decode(cmd.split(" ")[-1]) + script = to_text(b_command, 'utf-16-le') + in_data = to_text(in_data, errors="surrogate_or_strict", nonstring="passthru") + + if in_data and in_data.startswith(u"#!"): + # ANSIBALLZ wrapper, we need to get the interpreter and execute + # that as the script - note this won't work as basic.py relies + # on packages not available on Windows, once fixed we can enable + # this path + interpreter = to_native(in_data.splitlines()[0][2:]) + # script = "$input | &'%s' -" % interpreter + # in_data = to_text(in_data) + raise AnsibleError("cannot run the interpreter '%s' on the psrp " + "connection plugin" % interpreter) + + # call build_module_command to get the bootstrap wrapper text + bootstrap_wrapper = self._shell.build_module_command('', '', '') + if bootstrap_wrapper == cmd: + # Do not display to the user each invocation of the bootstrap wrapper + display.vvv("PSRP: EXEC (via pipeline wrapper)") + else: + display.vvv("PSRP: EXEC %s" % script, host=self._psrp_host) + else: + # In other cases we want to execute the cmd as the script. We add on the 'exit $LASTEXITCODE' to ensure the + # rc is propagated back to the connection plugin. + script = to_text(u"%s\nexit $LASTEXITCODE" % cmd) + display.vvv(u"PSRP: EXEC %s" % script, host=self._psrp_host) + + rc, stdout, stderr = self._exec_psrp_script(script, in_data) + return rc, stdout, stderr + + def put_file(self, in_path, out_path): + super(Connection, self).put_file(in_path, out_path) + + out_path = self._shell._unquote(out_path) + display.vvv("PUT %s TO %s" % (in_path, out_path), host=self._psrp_host) + + copy_script = '''begin { + $ErrorActionPreference = "Stop" + $WarningPreference = "Continue" + $path = $MyInvocation.UnboundArguments[0] + $fd = [System.IO.File]::Create($path) + $algo = [System.Security.Cryptography.SHA1CryptoServiceProvider]::Create() + $bytes = @() + + $bindingFlags = [System.Reflection.BindingFlags]'NonPublic, Instance' + Function Get-Property { + <# + .SYNOPSIS + Gets the private/internal property specified of the object passed in. + #> + Param ( + [Parameter(Mandatory=$true, ValueFromPipeline=$true)] + [System.Object] + $Object, + + [Parameter(Mandatory=$true, Position=1)] + [System.String] + $Name + ) + + $Object.GetType().GetProperty($Name, $bindingFlags).GetValue($Object, $null) + } + + Function Set-Property { + <# + .SYNOPSIS + Sets the private/internal property specified on the object passed in. + #> + Param ( + [Parameter(Mandatory=$true, ValueFromPipeline=$true)] + [System.Object] + $Object, + + [Parameter(Mandatory=$true, Position=1)] + [System.String] + $Name, + + [Parameter(Mandatory=$true, Position=2)] + [AllowNull()] + [System.Object] + $Value + ) + + $Object.GetType().GetProperty($Name, $bindingFlags).SetValue($Object, $Value, $null) + } + + Function Get-Field { + <# + .SYNOPSIS + Gets the private/internal field specified of the object passed in. + #> + Param ( + [Parameter(Mandatory=$true, ValueFromPipeline=$true)] + [System.Object] + $Object, + + [Parameter(Mandatory=$true, Position=1)] + [System.String] + $Name + ) + + $Object.GetType().GetField($Name, $bindingFlags).GetValue($Object) + } + + # MaximumAllowedMemory is required to be set to so we can send input data that exceeds the limit on a PS + # Runspace. We use reflection to access/set this property as it is not accessible publicly. This is not ideal + # but works on all PowerShell versions I've tested with. We originally used WinRS to send the raw bytes to the + # host but this falls flat if someone is using a custom PS configuration name so this is a workaround. This + # isn't required for smaller files so if it fails we ignore the error and hope it wasn't needed. + # https://github.com/PowerShell/PowerShell/blob/c8e72d1e664b1ee04a14f226adf655cced24e5f0/src/System.Management.Automation/engine/serialization.cs#L325 + try { + $Host | Get-Property 'ExternalHost' | ` + Get-Field '_transportManager' | ` + Get-Property 'Fragmentor' | ` + Get-Property 'DeserializationContext' | ` + Set-Property 'MaximumAllowedMemory' $null + } catch {} +} +process { + $bytes = [System.Convert]::FromBase64String($input) + $algo.TransformBlock($bytes, 0, $bytes.Length, $bytes, 0) > $null + $fd.Write($bytes, 0, $bytes.Length) +} +end { + $fd.Close() + + $algo.TransformFinalBlock($bytes, 0, 0) > $null + $hash = [System.BitConverter]::ToString($algo.Hash).Replace('-', '').ToLowerInvariant() + Write-Output -InputObject "{`"sha1`":`"$hash`"}" +} +''' + + # Get the buffer size of each fragment to send, subtract 82 for the fragment, message, and other header info + # fields that PSRP adds. Adjust to size of the base64 encoded bytes length. + buffer_size = int((self.runspace.connection.max_payload_size - 82) / 4 * 3) + + sha1_hash = sha1() + + b_in_path = to_bytes(in_path, errors='surrogate_or_strict') + if not os.path.exists(b_in_path): + raise AnsibleFileNotFound('file or module does not exist: "%s"' % to_native(in_path)) + + def read_gen(): + offset = 0 + + with open(b_in_path, 'rb') as src_fd: + for b_data in iter((lambda: src_fd.read(buffer_size)), b""): + data_len = len(b_data) + offset += data_len + sha1_hash.update(b_data) + + # PSRP technically supports sending raw bytes but that method requires a larger CLIXML message. + # Sending base64 is still more efficient here. + display.vvvvv("PSRP PUT %s to %s (offset=%d, size=%d" % (in_path, out_path, offset, data_len), + host=self._psrp_host) + b64_data = base64.b64encode(b_data) + yield [to_text(b64_data)] + + if offset == 0: # empty file + yield [""] + + rc, stdout, stderr = self._exec_psrp_script(copy_script, read_gen(), arguments=[out_path]) + + if rc != 0: + raise AnsibleError(to_native(stderr)) + + put_output = json.loads(to_text(stdout)) + local_sha1 = sha1_hash.hexdigest() + remote_sha1 = put_output.get("sha1") + + if not remote_sha1: + raise AnsibleError("Remote sha1 was not returned, stdout: '%s', stderr: '%s'" + % (to_native(stdout), to_native(stderr))) + + if not remote_sha1 == local_sha1: + raise AnsibleError("Remote sha1 hash %s does not match local hash %s" + % (to_native(remote_sha1), to_native(local_sha1))) + + def fetch_file(self, in_path, out_path): + super(Connection, self).fetch_file(in_path, out_path) + display.vvv("FETCH %s TO %s" % (in_path, out_path), + host=self._psrp_host) + + in_path = self._shell._unquote(in_path) + out_path = out_path.replace('\\', '/') + + # because we are dealing with base64 data we need to get the max size + # of the bytes that the base64 size would equal + max_b64_size = int(self.runspace.connection.max_payload_size - + (self.runspace.connection.max_payload_size / 4 * 3)) + buffer_size = max_b64_size - (max_b64_size % 1024) + + # setup the file stream with read only mode + setup_script = '''$ErrorActionPreference = "Stop" +$path = '%s' + +if (Test-Path -Path $path -PathType Leaf) { + $fs = New-Object -TypeName System.IO.FileStream -ArgumentList @( + $path, + [System.IO.FileMode]::Open, + [System.IO.FileAccess]::Read, + [System.IO.FileShare]::Read + ) + $buffer_size = %d +} elseif (Test-Path -Path $path -PathType Container) { + Write-Output -InputObject "[DIR]" +} else { + Write-Error -Message "$path does not exist" + $host.SetShouldExit(1) +}''' % (self._shell._escape(in_path), buffer_size) + + # read the file stream at the offset and return the b64 string + read_script = '''$ErrorActionPreference = "Stop" +$fs.Seek(%d, [System.IO.SeekOrigin]::Begin) > $null +$buffer = New-Object -TypeName byte[] -ArgumentList $buffer_size +$bytes_read = $fs.Read($buffer, 0, $buffer_size) + +if ($bytes_read -gt 0) { + $bytes = $buffer[0..($bytes_read - 1)] + Write-Output -InputObject ([System.Convert]::ToBase64String($bytes)) +}''' + + # need to run the setup script outside of the local scope so the + # file stream stays active between fetch operations + rc, stdout, stderr = self._exec_psrp_script(setup_script, + use_local_scope=False) + if rc != 0: + raise AnsibleError("failed to setup file stream for fetch '%s': %s" + % (out_path, to_native(stderr))) + elif stdout.strip() == '[DIR]': + # to be consistent with other connection plugins, we assume the caller has created the target dir + return + + b_out_path = to_bytes(out_path, errors='surrogate_or_strict') + # to be consistent with other connection plugins, we assume the caller has created the target dir + offset = 0 + with open(b_out_path, 'wb') as out_file: + while True: + display.vvvvv("PSRP FETCH %s to %s (offset=%d" % + (in_path, out_path, offset), host=self._psrp_host) + rc, stdout, stderr = self._exec_psrp_script(read_script % offset) + if rc != 0: + raise AnsibleError("failed to transfer file to '%s': %s" + % (out_path, to_native(stderr))) + + data = base64.b64decode(stdout.strip()) + out_file.write(data) + if len(data) < buffer_size: + break + offset += len(data) + + rc, stdout, stderr = self._exec_psrp_script("$fs.Close()") + if rc != 0: + display.warning("failed to close remote file stream of file " + "'%s': %s" % (in_path, to_native(stderr))) + + def close(self): + if self.runspace and self.runspace.state == RunspacePoolState.OPENED: + display.vvvvv("PSRP CLOSE RUNSPACE: %s" % (self.runspace.id), + host=self._psrp_host) + self.runspace.close() + self.runspace = None + self._connected = False + self._last_pipeline = None + + def _build_kwargs(self): + self._psrp_host = self.get_option('remote_addr') + self._psrp_user = self.get_option('remote_user') + self._psrp_pass = self.get_option('remote_password') + + protocol = self.get_option('protocol') + port = self.get_option('port') + if protocol is None and port is None: + protocol = 'https' + port = 5986 + elif protocol is None: + protocol = 'https' if int(port) != 5985 else 'http' + elif port is None: + port = 5986 if protocol == 'https' else 5985 + + self._psrp_protocol = protocol + self._psrp_port = int(port) + + self._psrp_path = self.get_option('path') + self._psrp_auth = self.get_option('auth') + # cert validation can either be a bool or a path to the cert + cert_validation = self.get_option('cert_validation') + cert_trust_path = self.get_option('ca_cert') + if cert_validation == 'ignore': + self._psrp_cert_validation = False + elif cert_trust_path is not None: + self._psrp_cert_validation = cert_trust_path + else: + self._psrp_cert_validation = True + + self._psrp_connection_timeout = self.get_option('connection_timeout') # Can be None + self._psrp_read_timeout = self.get_option('read_timeout') # Can be None + self._psrp_message_encryption = self.get_option('message_encryption') + self._psrp_proxy = self.get_option('proxy') + self._psrp_ignore_proxy = boolean(self.get_option('ignore_proxy')) + self._psrp_operation_timeout = int(self.get_option('operation_timeout')) + self._psrp_max_envelope_size = int(self.get_option('max_envelope_size')) + self._psrp_configuration_name = self.get_option('configuration_name') + self._psrp_reconnection_retries = int(self.get_option('reconnection_retries')) + self._psrp_reconnection_backoff = float(self.get_option('reconnection_backoff')) + + self._psrp_certificate_key_pem = self.get_option('certificate_key_pem') + self._psrp_certificate_pem = self.get_option('certificate_pem') + self._psrp_credssp_auth_mechanism = self.get_option('credssp_auth_mechanism') + self._psrp_credssp_disable_tlsv1_2 = self.get_option('credssp_disable_tlsv1_2') + self._psrp_credssp_minimum_version = self.get_option('credssp_minimum_version') + self._psrp_negotiate_send_cbt = self.get_option('negotiate_send_cbt') + self._psrp_negotiate_delegate = self.get_option('negotiate_delegate') + self._psrp_negotiate_hostname_override = self.get_option('negotiate_hostname_override') + self._psrp_negotiate_service = self.get_option('negotiate_service') + + supported_args = [] + for auth_kwarg in AUTH_KWARGS.values(): + supported_args.extend(auth_kwarg) + extra_args = {v.replace('ansible_psrp_', '') for v in self.get_option('_extras')} + unsupported_args = extra_args.difference(supported_args) + + for arg in unsupported_args: + display.warning("ansible_psrp_%s is unsupported by the current " + "psrp version installed" % arg) + + self._psrp_conn_kwargs = dict( + server=self._psrp_host, port=self._psrp_port, + username=self._psrp_user, password=self._psrp_pass, + ssl=self._psrp_protocol == 'https', path=self._psrp_path, + auth=self._psrp_auth, cert_validation=self._psrp_cert_validation, + connection_timeout=self._psrp_connection_timeout, + encryption=self._psrp_message_encryption, proxy=self._psrp_proxy, + no_proxy=self._psrp_ignore_proxy, + max_envelope_size=self._psrp_max_envelope_size, + operation_timeout=self._psrp_operation_timeout, + certificate_key_pem=self._psrp_certificate_key_pem, + certificate_pem=self._psrp_certificate_pem, + credssp_auth_mechanism=self._psrp_credssp_auth_mechanism, + credssp_disable_tlsv1_2=self._psrp_credssp_disable_tlsv1_2, + credssp_minimum_version=self._psrp_credssp_minimum_version, + negotiate_send_cbt=self._psrp_negotiate_send_cbt, + negotiate_delegate=self._psrp_negotiate_delegate, + negotiate_hostname_override=self._psrp_negotiate_hostname_override, + negotiate_service=self._psrp_negotiate_service, + ) + + # Check if PSRP version supports newer read_timeout argument (needs pypsrp 0.3.0+) + if hasattr(pypsrp, 'FEATURES') and 'wsman_read_timeout' in pypsrp.FEATURES: + self._psrp_conn_kwargs['read_timeout'] = self._psrp_read_timeout + elif self._psrp_read_timeout is not None: + display.warning("ansible_psrp_read_timeout is unsupported by the current psrp version installed, " + "using ansible_psrp_connection_timeout value for read_timeout instead.") + + # Check if PSRP version supports newer reconnection_retries argument (needs pypsrp 0.3.0+) + if hasattr(pypsrp, 'FEATURES') and 'wsman_reconnections' in pypsrp.FEATURES: + self._psrp_conn_kwargs['reconnection_retries'] = self._psrp_reconnection_retries + self._psrp_conn_kwargs['reconnection_backoff'] = self._psrp_reconnection_backoff + else: + if self._psrp_reconnection_retries is not None: + display.warning("ansible_psrp_reconnection_retries is unsupported by the current psrp version installed.") + if self._psrp_reconnection_backoff is not None: + display.warning("ansible_psrp_reconnection_backoff is unsupported by the current psrp version installed.") + + # add in the extra args that were set + for arg in extra_args.intersection(supported_args): + option = self.get_option('_extras')['ansible_psrp_%s' % arg] + self._psrp_conn_kwargs[arg] = option + + def _exec_psrp_script(self, script, input_data=None, use_local_scope=True, arguments=None): + # Check if there's a command on the current pipeline that still needs to be closed. + if self._last_pipeline: + # Current pypsrp versions raise an exception if the current state was not RUNNING. We manually set it so we + # can call stop without any issues. + self._last_pipeline.state = PSInvocationState.RUNNING + self._last_pipeline.stop() + self._last_pipeline = None + + ps = PowerShell(self.runspace) + ps.add_script(script, use_local_scope=use_local_scope) + if arguments: + for arg in arguments: + ps.add_argument(arg) + + ps.invoke(input=input_data) + + rc, stdout, stderr = self._parse_pipeline_result(ps) + + # We should really call .stop() on all pipelines that are run to decrement the concurrent command counter on + # PSSession but that involves another round trip and is done when the runspace is closed. We instead store the + # last pipeline which is closed if another command is run on the runspace. + self._last_pipeline = ps + + return rc, stdout, stderr + + def _parse_pipeline_result(self, pipeline): + """ + PSRP doesn't have the same concept as other protocols with its output. + We need some extra logic to convert the pipeline streams and host + output into the format that Ansible understands. + + :param pipeline: The finished PowerShell pipeline that invoked our + commands + :return: rc, stdout, stderr based on the pipeline output + """ + # we try and get the rc from our host implementation, this is set if + # exit or $host.SetShouldExit() is called in our pipeline, if not we + # set to 0 if the pipeline had not errors and 1 if it did + rc = self.host.rc or (1 if pipeline.had_errors else 0) + + # TODO: figure out a better way of merging this with the host output + stdout_list = [] + for output in pipeline.output: + # Not all pipeline outputs are a string or contain a __str__ value, + # we will create our own output based on the properties of the + # complex object if that is the case. + if isinstance(output, GenericComplexObject) and output.to_string is None: + obj_lines = output.property_sets + for key, value in output.adapted_properties.items(): + obj_lines.append(u"%s: %s" % (key, value)) + for key, value in output.extended_properties.items(): + obj_lines.append(u"%s: %s" % (key, value)) + output_msg = u"\n".join(obj_lines) + else: + output_msg = to_text(output, nonstring='simplerepr') + + stdout_list.append(output_msg) + + if len(self.host.ui.stdout) > 0: + stdout_list += self.host.ui.stdout + stdout = u"\r\n".join(stdout_list) + + stderr_list = [] + for error in pipeline.streams.error: + # the error record is not as fully fleshed out like we usually get + # in PS, we will manually create it here + command_name = "%s : " % error.command_name if error.command_name else '' + position = "%s\r\n" % error.invocation_position_message if error.invocation_position_message else '' + error_msg = "%s%s\r\n%s" \ + " + CategoryInfo : %s\r\n" \ + " + FullyQualifiedErrorId : %s" \ + % (command_name, str(error), position, + error.message, error.fq_error) + stacktrace = error.script_stacktrace + if display.verbosity >= 3 and stacktrace is not None: + error_msg += "\r\nStackTrace:\r\n%s" % stacktrace + stderr_list.append(error_msg) + + if len(self.host.ui.stderr) > 0: + stderr_list += self.host.ui.stderr + stderr = u"\r\n".join([to_text(o) for o in stderr_list]) + + display.vvvvv("PSRP RC: %d" % rc, host=self._psrp_host) + display.vvvvv("PSRP STDOUT: %s" % stdout, host=self._psrp_host) + display.vvvvv("PSRP STDERR: %s" % stderr, host=self._psrp_host) + + # reset the host back output back to defaults, needed if running + # multiple pipelines on the same RunspacePool + self.host.rc = 0 + self.host.ui.stdout = [] + self.host.ui.stderr = [] + + return rc, to_bytes(stdout, encoding='utf-8'), to_bytes(stderr, encoding='utf-8') diff --git a/lib/ansible/plugins/connection/ssh.py b/lib/ansible/plugins/connection/ssh.py new file mode 100644 index 0000000..e4d9628 --- /dev/null +++ b/lib/ansible/plugins/connection/ssh.py @@ -0,0 +1,1399 @@ +# Copyright (c) 2012, Michael DeHaan <michael.dehaan@gmail.com> +# Copyright 2015 Abhijit Menon-Sen <ams@2ndQuadrant.com> +# Copyright 2017 Toshio Kuratomi <tkuratomi@ansible.com> +# Copyright (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' + name: ssh + short_description: connect via SSH client binary + description: + - This connection plugin allows Ansible to communicate to the target machines through normal SSH command line. + - Ansible does not expose a channel to allow communication between the user and the SSH process to accept + a password manually to decrypt an SSH key when using this connection plugin (which is the default). The + use of C(ssh-agent) is highly recommended. + author: ansible (@core) + extends_documentation_fragment: + - connection_pipelining + version_added: historical + notes: + - Many options default to C(None) here but that only means we do not override the SSH tool's defaults and/or configuration. + For example, if you specify the port in this plugin it will override any C(Port) entry in your C(.ssh/config). + - The ssh CLI tool uses return code 255 as a 'connection error', this can conflict with commands/tools that + also return 255 as an error code and will look like an 'unreachable' condition or 'connection error' to this plugin. + options: + host: + description: Hostname/IP to connect to. + default: inventory_hostname + vars: + - name: inventory_hostname + - name: ansible_host + - name: ansible_ssh_host + - name: delegated_vars['ansible_host'] + - name: delegated_vars['ansible_ssh_host'] + host_key_checking: + description: Determines if SSH should check host keys. + default: True + type: boolean + ini: + - section: defaults + key: 'host_key_checking' + - section: ssh_connection + key: 'host_key_checking' + version_added: '2.5' + env: + - name: ANSIBLE_HOST_KEY_CHECKING + - name: ANSIBLE_SSH_HOST_KEY_CHECKING + version_added: '2.5' + vars: + - name: ansible_host_key_checking + version_added: '2.5' + - name: ansible_ssh_host_key_checking + version_added: '2.5' + password: + description: Authentication password for the C(remote_user). Can be supplied as CLI option. + vars: + - name: ansible_password + - name: ansible_ssh_pass + - name: ansible_ssh_password + sshpass_prompt: + description: + - Password prompt that sshpass should search for. Supported by sshpass 1.06 and up. + - Defaults to C(Enter PIN for) when pkcs11_provider is set. + default: '' + ini: + - section: 'ssh_connection' + key: 'sshpass_prompt' + env: + - name: ANSIBLE_SSHPASS_PROMPT + vars: + - name: ansible_sshpass_prompt + version_added: '2.10' + ssh_args: + description: Arguments to pass to all SSH CLI tools. + default: '-C -o ControlMaster=auto -o ControlPersist=60s' + ini: + - section: 'ssh_connection' + key: 'ssh_args' + env: + - name: ANSIBLE_SSH_ARGS + vars: + - name: ansible_ssh_args + version_added: '2.7' + ssh_common_args: + description: Common extra args for all SSH CLI tools. + ini: + - section: 'ssh_connection' + key: 'ssh_common_args' + version_added: '2.7' + env: + - name: ANSIBLE_SSH_COMMON_ARGS + version_added: '2.7' + vars: + - name: ansible_ssh_common_args + cli: + - name: ssh_common_args + default: '' + ssh_executable: + default: ssh + description: + - This defines the location of the SSH binary. It defaults to C(ssh) which will use the first SSH binary available in $PATH. + - This option is usually not required, it might be useful when access to system SSH is restricted, + or when using SSH wrappers to connect to remote hosts. + env: [{name: ANSIBLE_SSH_EXECUTABLE}] + ini: + - {key: ssh_executable, section: ssh_connection} + #const: ANSIBLE_SSH_EXECUTABLE + version_added: "2.2" + vars: + - name: ansible_ssh_executable + version_added: '2.7' + sftp_executable: + default: sftp + description: + - This defines the location of the sftp binary. It defaults to C(sftp) which will use the first binary available in $PATH. + env: [{name: ANSIBLE_SFTP_EXECUTABLE}] + ini: + - {key: sftp_executable, section: ssh_connection} + version_added: "2.6" + vars: + - name: ansible_sftp_executable + version_added: '2.7' + scp_executable: + default: scp + description: + - This defines the location of the scp binary. It defaults to C(scp) which will use the first binary available in $PATH. + env: [{name: ANSIBLE_SCP_EXECUTABLE}] + ini: + - {key: scp_executable, section: ssh_connection} + version_added: "2.6" + vars: + - name: ansible_scp_executable + version_added: '2.7' + scp_extra_args: + description: Extra exclusive to the C(scp) CLI + vars: + - name: ansible_scp_extra_args + env: + - name: ANSIBLE_SCP_EXTRA_ARGS + version_added: '2.7' + ini: + - key: scp_extra_args + section: ssh_connection + version_added: '2.7' + cli: + - name: scp_extra_args + default: '' + sftp_extra_args: + description: Extra exclusive to the C(sftp) CLI + vars: + - name: ansible_sftp_extra_args + env: + - name: ANSIBLE_SFTP_EXTRA_ARGS + version_added: '2.7' + ini: + - key: sftp_extra_args + section: ssh_connection + version_added: '2.7' + cli: + - name: sftp_extra_args + default: '' + ssh_extra_args: + description: Extra exclusive to the SSH CLI. + vars: + - name: ansible_ssh_extra_args + env: + - name: ANSIBLE_SSH_EXTRA_ARGS + version_added: '2.7' + ini: + - key: ssh_extra_args + section: ssh_connection + version_added: '2.7' + cli: + - name: ssh_extra_args + default: '' + reconnection_retries: + description: + - Number of attempts to connect. + - Ansible retries connections only if it gets an SSH error with a return code of 255. + - Any errors with return codes other than 255 indicate an issue with program execution. + default: 0 + type: integer + env: + - name: ANSIBLE_SSH_RETRIES + ini: + - section: connection + key: retries + - section: ssh_connection + key: retries + vars: + - name: ansible_ssh_retries + version_added: '2.7' + port: + description: Remote port to connect to. + type: int + ini: + - section: defaults + key: remote_port + env: + - name: ANSIBLE_REMOTE_PORT + vars: + - name: ansible_port + - name: ansible_ssh_port + keyword: + - name: port + remote_user: + description: + - User name with which to login to the remote server, normally set by the remote_user keyword. + - If no user is supplied, Ansible will let the SSH client binary choose the user as it normally. + ini: + - section: defaults + key: remote_user + env: + - name: ANSIBLE_REMOTE_USER + vars: + - name: ansible_user + - name: ansible_ssh_user + cli: + - name: user + keyword: + - name: remote_user + pipelining: + env: + - name: ANSIBLE_PIPELINING + - name: ANSIBLE_SSH_PIPELINING + ini: + - section: defaults + key: pipelining + - section: connection + key: pipelining + - section: ssh_connection + key: pipelining + vars: + - name: ansible_pipelining + - name: ansible_ssh_pipelining + + private_key_file: + description: + - Path to private key file to use for authentication. + ini: + - section: defaults + key: private_key_file + env: + - name: ANSIBLE_PRIVATE_KEY_FILE + vars: + - name: ansible_private_key_file + - name: ansible_ssh_private_key_file + cli: + - name: private_key_file + option: '--private-key' + + control_path: + description: + - This is the location to save SSH's ControlPath sockets, it uses SSH's variable substitution. + - Since 2.3, if null (default), ansible will generate a unique hash. Use ``%(directory)s`` to indicate where to use the control dir path setting. + - Before 2.3 it defaulted to ``control_path=%(directory)s/ansible-ssh-%%h-%%p-%%r``. + - Be aware that this setting is ignored if C(-o ControlPath) is set in ssh args. + env: + - name: ANSIBLE_SSH_CONTROL_PATH + ini: + - key: control_path + section: ssh_connection + vars: + - name: ansible_control_path + version_added: '2.7' + control_path_dir: + default: ~/.ansible/cp + description: + - This sets the directory to use for ssh control path if the control path setting is null. + - Also, provides the ``%(directory)s`` variable for the control path setting. + env: + - name: ANSIBLE_SSH_CONTROL_PATH_DIR + ini: + - section: ssh_connection + key: control_path_dir + vars: + - name: ansible_control_path_dir + version_added: '2.7' + sftp_batch_mode: + default: 'yes' + description: 'TODO: write it' + env: [{name: ANSIBLE_SFTP_BATCH_MODE}] + ini: + - {key: sftp_batch_mode, section: ssh_connection} + type: bool + vars: + - name: ansible_sftp_batch_mode + version_added: '2.7' + ssh_transfer_method: + description: + - "Preferred method to use when transferring files over ssh" + - Setting to 'smart' (default) will try them in order, until one succeeds or they all fail + - For OpenSSH >=9.0 you must add an additional option to enable scp (scp_extra_args="-O") + - Using 'piped' creates an ssh pipe with C(dd) on either side to copy the data + choices: ['sftp', 'scp', 'piped', 'smart'] + env: [{name: ANSIBLE_SSH_TRANSFER_METHOD}] + ini: + - {key: transfer_method, section: ssh_connection} + vars: + - name: ansible_ssh_transfer_method + version_added: '2.12' + scp_if_ssh: + deprecated: + why: In favor of the "ssh_transfer_method" option. + version: "2.17" + alternatives: ssh_transfer_method + default: smart + description: + - "Preferred method to use when transferring files over SSH." + - When set to I(smart), Ansible will try them until one succeeds or they all fail. + - If set to I(True), it will force 'scp', if I(False) it will use 'sftp'. + - For OpenSSH >=9.0 you must add an additional option to enable scp (scp_extra_args="-O") + - This setting will overridden by ssh_transfer_method if set. + env: [{name: ANSIBLE_SCP_IF_SSH}] + ini: + - {key: scp_if_ssh, section: ssh_connection} + vars: + - name: ansible_scp_if_ssh + version_added: '2.7' + use_tty: + version_added: '2.5' + default: 'yes' + description: add -tt to ssh commands to force tty allocation. + env: [{name: ANSIBLE_SSH_USETTY}] + ini: + - {key: usetty, section: ssh_connection} + type: bool + vars: + - name: ansible_ssh_use_tty + version_added: '2.7' + timeout: + default: 10 + description: + - This is the default amount of time we will wait while establishing an SSH connection. + - It also controls how long we can wait to access reading the connection once established (select on the socket). + env: + - name: ANSIBLE_TIMEOUT + - name: ANSIBLE_SSH_TIMEOUT + version_added: '2.11' + ini: + - key: timeout + section: defaults + - key: timeout + section: ssh_connection + version_added: '2.11' + vars: + - name: ansible_ssh_timeout + version_added: '2.11' + cli: + - name: timeout + type: integer + pkcs11_provider: + version_added: '2.12' + default: "" + description: + - "PKCS11 SmartCard provider such as opensc, example: /usr/local/lib/opensc-pkcs11.so" + - Requires sshpass version 1.06+, sshpass must support the -P option. + env: [{name: ANSIBLE_PKCS11_PROVIDER}] + ini: + - {key: pkcs11_provider, section: ssh_connection} + vars: + - name: ansible_ssh_pkcs11_provider +''' + +import errno +import fcntl +import hashlib +import os +import pty +import re +import shlex +import subprocess +import time + +from functools import wraps +from ansible.errors import ( + AnsibleAuthenticationFailure, + AnsibleConnectionFailure, + AnsibleError, + AnsibleFileNotFound, +) +from ansible.errors import AnsibleOptionsError +from ansible.module_utils.compat import selectors +from ansible.module_utils.six import PY3, text_type, binary_type +from ansible.module_utils._text import to_bytes, to_native, to_text +from ansible.module_utils.parsing.convert_bool import BOOLEANS, boolean +from ansible.plugins.connection import ConnectionBase, BUFSIZE +from ansible.plugins.shell.powershell import _parse_clixml +from ansible.utils.display import Display +from ansible.utils.path import unfrackpath, makedirs_safe + +display = Display() + +# error messages that indicate 255 return code is not from ssh itself. +b_NOT_SSH_ERRORS = (b'Traceback (most recent call last):', # Python-2.6 when there's an exception + # while invoking a script via -m + b'PHP Parse error:', # Php always returns with error + b'chmod: invalid mode', # chmod, but really only on AIX + b'chmod: A flag or octal number is not correct.', # chmod, other AIX + ) + +SSHPASS_AVAILABLE = None +SSH_DEBUG = re.compile(r'^debug\d+: .*') + + +class AnsibleControlPersistBrokenPipeError(AnsibleError): + ''' ControlPersist broken pipe ''' + pass + + +def _handle_error(remaining_retries, command, return_tuple, no_log, host, display=display): + + # sshpass errors + if command == b'sshpass': + # Error 5 is invalid/incorrect password. Raise an exception to prevent retries from locking the account. + if return_tuple[0] == 5: + msg = 'Invalid/incorrect username/password. Skipping remaining {0} retries to prevent account lockout:'.format(remaining_retries) + if remaining_retries <= 0: + msg = 'Invalid/incorrect password:' + if no_log: + msg = '{0} <error censored due to no log>'.format(msg) + else: + msg = '{0} {1}'.format(msg, to_native(return_tuple[2]).rstrip()) + raise AnsibleAuthenticationFailure(msg) + + # sshpass returns codes are 1-6. We handle 5 previously, so this catches other scenarios. + # No exception is raised, so the connection is retried - except when attempting to use + # sshpass_prompt with an sshpass that won't let us pass -P, in which case we fail loudly. + elif return_tuple[0] in [1, 2, 3, 4, 6]: + msg = 'sshpass error:' + if no_log: + msg = '{0} <error censored due to no log>'.format(msg) + else: + details = to_native(return_tuple[2]).rstrip() + if "sshpass: invalid option -- 'P'" in details: + details = 'Installed sshpass version does not support customized password prompts. ' \ + 'Upgrade sshpass to use sshpass_prompt, or otherwise switch to ssh keys.' + raise AnsibleError('{0} {1}'.format(msg, details)) + msg = '{0} {1}'.format(msg, details) + + if return_tuple[0] == 255: + SSH_ERROR = True + for signature in b_NOT_SSH_ERRORS: + # 1 == stout, 2 == stderr + if signature in return_tuple[1] or signature in return_tuple[2]: + SSH_ERROR = False + break + + if SSH_ERROR: + msg = "Failed to connect to the host via ssh:" + if no_log: + msg = '{0} <error censored due to no log>'.format(msg) + else: + msg = '{0} {1}'.format(msg, to_native(return_tuple[2]).rstrip()) + raise AnsibleConnectionFailure(msg) + + # For other errors, no exception is raised so the connection is retried and we only log the messages + if 1 <= return_tuple[0] <= 254: + msg = u"Failed to connect to the host via ssh:" + if no_log: + msg = u'{0} <error censored due to no log>'.format(msg) + else: + msg = u'{0} {1}'.format(msg, to_text(return_tuple[2]).rstrip()) + display.vvv(msg, host=host) + + +def _ssh_retry(func): + """ + Decorator to retry ssh/scp/sftp in the case of a connection failure + + Will retry if: + * an exception is caught + * ssh returns 255 + Will not retry if + * sshpass returns 5 (invalid password, to prevent account lockouts) + * remaining_tries is < 2 + * retries limit reached + """ + @wraps(func) + def wrapped(self, *args, **kwargs): + remaining_tries = int(self.get_option('reconnection_retries')) + 1 + cmd_summary = u"%s..." % to_text(args[0]) + conn_password = self.get_option('password') or self._play_context.password + for attempt in range(remaining_tries): + cmd = args[0] + if attempt != 0 and conn_password and isinstance(cmd, list): + # If this is a retry, the fd/pipe for sshpass is closed, and we need a new one + self.sshpass_pipe = os.pipe() + cmd[1] = b'-d' + to_bytes(self.sshpass_pipe[0], nonstring='simplerepr', errors='surrogate_or_strict') + + try: + try: + return_tuple = func(self, *args, **kwargs) + # TODO: this should come from task + if self._play_context.no_log: + display.vvv(u'rc=%s, stdout and stderr censored due to no log' % return_tuple[0], host=self.host) + else: + display.vvv(return_tuple, host=self.host) + # 0 = success + # 1-254 = remote command return code + # 255 could be a failure from the ssh command itself + except (AnsibleControlPersistBrokenPipeError): + # Retry one more time because of the ControlPersist broken pipe (see #16731) + cmd = args[0] + if conn_password and isinstance(cmd, list): + # This is a retry, so the fd/pipe for sshpass is closed, and we need a new one + self.sshpass_pipe = os.pipe() + cmd[1] = b'-d' + to_bytes(self.sshpass_pipe[0], nonstring='simplerepr', errors='surrogate_or_strict') + display.vvv(u"RETRYING BECAUSE OF CONTROLPERSIST BROKEN PIPE") + return_tuple = func(self, *args, **kwargs) + + remaining_retries = remaining_tries - attempt - 1 + _handle_error(remaining_retries, cmd[0], return_tuple, self._play_context.no_log, self.host) + + break + + # 5 = Invalid/incorrect password from sshpass + except AnsibleAuthenticationFailure: + # Raising this exception, which is subclassed from AnsibleConnectionFailure, prevents further retries + raise + + except (AnsibleConnectionFailure, Exception) as e: + + if attempt == remaining_tries - 1: + raise + else: + pause = 2 ** attempt - 1 + if pause > 30: + pause = 30 + + if isinstance(e, AnsibleConnectionFailure): + msg = u"ssh_retry: attempt: %d, ssh return code is 255. cmd (%s), pausing for %d seconds" % (attempt + 1, cmd_summary, pause) + else: + msg = (u"ssh_retry: attempt: %d, caught exception(%s) from cmd (%s), " + u"pausing for %d seconds" % (attempt + 1, to_text(e), cmd_summary, pause)) + + display.vv(msg, host=self.host) + + time.sleep(pause) + continue + + return return_tuple + return wrapped + + +class Connection(ConnectionBase): + ''' ssh based connections ''' + + transport = 'ssh' + has_pipelining = True + + def __init__(self, *args, **kwargs): + super(Connection, self).__init__(*args, **kwargs) + + # TODO: all should come from get_option(), but not might be set at this point yet + self.host = self._play_context.remote_addr + self.port = self._play_context.port + self.user = self._play_context.remote_user + self.control_path = None + self.control_path_dir = None + + # Windows operates differently from a POSIX connection/shell plugin, + # we need to set various properties to ensure SSH on Windows continues + # to work + if getattr(self._shell, "_IS_WINDOWS", False): + self.has_native_async = True + self.always_pipeline_modules = True + self.module_implementation_preferences = ('.ps1', '.exe', '') + self.allow_executable = False + + # The connection is created by running ssh/scp/sftp from the exec_command, + # put_file, and fetch_file methods, so we don't need to do any connection + # management here. + + def _connect(self): + return self + + @staticmethod + def _create_control_path(host, port, user, connection=None, pid=None): + '''Make a hash for the controlpath based on con attributes''' + pstring = '%s-%s-%s' % (host, port, user) + if connection: + pstring += '-%s' % connection + if pid: + pstring += '-%s' % to_text(pid) + m = hashlib.sha1() + m.update(to_bytes(pstring)) + digest = m.hexdigest() + cpath = '%(directory)s/' + digest[:10] + return cpath + + @staticmethod + def _sshpass_available(): + global SSHPASS_AVAILABLE + + # We test once if sshpass is available, and remember the result. It + # would be nice to use distutils.spawn.find_executable for this, but + # distutils isn't always available; shutils.which() is Python3-only. + + if SSHPASS_AVAILABLE is None: + try: + p = subprocess.Popen(["sshpass"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p.communicate() + SSHPASS_AVAILABLE = True + except OSError: + SSHPASS_AVAILABLE = False + + return SSHPASS_AVAILABLE + + @staticmethod + def _persistence_controls(b_command): + ''' + Takes a command array and scans it for ControlPersist and ControlPath + settings and returns two booleans indicating whether either was found. + This could be smarter, e.g. returning false if ControlPersist is 'no', + but for now we do it simple way. + ''' + + controlpersist = False + controlpath = False + + for b_arg in (a.lower() for a in b_command): + if b'controlpersist' in b_arg: + controlpersist = True + elif b'controlpath' in b_arg: + controlpath = True + + return controlpersist, controlpath + + def _add_args(self, b_command, b_args, explanation): + """ + Adds arguments to the ssh command and displays a caller-supplied explanation of why. + + :arg b_command: A list containing the command to add the new arguments to. + This list will be modified by this method. + :arg b_args: An iterable of new arguments to add. This iterable is used + more than once so it must be persistent (ie: a list is okay but a + StringIO would not) + :arg explanation: A text string containing explaining why the arguments + were added. It will be displayed with a high enough verbosity. + .. note:: This function does its work via side-effect. The b_command list has the new arguments appended. + """ + display.vvvvv(u'SSH: %s: (%s)' % (explanation, ')('.join(to_text(a) for a in b_args)), host=self.host) + b_command += b_args + + def _build_command(self, binary, subsystem, *other_args): + ''' + Takes a executable (ssh, scp, sftp or wrapper) and optional extra arguments and returns the remote command + wrapped in local ssh shell commands and ready for execution. + + :arg binary: actual executable to use to execute command. + :arg subsystem: type of executable provided, ssh/sftp/scp, needed because wrappers for ssh might have diff names. + :arg other_args: dict of, value pairs passed as arguments to the ssh binary + + ''' + + b_command = [] + conn_password = self.get_option('password') or self._play_context.password + + # + # First, the command to invoke + # + + # If we want to use password authentication, we have to set up a pipe to + # write the password to sshpass. + pkcs11_provider = self.get_option("pkcs11_provider") + if conn_password or pkcs11_provider: + if not self._sshpass_available(): + raise AnsibleError("to use the 'ssh' connection type with passwords or pkcs11_provider, you must install the sshpass program") + if not conn_password and pkcs11_provider: + raise AnsibleError("to use pkcs11_provider you must specify a password/pin") + + self.sshpass_pipe = os.pipe() + b_command += [b'sshpass', b'-d' + to_bytes(self.sshpass_pipe[0], nonstring='simplerepr', errors='surrogate_or_strict')] + + password_prompt = self.get_option('sshpass_prompt') + if not password_prompt and pkcs11_provider: + # Set default password prompt for pkcs11_provider to make it clear its a PIN + password_prompt = 'Enter PIN for ' + + if password_prompt: + b_command += [b'-P', to_bytes(password_prompt, errors='surrogate_or_strict')] + + b_command += [to_bytes(binary, errors='surrogate_or_strict')] + + # + # Next, additional arguments based on the configuration. + # + + # pkcs11 mode allows the use of Smartcards or Yubikey devices + if conn_password and pkcs11_provider: + self._add_args(b_command, + (b"-o", b"KbdInteractiveAuthentication=no", + b"-o", b"PreferredAuthentications=publickey", + b"-o", b"PasswordAuthentication=no", + b'-o', to_bytes(u'PKCS11Provider=%s' % pkcs11_provider)), + u'Enable pkcs11') + + # sftp batch mode allows us to correctly catch failed transfers, but can + # be disabled if the client side doesn't support the option. However, + # sftp batch mode does not prompt for passwords so it must be disabled + # if not using controlpersist and using sshpass + if subsystem == 'sftp' and self.get_option('sftp_batch_mode'): + if conn_password: + b_args = [b'-o', b'BatchMode=no'] + self._add_args(b_command, b_args, u'disable batch mode for sshpass') + b_command += [b'-b', b'-'] + + if display.verbosity > 3: + b_command.append(b'-vvv') + + # Next, we add ssh_args + ssh_args = self.get_option('ssh_args') + if ssh_args: + b_args = [to_bytes(a, errors='surrogate_or_strict') for a in + self._split_ssh_args(ssh_args)] + self._add_args(b_command, b_args, u"ansible.cfg set ssh_args") + + # Now we add various arguments that have their own specific settings defined in docs above. + if self.get_option('host_key_checking') is False: + b_args = (b"-o", b"StrictHostKeyChecking=no") + self._add_args(b_command, b_args, u"ANSIBLE_HOST_KEY_CHECKING/host_key_checking disabled") + + self.port = self.get_option('port') + if self.port is not None: + b_args = (b"-o", b"Port=" + to_bytes(self.port, nonstring='simplerepr', errors='surrogate_or_strict')) + self._add_args(b_command, b_args, u"ANSIBLE_REMOTE_PORT/remote_port/ansible_port set") + + key = self.get_option('private_key_file') + if key: + b_args = (b"-o", b'IdentityFile="' + to_bytes(os.path.expanduser(key), errors='surrogate_or_strict') + b'"') + self._add_args(b_command, b_args, u"ANSIBLE_PRIVATE_KEY_FILE/private_key_file/ansible_ssh_private_key_file set") + + if not conn_password: + self._add_args( + b_command, ( + b"-o", b"KbdInteractiveAuthentication=no", + b"-o", b"PreferredAuthentications=gssapi-with-mic,gssapi-keyex,hostbased,publickey", + b"-o", b"PasswordAuthentication=no" + ), + u"ansible_password/ansible_ssh_password not set" + ) + + self.user = self.get_option('remote_user') + if self.user: + self._add_args( + b_command, + (b"-o", b'User="%s"' % to_bytes(self.user, errors='surrogate_or_strict')), + u"ANSIBLE_REMOTE_USER/remote_user/ansible_user/user/-u set" + ) + + timeout = self.get_option('timeout') + self._add_args( + b_command, + (b"-o", b"ConnectTimeout=" + to_bytes(timeout, errors='surrogate_or_strict', nonstring='simplerepr')), + u"ANSIBLE_TIMEOUT/timeout set" + ) + + # Add in any common or binary-specific arguments from the PlayContext + # (i.e. inventory or task settings or overrides on the command line). + + for opt in (u'ssh_common_args', u'{0}_extra_args'.format(subsystem)): + attr = self.get_option(opt) + if attr is not None: + b_args = [to_bytes(a, errors='surrogate_or_strict') for a in self._split_ssh_args(attr)] + self._add_args(b_command, b_args, u"Set %s" % opt) + + # Check if ControlPersist is enabled and add a ControlPath if one hasn't + # already been set. + + controlpersist, controlpath = self._persistence_controls(b_command) + + if controlpersist: + self._persistent = True + + if not controlpath: + self.control_path_dir = self.get_option('control_path_dir') + cpdir = unfrackpath(self.control_path_dir) + b_cpdir = to_bytes(cpdir, errors='surrogate_or_strict') + + # The directory must exist and be writable. + makedirs_safe(b_cpdir, 0o700) + if not os.access(b_cpdir, os.W_OK): + raise AnsibleError("Cannot write to ControlPath %s" % to_native(cpdir)) + + self.control_path = self.get_option('control_path') + if not self.control_path: + self.control_path = self._create_control_path( + self.host, + self.port, + self.user + ) + b_args = (b"-o", b'ControlPath="%s"' % to_bytes(self.control_path % dict(directory=cpdir), errors='surrogate_or_strict')) + self._add_args(b_command, b_args, u"found only ControlPersist; added ControlPath") + + # Finally, we add any caller-supplied extras. + if other_args: + b_command += [to_bytes(a) for a in other_args] + + return b_command + + def _send_initial_data(self, fh, in_data, ssh_process): + ''' + Writes initial data to the stdin filehandle of the subprocess and closes + it. (The handle must be closed; otherwise, for example, "sftp -b -" will + just hang forever waiting for more commands.) + ''' + + display.debug(u'Sending initial data') + + try: + fh.write(to_bytes(in_data)) + fh.close() + except (OSError, IOError) as e: + # The ssh connection may have already terminated at this point, with a more useful error + # Only raise AnsibleConnectionFailure if the ssh process is still alive + time.sleep(0.001) + ssh_process.poll() + if getattr(ssh_process, 'returncode', None) is None: + raise AnsibleConnectionFailure( + 'Data could not be sent to remote host "%s". Make sure this host can be reached ' + 'over ssh: %s' % (self.host, to_native(e)), orig_exc=e + ) + + display.debug(u'Sent initial data (%d bytes)' % len(in_data)) + + # Used by _run() to kill processes on failures + @staticmethod + def _terminate_process(p): + """ Terminate a process, ignoring errors """ + try: + p.terminate() + except (OSError, IOError): + pass + + # This is separate from _run() because we need to do the same thing for stdout + # and stderr. + def _examine_output(self, source, state, b_chunk, sudoable): + ''' + Takes a string, extracts complete lines from it, tests to see if they + are a prompt, error message, etc., and sets appropriate flags in self. + Prompt and success lines are removed. + + Returns the processed (i.e. possibly-edited) output and the unprocessed + remainder (to be processed with the next chunk) as strings. + ''' + + output = [] + for b_line in b_chunk.splitlines(True): + display_line = to_text(b_line).rstrip('\r\n') + suppress_output = False + + # display.debug("Examining line (source=%s, state=%s): '%s'" % (source, state, display_line)) + if SSH_DEBUG.match(display_line): + # skip lines from ssh debug output to avoid false matches + pass + elif self.become.expect_prompt() and self.become.check_password_prompt(b_line): + display.debug(u"become_prompt: (source=%s, state=%s): '%s'" % (source, state, display_line)) + self._flags['become_prompt'] = True + suppress_output = True + elif self.become.success and self.become.check_success(b_line): + display.debug(u"become_success: (source=%s, state=%s): '%s'" % (source, state, display_line)) + self._flags['become_success'] = True + suppress_output = True + elif sudoable and self.become.check_incorrect_password(b_line): + display.debug(u"become_error: (source=%s, state=%s): '%s'" % (source, state, display_line)) + self._flags['become_error'] = True + elif sudoable and self.become.check_missing_password(b_line): + display.debug(u"become_nopasswd_error: (source=%s, state=%s): '%s'" % (source, state, display_line)) + self._flags['become_nopasswd_error'] = True + + if not suppress_output: + output.append(b_line) + + # The chunk we read was most likely a series of complete lines, but just + # in case the last line was incomplete (and not a prompt, which we would + # have removed from the output), we retain it to be processed with the + # next chunk. + + remainder = b'' + if output and not output[-1].endswith(b'\n'): + remainder = output[-1] + output = output[:-1] + + return b''.join(output), remainder + + def _bare_run(self, cmd, in_data, sudoable=True, checkrc=True): + ''' + Starts the command and communicates with it until it ends. + ''' + + # We don't use _shell.quote as this is run on the controller and independent from the shell plugin chosen + display_cmd = u' '.join(shlex.quote(to_text(c)) for c in cmd) + display.vvv(u'SSH: EXEC {0}'.format(display_cmd), host=self.host) + + # Start the given command. If we don't need to pipeline data, we can try + # to use a pseudo-tty (ssh will have been invoked with -tt). If we are + # pipelining data, or can't create a pty, we fall back to using plain + # old pipes. + + p = None + + if isinstance(cmd, (text_type, binary_type)): + cmd = to_bytes(cmd) + else: + cmd = list(map(to_bytes, cmd)) + + conn_password = self.get_option('password') or self._play_context.password + + if not in_data: + try: + # Make sure stdin is a proper pty to avoid tcgetattr errors + master, slave = pty.openpty() + if PY3 and conn_password: + # pylint: disable=unexpected-keyword-arg + p = subprocess.Popen(cmd, stdin=slave, stdout=subprocess.PIPE, stderr=subprocess.PIPE, pass_fds=self.sshpass_pipe) + else: + p = subprocess.Popen(cmd, stdin=slave, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdin = os.fdopen(master, 'wb', 0) + os.close(slave) + except (OSError, IOError): + p = None + + if not p: + try: + if PY3 and conn_password: + # pylint: disable=unexpected-keyword-arg + p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, pass_fds=self.sshpass_pipe) + else: + p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdin = p.stdin + except (OSError, IOError) as e: + raise AnsibleError('Unable to execute ssh command line on a controller due to: %s' % to_native(e)) + + # If we are using SSH password authentication, write the password into + # the pipe we opened in _build_command. + + if conn_password: + os.close(self.sshpass_pipe[0]) + try: + os.write(self.sshpass_pipe[1], to_bytes(conn_password) + b'\n') + except OSError as e: + # Ignore broken pipe errors if the sshpass process has exited. + if e.errno != errno.EPIPE or p.poll() is None: + raise + os.close(self.sshpass_pipe[1]) + + # + # SSH state machine + # + + # Now we read and accumulate output from the running process until it + # exits. Depending on the circumstances, we may also need to write an + # escalation password and/or pipelined input to the process. + + states = [ + 'awaiting_prompt', 'awaiting_escalation', 'ready_to_send', 'awaiting_exit' + ] + + # Are we requesting privilege escalation? Right now, we may be invoked + # to execute sftp/scp with sudoable=True, but we can request escalation + # only when using ssh. Otherwise we can send initial data straightaway. + + state = states.index('ready_to_send') + if to_bytes(self.get_option('ssh_executable')) in cmd and sudoable: + prompt = getattr(self.become, 'prompt', None) + if prompt: + # We're requesting escalation with a password, so we have to + # wait for a password prompt. + state = states.index('awaiting_prompt') + display.debug(u'Initial state: %s: %s' % (states[state], to_text(prompt))) + elif self.become and self.become.success: + # We're requesting escalation without a password, so we have to + # detect success/failure before sending any initial data. + state = states.index('awaiting_escalation') + display.debug(u'Initial state: %s: %s' % (states[state], to_text(self.become.success))) + + # We store accumulated stdout and stderr output from the process here, + # but strip any privilege escalation prompt/confirmation lines first. + # Output is accumulated into tmp_*, complete lines are extracted into + # an array, then checked and removed or copied to stdout or stderr. We + # set any flags based on examining the output in self._flags. + + b_stdout = b_stderr = b'' + b_tmp_stdout = b_tmp_stderr = b'' + + self._flags = dict( + become_prompt=False, become_success=False, + become_error=False, become_nopasswd_error=False + ) + + # select timeout should be longer than the connect timeout, otherwise + # they will race each other when we can't connect, and the connect + # timeout usually fails + timeout = 2 + self.get_option('timeout') + for fd in (p.stdout, p.stderr): + fcntl.fcntl(fd, fcntl.F_SETFL, fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK) + + # TODO: bcoca would like to use SelectSelector() when open + # select is faster when filehandles is low and we only ever handle 1. + selector = selectors.DefaultSelector() + selector.register(p.stdout, selectors.EVENT_READ) + selector.register(p.stderr, selectors.EVENT_READ) + + # If we can send initial data without waiting for anything, we do so + # before we start polling + if states[state] == 'ready_to_send' and in_data: + self._send_initial_data(stdin, in_data, p) + state += 1 + + try: + while True: + poll = p.poll() + events = selector.select(timeout) + + # We pay attention to timeouts only while negotiating a prompt. + + if not events: + # We timed out + if state <= states.index('awaiting_escalation'): + # If the process has already exited, then it's not really a + # timeout; we'll let the normal error handling deal with it. + if poll is not None: + break + self._terminate_process(p) + raise AnsibleError('Timeout (%ds) waiting for privilege escalation prompt: %s' % (timeout, to_native(b_stdout))) + + # Read whatever output is available on stdout and stderr, and stop + # listening to the pipe if it's been closed. + + for key, event in events: + if key.fileobj == p.stdout: + b_chunk = p.stdout.read() + if b_chunk == b'': + # stdout has been closed, stop watching it + selector.unregister(p.stdout) + # When ssh has ControlMaster (+ControlPath/Persist) enabled, the + # first connection goes into the background and we never see EOF + # on stderr. If we see EOF on stdout, lower the select timeout + # to reduce the time wasted selecting on stderr if we observe + # that the process has not yet existed after this EOF. Otherwise + # we may spend a long timeout period waiting for an EOF that is + # not going to arrive until the persisted connection closes. + timeout = 1 + b_tmp_stdout += b_chunk + display.debug(u"stdout chunk (state=%s):\n>>>%s<<<\n" % (state, to_text(b_chunk))) + elif key.fileobj == p.stderr: + b_chunk = p.stderr.read() + if b_chunk == b'': + # stderr has been closed, stop watching it + selector.unregister(p.stderr) + b_tmp_stderr += b_chunk + display.debug("stderr chunk (state=%s):\n>>>%s<<<\n" % (state, to_text(b_chunk))) + + # We examine the output line-by-line until we have negotiated any + # privilege escalation prompt and subsequent success/error message. + # Afterwards, we can accumulate output without looking at it. + + if state < states.index('ready_to_send'): + if b_tmp_stdout: + b_output, b_unprocessed = self._examine_output('stdout', states[state], b_tmp_stdout, sudoable) + b_stdout += b_output + b_tmp_stdout = b_unprocessed + + if b_tmp_stderr: + b_output, b_unprocessed = self._examine_output('stderr', states[state], b_tmp_stderr, sudoable) + b_stderr += b_output + b_tmp_stderr = b_unprocessed + else: + b_stdout += b_tmp_stdout + b_stderr += b_tmp_stderr + b_tmp_stdout = b_tmp_stderr = b'' + + # If we see a privilege escalation prompt, we send the password. + # (If we're expecting a prompt but the escalation succeeds, we + # didn't need the password and can carry on regardless.) + + if states[state] == 'awaiting_prompt': + if self._flags['become_prompt']: + display.debug(u'Sending become_password in response to prompt') + become_pass = self.become.get_option('become_pass', playcontext=self._play_context) + stdin.write(to_bytes(become_pass, errors='surrogate_or_strict') + b'\n') + # On python3 stdin is a BufferedWriter, and we don't have a guarantee + # that the write will happen without a flush + stdin.flush() + self._flags['become_prompt'] = False + state += 1 + elif self._flags['become_success']: + state += 1 + + # We've requested escalation (with or without a password), now we + # wait for an error message or a successful escalation. + + if states[state] == 'awaiting_escalation': + if self._flags['become_success']: + display.vvv(u'Escalation succeeded') + self._flags['become_success'] = False + state += 1 + elif self._flags['become_error']: + display.vvv(u'Escalation failed') + self._terminate_process(p) + self._flags['become_error'] = False + raise AnsibleError('Incorrect %s password' % self.become.name) + elif self._flags['become_nopasswd_error']: + display.vvv(u'Escalation requires password') + self._terminate_process(p) + self._flags['become_nopasswd_error'] = False + raise AnsibleError('Missing %s password' % self.become.name) + elif self._flags['become_prompt']: + # This shouldn't happen, because we should see the "Sorry, + # try again" message first. + display.vvv(u'Escalation prompt repeated') + self._terminate_process(p) + self._flags['become_prompt'] = False + raise AnsibleError('Incorrect %s password' % self.become.name) + + # Once we're sure that the privilege escalation prompt, if any, has + # been dealt with, we can send any initial data and start waiting + # for output. + + if states[state] == 'ready_to_send': + if in_data: + self._send_initial_data(stdin, in_data, p) + state += 1 + + # Now we're awaiting_exit: has the child process exited? If it has, + # and we've read all available output from it, we're done. + + if poll is not None: + if not selector.get_map() or not events: + break + # We should not see further writes to the stdout/stderr file + # descriptors after the process has closed, set the select + # timeout to gather any last writes we may have missed. + timeout = 0 + continue + + # If the process has not yet exited, but we've already read EOF from + # its stdout and stderr (and thus no longer watching any file + # descriptors), we can just wait for it to exit. + + elif not selector.get_map(): + p.wait() + break + + # Otherwise there may still be outstanding data to read. + finally: + selector.close() + # close stdin, stdout, and stderr after process is terminated and + # stdout/stderr are read completely (see also issues #848, #64768). + stdin.close() + p.stdout.close() + p.stderr.close() + + if self.get_option('host_key_checking'): + if cmd[0] == b"sshpass" and p.returncode == 6: + raise AnsibleError('Using a SSH password instead of a key is not possible because Host Key checking is enabled and sshpass does not support ' + 'this. Please add this host\'s fingerprint to your known_hosts file to manage this host.') + + controlpersisterror = b'Bad configuration option: ControlPersist' in b_stderr or b'unknown configuration option: ControlPersist' in b_stderr + if p.returncode != 0 and controlpersisterror: + raise AnsibleError('using -c ssh on certain older ssh versions may not support ControlPersist, set ANSIBLE_SSH_ARGS="" ' + '(or ssh_args in [ssh_connection] section of the config file) before running again') + + # If we find a broken pipe because of ControlPersist timeout expiring (see #16731), + # we raise a special exception so that we can retry a connection. + controlpersist_broken_pipe = b'mux_client_hello_exchange: write packet: Broken pipe' in b_stderr + if p.returncode == 255: + + additional = to_native(b_stderr) + if controlpersist_broken_pipe: + raise AnsibleControlPersistBrokenPipeError('Data could not be sent because of ControlPersist broken pipe: %s' % additional) + + elif in_data and checkrc: + raise AnsibleConnectionFailure('Data could not be sent to remote host "%s". Make sure this host can be reached over ssh: %s' + % (self.host, additional)) + + return (p.returncode, b_stdout, b_stderr) + + @_ssh_retry + def _run(self, cmd, in_data, sudoable=True, checkrc=True): + """Wrapper around _bare_run that retries the connection + """ + return self._bare_run(cmd, in_data, sudoable=sudoable, checkrc=checkrc) + + @_ssh_retry + def _file_transport_command(self, in_path, out_path, sftp_action): + # scp and sftp require square brackets for IPv6 addresses, but + # accept them for hostnames and IPv4 addresses too. + host = '[%s]' % self.host + + smart_methods = ['sftp', 'scp', 'piped'] + + # Windows does not support dd so we cannot use the piped method + if getattr(self._shell, "_IS_WINDOWS", False): + smart_methods.remove('piped') + + # Transfer methods to try + methods = [] + + # Use the transfer_method option if set, otherwise use scp_if_ssh + ssh_transfer_method = self.get_option('ssh_transfer_method') + scp_if_ssh = self.get_option('scp_if_ssh') + if ssh_transfer_method is None and scp_if_ssh == 'smart': + ssh_transfer_method = 'smart' + + if ssh_transfer_method is not None: + if ssh_transfer_method == 'smart': + methods = smart_methods + else: + methods = [ssh_transfer_method] + else: + # since this can be a non-bool now, we need to handle it correctly + if not isinstance(scp_if_ssh, bool): + scp_if_ssh = scp_if_ssh.lower() + if scp_if_ssh in BOOLEANS: + scp_if_ssh = boolean(scp_if_ssh, strict=False) + elif scp_if_ssh != 'smart': + raise AnsibleOptionsError('scp_if_ssh needs to be one of [smart|True|False]') + if scp_if_ssh == 'smart': + methods = smart_methods + elif scp_if_ssh is True: + methods = ['scp'] + else: + methods = ['sftp'] + + for method in methods: + returncode = stdout = stderr = None + if method == 'sftp': + cmd = self._build_command(self.get_option('sftp_executable'), 'sftp', to_bytes(host)) + in_data = u"{0} {1} {2}\n".format(sftp_action, shlex.quote(in_path), shlex.quote(out_path)) + in_data = to_bytes(in_data, nonstring='passthru') + (returncode, stdout, stderr) = self._bare_run(cmd, in_data, checkrc=False) + elif method == 'scp': + scp = self.get_option('scp_executable') + + if sftp_action == 'get': + cmd = self._build_command(scp, 'scp', u'{0}:{1}'.format(host, self._shell.quote(in_path)), out_path) + else: + cmd = self._build_command(scp, 'scp', in_path, u'{0}:{1}'.format(host, self._shell.quote(out_path))) + in_data = None + (returncode, stdout, stderr) = self._bare_run(cmd, in_data, checkrc=False) + elif method == 'piped': + if sftp_action == 'get': + # we pass sudoable=False to disable pty allocation, which + # would end up mixing stdout/stderr and screwing with newlines + (returncode, stdout, stderr) = self.exec_command('dd if=%s bs=%s' % (in_path, BUFSIZE), sudoable=False) + with open(to_bytes(out_path, errors='surrogate_or_strict'), 'wb+') as out_file: + out_file.write(stdout) + else: + with open(to_bytes(in_path, errors='surrogate_or_strict'), 'rb') as f: + in_data = to_bytes(f.read(), nonstring='passthru') + if not in_data: + count = ' count=0' + else: + count = '' + (returncode, stdout, stderr) = self.exec_command('dd of=%s bs=%s%s' % (out_path, BUFSIZE, count), in_data=in_data, sudoable=False) + + # Check the return code and rollover to next method if failed + if returncode == 0: + return (returncode, stdout, stderr) + else: + # If not in smart mode, the data will be printed by the raise below + if len(methods) > 1: + display.warning(u'%s transfer mechanism failed on %s. Use ANSIBLE_DEBUG=1 to see detailed information' % (method, host)) + display.debug(u'%s' % to_text(stdout)) + display.debug(u'%s' % to_text(stderr)) + + if returncode == 255: + raise AnsibleConnectionFailure("Failed to connect to the host via %s: %s" % (method, to_native(stderr))) + else: + raise AnsibleError("failed to transfer file to %s %s:\n%s\n%s" % + (to_native(in_path), to_native(out_path), to_native(stdout), to_native(stderr))) + + def _escape_win_path(self, path): + """ converts a Windows path to one that's supported by SFTP and SCP """ + # If using a root path then we need to start with / + prefix = "" + if re.match(r'^\w{1}:', path): + prefix = "/" + + # Convert all '\' to '/' + return "%s%s" % (prefix, path.replace("\\", "/")) + + # + # Main public methods + # + def exec_command(self, cmd, in_data=None, sudoable=True): + ''' run a command on the remote host ''' + + super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable) + + self.host = self.get_option('host') or self._play_context.remote_addr + + display.vvv(u"ESTABLISH SSH CONNECTION FOR USER: {0}".format(self.user), host=self.host) + + if getattr(self._shell, "_IS_WINDOWS", False): + # Become method 'runas' is done in the wrapper that is executed, + # need to disable sudoable so the bare_run is not waiting for a + # prompt that will not occur + sudoable = False + + # Make sure our first command is to set the console encoding to + # utf-8, this must be done via chcp to get utf-8 (65001) + cmd_parts = ["chcp.com", "65001", self._shell._SHELL_REDIRECT_ALLNULL, self._shell._SHELL_AND] + cmd_parts.extend(self._shell._encode_script(cmd, as_list=True, strict_mode=False, preserve_rc=False)) + cmd = ' '.join(cmd_parts) + + # we can only use tty when we are not pipelining the modules. piping + # data into /usr/bin/python inside a tty automatically invokes the + # python interactive-mode but the modules are not compatible with the + # interactive-mode ("unexpected indent" mainly because of empty lines) + + ssh_executable = self.get_option('ssh_executable') + + # -tt can cause various issues in some environments so allow the user + # to disable it as a troubleshooting method. + use_tty = self.get_option('use_tty') + + if not in_data and sudoable and use_tty: + args = ('-tt', self.host, cmd) + else: + args = (self.host, cmd) + + cmd = self._build_command(ssh_executable, 'ssh', *args) + (returncode, stdout, stderr) = self._run(cmd, in_data, sudoable=sudoable) + + # When running on Windows, stderr may contain CLIXML encoded output + if getattr(self._shell, "_IS_WINDOWS", False) and stderr.startswith(b"#< CLIXML"): + stderr = _parse_clixml(stderr) + + return (returncode, stdout, stderr) + + def put_file(self, in_path, out_path): + ''' transfer a file from local to remote ''' + + super(Connection, self).put_file(in_path, out_path) + + self.host = self.get_option('host') or self._play_context.remote_addr + + display.vvv(u"PUT {0} TO {1}".format(in_path, out_path), host=self.host) + if not os.path.exists(to_bytes(in_path, errors='surrogate_or_strict')): + raise AnsibleFileNotFound("file or module does not exist: {0}".format(to_native(in_path))) + + if getattr(self._shell, "_IS_WINDOWS", False): + out_path = self._escape_win_path(out_path) + + return self._file_transport_command(in_path, out_path, 'put') + + def fetch_file(self, in_path, out_path): + ''' fetch a file from remote to local ''' + + super(Connection, self).fetch_file(in_path, out_path) + + self.host = self.get_option('host') or self._play_context.remote_addr + + display.vvv(u"FETCH {0} TO {1}".format(in_path, out_path), host=self.host) + + # need to add / if path is rooted + if getattr(self._shell, "_IS_WINDOWS", False): + in_path = self._escape_win_path(in_path) + + return self._file_transport_command(in_path, out_path, 'get') + + def reset(self): + + run_reset = False + self.host = self.get_option('host') or self._play_context.remote_addr + + # If we have a persistent ssh connection (ControlPersist), we can ask it to stop listening. + # only run the reset if the ControlPath already exists or if it isn't configured and ControlPersist is set + # 'check' will determine this. + cmd = self._build_command(self.get_option('ssh_executable'), 'ssh', '-O', 'check', self.host) + display.vvv(u'sending connection check: %s' % to_text(cmd)) + p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + status_code = p.wait() + if status_code != 0: + display.vvv(u"No connection to reset: %s" % to_text(stderr)) + else: + run_reset = True + + if run_reset: + cmd = self._build_command(self.get_option('ssh_executable'), 'ssh', '-O', 'stop', self.host) + display.vvv(u'sending connection stop: %s' % to_text(cmd)) + p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + status_code = p.wait() + if status_code != 0: + display.warning(u"Failed to reset connection:%s" % to_text(stderr)) + + self.close() + + def close(self): + self._connected = False diff --git a/lib/ansible/plugins/connection/winrm.py b/lib/ansible/plugins/connection/winrm.py new file mode 100644 index 0000000..13c80ec --- /dev/null +++ b/lib/ansible/plugins/connection/winrm.py @@ -0,0 +1,755 @@ +# (c) 2014, Chris Church <chris@ninemoreminutes.com> +# Copyright (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = """ + author: Ansible Core Team + name: winrm + short_description: Run tasks over Microsoft's WinRM + description: + - Run commands or put/fetch on a target via WinRM + - This plugin allows extra arguments to be passed that are supported by the protocol but not explicitly defined here. + They should take the form of variables declared with the following pattern C(ansible_winrm_<option>). + version_added: "2.0" + extends_documentation_fragment: + - connection_pipelining + requirements: + - pywinrm (python library) + options: + # figure out more elegant 'delegation' + remote_addr: + description: + - Address of the windows machine + default: inventory_hostname + vars: + - name: inventory_hostname + - name: ansible_host + - name: ansible_winrm_host + type: str + remote_user: + description: + - The user to log in as to the Windows machine + vars: + - name: ansible_user + - name: ansible_winrm_user + keyword: + - name: remote_user + type: str + remote_password: + description: Authentication password for the C(remote_user). Can be supplied as CLI option. + vars: + - name: ansible_password + - name: ansible_winrm_pass + - name: ansible_winrm_password + type: str + aliases: + - password # Needed for --ask-pass to come through on delegation + port: + description: + - port for winrm to connect on remote target + - The default is the https (5986) port, if using http it should be 5985 + vars: + - name: ansible_port + - name: ansible_winrm_port + default: 5986 + keyword: + - name: port + type: integer + scheme: + description: + - URI scheme to use + - If not set, then will default to C(https) or C(http) if I(port) is + C(5985). + choices: [http, https] + vars: + - name: ansible_winrm_scheme + type: str + path: + description: URI path to connect to + default: '/wsman' + vars: + - name: ansible_winrm_path + type: str + transport: + description: + - List of winrm transports to attempt to use (ssl, plaintext, kerberos, etc) + - If None (the default) the plugin will try to automatically guess the correct list + - The choices available depend on your version of pywinrm + type: list + elements: string + vars: + - name: ansible_winrm_transport + kerberos_command: + description: kerberos command to use to request a authentication ticket + default: kinit + vars: + - name: ansible_winrm_kinit_cmd + type: str + kinit_args: + description: + - Extra arguments to pass to C(kinit) when getting the Kerberos authentication ticket. + - By default no extra arguments are passed into C(kinit) unless I(ansible_winrm_kerberos_delegation) is also + set. In that case C(-f) is added to the C(kinit) args so a forwardable ticket is retrieved. + - If set, the args will overwrite any existing defaults for C(kinit), including C(-f) for a delegated ticket. + type: str + vars: + - name: ansible_winrm_kinit_args + version_added: '2.11' + kinit_env_vars: + description: + - A list of environment variables to pass through to C(kinit) when getting the Kerberos authentication ticket. + - By default no environment variables are passed through and C(kinit) is run with a blank slate. + - The environment variable C(KRB5CCNAME) cannot be specified here as it's used to store the temp Kerberos + ticket used by WinRM. + type: list + elements: str + default: [] + ini: + - section: winrm + key: kinit_env_vars + vars: + - name: ansible_winrm_kinit_env_vars + version_added: '2.12' + kerberos_mode: + description: + - kerberos usage mode. + - The managed option means Ansible will obtain kerberos ticket. + - While the manual one means a ticket must already have been obtained by the user. + - If having issues with Ansible freezing when trying to obtain the + Kerberos ticket, you can either set this to C(manual) and obtain + it outside Ansible or install C(pexpect) through pip and try + again. + choices: [managed, manual] + vars: + - name: ansible_winrm_kinit_mode + type: str + connection_timeout: + description: + - Sets the operation and read timeout settings for the WinRM + connection. + - Corresponds to the C(operation_timeout_sec) and + C(read_timeout_sec) args in pywinrm so avoid setting these vars + with this one. + - The default value is whatever is set in the installed version of + pywinrm. + vars: + - name: ansible_winrm_connection_timeout + type: int +""" + +import base64 +import logging +import os +import re +import traceback +import json +import tempfile +import shlex +import subprocess + +from inspect import getfullargspec +from urllib.parse import urlunsplit + +HAVE_KERBEROS = False +try: + import kerberos + HAVE_KERBEROS = True +except ImportError: + pass + +from ansible import constants as C +from ansible.errors import AnsibleError, AnsibleConnectionFailure +from ansible.errors import AnsibleFileNotFound +from ansible.module_utils.json_utils import _filter_non_json_lines +from ansible.module_utils.parsing.convert_bool import boolean +from ansible.module_utils._text import to_bytes, to_native, to_text +from ansible.module_utils.six import binary_type +from ansible.plugins.connection import ConnectionBase +from ansible.plugins.shell.powershell import _parse_clixml +from ansible.utils.hashing import secure_hash +from ansible.utils.display import Display + + +try: + import winrm + from winrm import Response + from winrm.protocol import Protocol + import requests.exceptions + HAS_WINRM = True + WINRM_IMPORT_ERR = None +except ImportError as e: + HAS_WINRM = False + WINRM_IMPORT_ERR = e + +try: + import xmltodict + HAS_XMLTODICT = True + XMLTODICT_IMPORT_ERR = None +except ImportError as e: + HAS_XMLTODICT = False + XMLTODICT_IMPORT_ERR = e + +HAS_PEXPECT = False +try: + import pexpect + # echo was added in pexpect 3.3+ which is newer than the RHEL package + # we can only use pexpect for kerb auth if echo is a valid kwarg + # https://github.com/ansible/ansible/issues/43462 + if hasattr(pexpect, 'spawn'): + argspec = getfullargspec(pexpect.spawn.__init__) + if 'echo' in argspec.args: + HAS_PEXPECT = True +except ImportError as e: + pass + +# used to try and parse the hostname and detect if IPv6 is being used +try: + import ipaddress + HAS_IPADDRESS = True +except ImportError: + HAS_IPADDRESS = False + +display = Display() + + +class Connection(ConnectionBase): + '''WinRM connections over HTTP/HTTPS.''' + + transport = 'winrm' + module_implementation_preferences = ('.ps1', '.exe', '') + allow_executable = False + has_pipelining = True + allow_extras = True + + def __init__(self, *args, **kwargs): + + self.always_pipeline_modules = True + self.has_native_async = True + + self.protocol = None + self.shell_id = None + self.delegate = None + self._shell_type = 'powershell' + + super(Connection, self).__init__(*args, **kwargs) + + if not C.DEFAULT_DEBUG: + logging.getLogger('requests_credssp').setLevel(logging.INFO) + logging.getLogger('requests_kerberos').setLevel(logging.INFO) + logging.getLogger('urllib3').setLevel(logging.INFO) + + def _build_winrm_kwargs(self): + # this used to be in set_options, as win_reboot needs to be able to + # override the conn timeout, we need to be able to build the args + # after setting individual options. This is called by _connect before + # starting the WinRM connection + self._winrm_host = self.get_option('remote_addr') + self._winrm_user = self.get_option('remote_user') + self._winrm_pass = self.get_option('remote_password') + + self._winrm_port = self.get_option('port') + + self._winrm_scheme = self.get_option('scheme') + # old behaviour, scheme should default to http if not set and the port + # is 5985 otherwise https + if self._winrm_scheme is None: + self._winrm_scheme = 'http' if self._winrm_port == 5985 else 'https' + + self._winrm_path = self.get_option('path') + self._kinit_cmd = self.get_option('kerberos_command') + self._winrm_transport = self.get_option('transport') + self._winrm_connection_timeout = self.get_option('connection_timeout') + + if hasattr(winrm, 'FEATURE_SUPPORTED_AUTHTYPES'): + self._winrm_supported_authtypes = set(winrm.FEATURE_SUPPORTED_AUTHTYPES) + else: + # for legacy versions of pywinrm, use the values we know are supported + self._winrm_supported_authtypes = set(['plaintext', 'ssl', 'kerberos']) + + # calculate transport if needed + if self._winrm_transport is None or self._winrm_transport[0] is None: + # TODO: figure out what we want to do with auto-transport selection in the face of NTLM/Kerb/CredSSP/Cert/Basic + transport_selector = ['ssl'] if self._winrm_scheme == 'https' else ['plaintext'] + + if HAVE_KERBEROS and ((self._winrm_user and '@' in self._winrm_user)): + self._winrm_transport = ['kerberos'] + transport_selector + else: + self._winrm_transport = transport_selector + + unsupported_transports = set(self._winrm_transport).difference(self._winrm_supported_authtypes) + + if unsupported_transports: + raise AnsibleError('The installed version of WinRM does not support transport(s) %s' % + to_native(list(unsupported_transports), nonstring='simplerepr')) + + # if kerberos is among our transports and there's a password specified, we're managing the tickets + kinit_mode = self.get_option('kerberos_mode') + if kinit_mode is None: + # HACK: ideally, remove multi-transport stuff + self._kerb_managed = "kerberos" in self._winrm_transport and (self._winrm_pass is not None and self._winrm_pass != "") + elif kinit_mode == "managed": + self._kerb_managed = True + elif kinit_mode == "manual": + self._kerb_managed = False + + # arg names we're going passing directly + internal_kwarg_mask = {'self', 'endpoint', 'transport', 'username', 'password', 'scheme', 'path', 'kinit_mode', 'kinit_cmd'} + + self._winrm_kwargs = dict(username=self._winrm_user, password=self._winrm_pass) + argspec = getfullargspec(Protocol.__init__) + supported_winrm_args = set(argspec.args) + supported_winrm_args.update(internal_kwarg_mask) + passed_winrm_args = {v.replace('ansible_winrm_', '') for v in self.get_option('_extras')} + unsupported_args = passed_winrm_args.difference(supported_winrm_args) + + # warn for kwargs unsupported by the installed version of pywinrm + for arg in unsupported_args: + display.warning("ansible_winrm_{0} unsupported by pywinrm (is an up-to-date version of pywinrm installed?)".format(arg)) + + # pass through matching extras, excluding the list we want to treat specially + for arg in passed_winrm_args.difference(internal_kwarg_mask).intersection(supported_winrm_args): + self._winrm_kwargs[arg] = self.get_option('_extras')['ansible_winrm_%s' % arg] + + # Until pykerberos has enough goodies to implement a rudimentary kinit/klist, simplest way is to let each connection + # auth itself with a private CCACHE. + def _kerb_auth(self, principal, password): + if password is None: + password = "" + + self._kerb_ccache = tempfile.NamedTemporaryFile() + display.vvvvv("creating Kerberos CC at %s" % self._kerb_ccache.name) + krb5ccname = "FILE:%s" % self._kerb_ccache.name + os.environ["KRB5CCNAME"] = krb5ccname + krb5env = dict(PATH=os.environ["PATH"], KRB5CCNAME=krb5ccname) + + # Add any explicit environment vars into the krb5env block + kinit_env_vars = self.get_option('kinit_env_vars') + for var in kinit_env_vars: + if var not in krb5env and var in os.environ: + krb5env[var] = os.environ[var] + + # Stores various flags to call with kinit, these could be explicit args set by 'ansible_winrm_kinit_args' OR + # '-f' if kerberos delegation is requested (ansible_winrm_kerberos_delegation). + kinit_cmdline = [self._kinit_cmd] + kinit_args = self.get_option('kinit_args') + if kinit_args: + kinit_args = [to_text(a) for a in shlex.split(kinit_args) if a.strip()] + kinit_cmdline.extend(kinit_args) + + elif boolean(self.get_option('_extras').get('ansible_winrm_kerberos_delegation', False)): + kinit_cmdline.append('-f') + + kinit_cmdline.append(principal) + + # pexpect runs the process in its own pty so it can correctly send + # the password as input even on MacOS which blocks subprocess from + # doing so. Unfortunately it is not available on the built in Python + # so we can only use it if someone has installed it + if HAS_PEXPECT: + proc_mechanism = "pexpect" + command = kinit_cmdline.pop(0) + password = to_text(password, encoding='utf-8', + errors='surrogate_or_strict') + + display.vvvv("calling kinit with pexpect for principal %s" + % principal) + try: + child = pexpect.spawn(command, kinit_cmdline, timeout=60, + env=krb5env, echo=False) + except pexpect.ExceptionPexpect as err: + err_msg = "Kerberos auth failure when calling kinit cmd " \ + "'%s': %s" % (command, to_native(err)) + raise AnsibleConnectionFailure(err_msg) + + try: + child.expect(".*:") + child.sendline(password) + except OSError as err: + # child exited before the pass was sent, Ansible will raise + # error based on the rc below, just display the error here + display.vvvv("kinit with pexpect raised OSError: %s" + % to_native(err)) + + # technically this is the stdout + stderr but to match the + # subprocess error checking behaviour, we will call it stderr + stderr = child.read() + child.wait() + rc = child.exitstatus + else: + proc_mechanism = "subprocess" + password = to_bytes(password, encoding='utf-8', + errors='surrogate_or_strict') + + display.vvvv("calling kinit with subprocess for principal %s" + % principal) + try: + p = subprocess.Popen(kinit_cmdline, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=krb5env) + + except OSError as err: + err_msg = "Kerberos auth failure when calling kinit cmd " \ + "'%s': %s" % (self._kinit_cmd, to_native(err)) + raise AnsibleConnectionFailure(err_msg) + + stdout, stderr = p.communicate(password + b'\n') + rc = p.returncode != 0 + + if rc != 0: + # one last attempt at making sure the password does not exist + # in the output + exp_msg = to_native(stderr.strip()) + exp_msg = exp_msg.replace(to_native(password), "<redacted>") + + err_msg = "Kerberos auth failure for principal %s with %s: %s" \ + % (principal, proc_mechanism, exp_msg) + raise AnsibleConnectionFailure(err_msg) + + display.vvvvv("kinit succeeded for principal %s" % principal) + + def _winrm_connect(self): + ''' + Establish a WinRM connection over HTTP/HTTPS. + ''' + display.vvv("ESTABLISH WINRM CONNECTION FOR USER: %s on PORT %s TO %s" % + (self._winrm_user, self._winrm_port, self._winrm_host), host=self._winrm_host) + + winrm_host = self._winrm_host + if HAS_IPADDRESS: + display.debug("checking if winrm_host %s is an IPv6 address" % winrm_host) + try: + ipaddress.IPv6Address(winrm_host) + except ipaddress.AddressValueError: + pass + else: + winrm_host = "[%s]" % winrm_host + + netloc = '%s:%d' % (winrm_host, self._winrm_port) + endpoint = urlunsplit((self._winrm_scheme, netloc, self._winrm_path, '', '')) + errors = [] + for transport in self._winrm_transport: + if transport == 'kerberos': + if not HAVE_KERBEROS: + errors.append('kerberos: the python kerberos library is not installed') + continue + if self._kerb_managed: + self._kerb_auth(self._winrm_user, self._winrm_pass) + display.vvvvv('WINRM CONNECT: transport=%s endpoint=%s' % (transport, endpoint), host=self._winrm_host) + try: + winrm_kwargs = self._winrm_kwargs.copy() + if self._winrm_connection_timeout: + winrm_kwargs['operation_timeout_sec'] = self._winrm_connection_timeout + winrm_kwargs['read_timeout_sec'] = self._winrm_connection_timeout + 1 + protocol = Protocol(endpoint, transport=transport, **winrm_kwargs) + + # open the shell from connect so we know we're able to talk to the server + if not self.shell_id: + self.shell_id = protocol.open_shell(codepage=65001) # UTF-8 + display.vvvvv('WINRM OPEN SHELL: %s' % self.shell_id, host=self._winrm_host) + + return protocol + except Exception as e: + err_msg = to_text(e).strip() + if re.search(to_text(r'Operation\s+?timed\s+?out'), err_msg, re.I): + raise AnsibleError('the connection attempt timed out') + m = re.search(to_text(r'Code\s+?(\d{3})'), err_msg) + if m: + code = int(m.groups()[0]) + if code == 401: + err_msg = 'the specified credentials were rejected by the server' + elif code == 411: + return protocol + errors.append(u'%s: %s' % (transport, err_msg)) + display.vvvvv(u'WINRM CONNECTION ERROR: %s\n%s' % (err_msg, to_text(traceback.format_exc())), host=self._winrm_host) + if errors: + raise AnsibleConnectionFailure(', '.join(map(to_native, errors))) + else: + raise AnsibleError('No transport found for WinRM connection') + + def _winrm_send_input(self, protocol, shell_id, command_id, stdin, eof=False): + rq = {'env:Envelope': protocol._get_soap_header( + resource_uri='http://schemas.microsoft.com/wbem/wsman/1/windows/shell/cmd', + action='http://schemas.microsoft.com/wbem/wsman/1/windows/shell/Send', + shell_id=shell_id)} + stream = rq['env:Envelope'].setdefault('env:Body', {}).setdefault('rsp:Send', {})\ + .setdefault('rsp:Stream', {}) + stream['@Name'] = 'stdin' + stream['@CommandId'] = command_id + stream['#text'] = base64.b64encode(to_bytes(stdin)) + if eof: + stream['@End'] = 'true' + protocol.send_message(xmltodict.unparse(rq)) + + def _winrm_exec(self, command, args=(), from_exec=False, stdin_iterator=None): + if not self.protocol: + self.protocol = self._winrm_connect() + self._connected = True + if from_exec: + display.vvvvv("WINRM EXEC %r %r" % (command, args), host=self._winrm_host) + else: + display.vvvvvv("WINRM EXEC %r %r" % (command, args), host=self._winrm_host) + command_id = None + try: + stdin_push_failed = False + command_id = self.protocol.run_command(self.shell_id, to_bytes(command), map(to_bytes, args), console_mode_stdin=(stdin_iterator is None)) + + try: + if stdin_iterator: + for (data, is_last) in stdin_iterator: + self._winrm_send_input(self.protocol, self.shell_id, command_id, data, eof=is_last) + + except Exception as ex: + display.warning("ERROR DURING WINRM SEND INPUT - attempting to recover: %s %s" + % (type(ex).__name__, to_text(ex))) + display.debug(traceback.format_exc()) + stdin_push_failed = True + + # NB: this can hang if the receiver is still running (eg, network failed a Send request but the server's still happy). + # FUTURE: Consider adding pywinrm status check/abort operations to see if the target is still running after a failure. + resptuple = self.protocol.get_command_output(self.shell_id, command_id) + # ensure stdout/stderr are text for py3 + # FUTURE: this should probably be done internally by pywinrm + response = Response(tuple(to_text(v) if isinstance(v, binary_type) else v for v in resptuple)) + + # TODO: check result from response and set stdin_push_failed if we have nonzero + if from_exec: + display.vvvvv('WINRM RESULT %r' % to_text(response), host=self._winrm_host) + else: + display.vvvvvv('WINRM RESULT %r' % to_text(response), host=self._winrm_host) + + display.vvvvvv('WINRM STDOUT %s' % to_text(response.std_out), host=self._winrm_host) + display.vvvvvv('WINRM STDERR %s' % to_text(response.std_err), host=self._winrm_host) + + if stdin_push_failed: + # There are cases where the stdin input failed but the WinRM service still processed it. We attempt to + # see if stdout contains a valid json return value so we can ignore this error + try: + filtered_output, dummy = _filter_non_json_lines(response.std_out) + json.loads(filtered_output) + except ValueError: + # stdout does not contain a return response, stdin input was a fatal error + stderr = to_bytes(response.std_err, encoding='utf-8') + if stderr.startswith(b"#< CLIXML"): + stderr = _parse_clixml(stderr) + + raise AnsibleError('winrm send_input failed; \nstdout: %s\nstderr %s' + % (to_native(response.std_out), to_native(stderr))) + + return response + except requests.exceptions.Timeout as exc: + raise AnsibleConnectionFailure('winrm connection error: %s' % to_native(exc)) + finally: + if command_id: + self.protocol.cleanup_command(self.shell_id, command_id) + + def _connect(self): + + if not HAS_WINRM: + raise AnsibleError("winrm or requests is not installed: %s" % to_native(WINRM_IMPORT_ERR)) + elif not HAS_XMLTODICT: + raise AnsibleError("xmltodict is not installed: %s" % to_native(XMLTODICT_IMPORT_ERR)) + + super(Connection, self)._connect() + if not self.protocol: + self._build_winrm_kwargs() # build the kwargs from the options set + self.protocol = self._winrm_connect() + self._connected = True + return self + + def reset(self): + if not self._connected: + return + self.protocol = None + self.shell_id = None + self._connect() + + def _wrapper_payload_stream(self, payload, buffer_size=200000): + payload_bytes = to_bytes(payload) + byte_count = len(payload_bytes) + for i in range(0, byte_count, buffer_size): + yield payload_bytes[i:i + buffer_size], i + buffer_size >= byte_count + + def exec_command(self, cmd, in_data=None, sudoable=True): + super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable) + cmd_parts = self._shell._encode_script(cmd, as_list=True, strict_mode=False, preserve_rc=False) + + # TODO: display something meaningful here + display.vvv("EXEC (via pipeline wrapper)") + + stdin_iterator = None + + if in_data: + stdin_iterator = self._wrapper_payload_stream(in_data) + + result = self._winrm_exec(cmd_parts[0], cmd_parts[1:], from_exec=True, stdin_iterator=stdin_iterator) + + result.std_out = to_bytes(result.std_out) + result.std_err = to_bytes(result.std_err) + + # parse just stderr from CLIXML output + if result.std_err.startswith(b"#< CLIXML"): + try: + result.std_err = _parse_clixml(result.std_err) + except Exception: + # unsure if we're guaranteed a valid xml doc- use raw output in case of error + pass + + return (result.status_code, result.std_out, result.std_err) + + # FUTURE: determine buffer size at runtime via remote winrm config? + def _put_file_stdin_iterator(self, in_path, out_path, buffer_size=250000): + in_size = os.path.getsize(to_bytes(in_path, errors='surrogate_or_strict')) + offset = 0 + with open(to_bytes(in_path, errors='surrogate_or_strict'), 'rb') as in_file: + for out_data in iter((lambda: in_file.read(buffer_size)), b''): + offset += len(out_data) + self._display.vvvvv('WINRM PUT "%s" to "%s" (offset=%d size=%d)' % (in_path, out_path, offset, len(out_data)), host=self._winrm_host) + # yes, we're double-encoding over the wire in this case- we want to ensure that the data shipped to the end PS pipeline is still b64-encoded + b64_data = base64.b64encode(out_data) + b'\r\n' + # cough up the data, as well as an indicator if this is the last chunk so winrm_send knows to set the End signal + yield b64_data, (in_file.tell() == in_size) + + if offset == 0: # empty file, return an empty buffer + eof to close it + yield "", True + + def put_file(self, in_path, out_path): + super(Connection, self).put_file(in_path, out_path) + out_path = self._shell._unquote(out_path) + display.vvv('PUT "%s" TO "%s"' % (in_path, out_path), host=self._winrm_host) + if not os.path.exists(to_bytes(in_path, errors='surrogate_or_strict')): + raise AnsibleFileNotFound('file or module does not exist: "%s"' % to_native(in_path)) + + script_template = u''' + begin {{ + $path = '{0}' + + $DebugPreference = "Continue" + $ErrorActionPreference = "Stop" + Set-StrictMode -Version 2 + + $fd = [System.IO.File]::Create($path) + + $sha1 = [System.Security.Cryptography.SHA1CryptoServiceProvider]::Create() + + $bytes = @() #initialize for empty file case + }} + process {{ + $bytes = [System.Convert]::FromBase64String($input) + $sha1.TransformBlock($bytes, 0, $bytes.Length, $bytes, 0) | Out-Null + $fd.Write($bytes, 0, $bytes.Length) + }} + end {{ + $sha1.TransformFinalBlock($bytes, 0, 0) | Out-Null + + $hash = [System.BitConverter]::ToString($sha1.Hash).Replace("-", "").ToLowerInvariant() + + $fd.Close() + + Write-Output "{{""sha1"":""$hash""}}" + }} + ''' + + script = script_template.format(self._shell._escape(out_path)) + cmd_parts = self._shell._encode_script(script, as_list=True, strict_mode=False, preserve_rc=False) + + result = self._winrm_exec(cmd_parts[0], cmd_parts[1:], stdin_iterator=self._put_file_stdin_iterator(in_path, out_path)) + # TODO: improve error handling + if result.status_code != 0: + raise AnsibleError(to_native(result.std_err)) + + try: + put_output = json.loads(result.std_out) + except ValueError: + # stdout does not contain a valid response + stderr = to_bytes(result.std_err, encoding='utf-8') + if stderr.startswith(b"#< CLIXML"): + stderr = _parse_clixml(stderr) + raise AnsibleError('winrm put_file failed; \nstdout: %s\nstderr %s' % (to_native(result.std_out), to_native(stderr))) + + remote_sha1 = put_output.get("sha1") + if not remote_sha1: + raise AnsibleError("Remote sha1 was not returned") + + local_sha1 = secure_hash(in_path) + + if not remote_sha1 == local_sha1: + raise AnsibleError("Remote sha1 hash {0} does not match local hash {1}".format(to_native(remote_sha1), to_native(local_sha1))) + + def fetch_file(self, in_path, out_path): + super(Connection, self).fetch_file(in_path, out_path) + in_path = self._shell._unquote(in_path) + out_path = out_path.replace('\\', '/') + # consistent with other connection plugins, we assume the caller has created the target dir + display.vvv('FETCH "%s" TO "%s"' % (in_path, out_path), host=self._winrm_host) + buffer_size = 2**19 # 0.5MB chunks + out_file = None + try: + offset = 0 + while True: + try: + script = ''' + $path = '%(path)s' + If (Test-Path -Path $path -PathType Leaf) + { + $buffer_size = %(buffer_size)d + $offset = %(offset)d + + $stream = New-Object -TypeName IO.FileStream($path, [IO.FileMode]::Open, [IO.FileAccess]::Read, [IO.FileShare]::ReadWrite) + $stream.Seek($offset, [System.IO.SeekOrigin]::Begin) > $null + $buffer = New-Object -TypeName byte[] $buffer_size + $bytes_read = $stream.Read($buffer, 0, $buffer_size) + if ($bytes_read -gt 0) { + $bytes = $buffer[0..($bytes_read - 1)] + [System.Convert]::ToBase64String($bytes) + } + $stream.Close() > $null + } + ElseIf (Test-Path -Path $path -PathType Container) + { + Write-Host "[DIR]"; + } + Else + { + Write-Error "$path does not exist"; + Exit 1; + } + ''' % dict(buffer_size=buffer_size, path=self._shell._escape(in_path), offset=offset) + display.vvvvv('WINRM FETCH "%s" to "%s" (offset=%d)' % (in_path, out_path, offset), host=self._winrm_host) + cmd_parts = self._shell._encode_script(script, as_list=True, preserve_rc=False) + result = self._winrm_exec(cmd_parts[0], cmd_parts[1:]) + if result.status_code != 0: + raise IOError(to_native(result.std_err)) + if result.std_out.strip() == '[DIR]': + data = None + else: + data = base64.b64decode(result.std_out.strip()) + if data is None: + break + else: + if not out_file: + # If out_path is a directory and we're expecting a file, bail out now. + if os.path.isdir(to_bytes(out_path, errors='surrogate_or_strict')): + break + out_file = open(to_bytes(out_path, errors='surrogate_or_strict'), 'wb') + out_file.write(data) + if len(data) < buffer_size: + break + offset += len(data) + except Exception: + traceback.print_exc() + raise AnsibleError('failed to transfer file to "%s"' % to_native(out_path)) + finally: + if out_file: + out_file.close() + + def close(self): + if self.protocol and self.shell_id: + display.vvvvv('WINRM CLOSE SHELL: %s' % self.shell_id, host=self._winrm_host) + self.protocol.close_shell(self.shell_id) + self.shell_id = None + self.protocol = None + self._connected = False diff --git a/lib/ansible/plugins/doc_fragments/__init__.py b/lib/ansible/plugins/doc_fragments/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/lib/ansible/plugins/doc_fragments/__init__.py diff --git a/lib/ansible/plugins/doc_fragments/action_common_attributes.py b/lib/ansible/plugins/doc_fragments/action_common_attributes.py new file mode 100644 index 0000000..c135df5 --- /dev/null +++ b/lib/ansible/plugins/doc_fragments/action_common_attributes.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +# Copyright: Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +class ModuleDocFragment(object): + + # Standard documentation fragment + DOCUMENTATION = r''' +attributes: + check_mode: + description: Can run in check_mode and return changed status prediction without modifying target + diff_mode: + description: Will return details on what has changed (or possibly needs changing in check_mode), when in diff mode + platform: + description: Target OS/families that can be operated against + support: N/A +''' + + ACTIONGROUPS = r''' +attributes: + action_group: + description: Action is part of action_group(s), for convenient setting of module_defaults. + support: N/A + membership: [] +''' + + CONN = r''' +attributes: + become: + description: Is usable alongside become keywords + connection: + description: Uses the target's configured connection information to execute code on it + delegation: + description: Can be used in conjunction with delegate_to and related keywords +''' + + FACTS = r''' +attributes: + facts: + description: Action returns an C(ansible_facts) dictionary that will update existing host facts +''' + + FILES = r''' +attributes: + safe_file_operations: + description: Uses Ansible's strict file operation functions to ensure proper permissions and avoid data corruption + vault: + description: Can automatically decrypt Ansible vaulted files +''' + + FLOW = r''' +attributes: + action: + description: Indicates this has a corresponding action plugin so some parts of the options can be executed on the controller + async: + description: Supports being used with the C(async) keyword + bypass_host_loop: + description: + - Forces a 'global' task that does not execute per host, this bypasses per host templating and serial, + throttle and other loop considerations + - Conditionals will work as if C(run_once) is being used, variables used will be from the first available host + - This action will not work normally outside of lockstep strategies +''' + RAW = r''' +attributes: + raw: + description: Indicates if an action takes a 'raw' or 'free form' string as an option and has it's own special parsing of it +''' diff --git a/lib/ansible/plugins/doc_fragments/action_core.py b/lib/ansible/plugins/doc_fragments/action_core.py new file mode 100644 index 0000000..931ca14 --- /dev/null +++ b/lib/ansible/plugins/doc_fragments/action_core.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +# Copyright: (c) , Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +# WARNING: this is mostly here as a convinence for documenting core behaviours, no plugin outside of ansible-core should use this file +class ModuleDocFragment(object): + + # requires action_common + DOCUMENTATION = r''' +attributes: + async: + support: none + become: + support: none + bypass_task_loop: + description: These tasks ignore the C(loop) and C(with_) keywords + core: + description: This is a 'core engine' feature and is not implemented like most task actions, so it is not overridable in any way via the plugin system. + support: full + connection: + support: none + ignore_conditional: + support: none + description: The action is not subject to conditional execution so it will ignore the C(when:) keyword + platform: + support: full + platforms: all + until: + description: Denotes if this action objeys until/retry/poll keywords + support: full + tags: + description: Allows for the 'tags' keyword to control the selection of this action for execution + support: full +''' + + # also requires core above + IMPORT = r''' +attributes: + action: + details: While this action executes locally on the controller it is not governed by an action plugin + support: none + bypass_host_loop: + details: While the import can be host specific and runs per host it is not dealing with all available host variables, + use an include instead for those cases + support: partial + bypass_task_loop: + details: The task itself is not looped, but the loop is applied to each imported task + support: partial + delegation: + details: Since there are no connection nor facts, there is no sense in delegating imports + support: none + ignore_conditional: + details: While the action itself will ignore the conditional, it will be inherited by the imported tasks themselves + support: partial + tags: + details: Tags are not interpreted for this action, they are applied to the imported tasks + support: none + until: + support: none +''' + # also requires core above + INCLUDE = r''' +attributes: + action: + details: While this action executes locally on the controller it is not governed by an action plugin + support: none + bypass_host_loop: + support: none + bypass_task_loop: + support: none + delegation: + details: Since there are no connection nor facts, there is no sense in delegating includes + support: none + tags: + details: Tags are interpreted by this action but are not automatically inherited by the include tasks, see C(apply) + support: partial +''' diff --git a/lib/ansible/plugins/doc_fragments/backup.py b/lib/ansible/plugins/doc_fragments/backup.py new file mode 100644 index 0000000..d2e76dc --- /dev/null +++ b/lib/ansible/plugins/doc_fragments/backup.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- + +# Copyright: (c) 2015, Ansible, Inc +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +class ModuleDocFragment(object): + + # Standard documentation fragment + DOCUMENTATION = r''' +options: + backup: + description: + - Create a backup file including the timestamp information so you can get + the original file back if you somehow clobbered it incorrectly. + type: bool + default: no +''' diff --git a/lib/ansible/plugins/doc_fragments/connection_pipelining.py b/lib/ansible/plugins/doc_fragments/connection_pipelining.py new file mode 100644 index 0000000..fa18265 --- /dev/null +++ b/lib/ansible/plugins/doc_fragments/connection_pipelining.py @@ -0,0 +1,31 @@ +# Copyright (c) 2021 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +class ModuleDocFragment(object): + + # common shelldocumentation fragment + DOCUMENTATION = """ +options: + pipelining: + default: false + description: + - Pipelining reduces the number of connection operations required to execute a module on the remote server, + by executing many Ansible modules without actual file transfers. + - This can result in a very significant performance improvement when enabled. + - However this can conflict with privilege escalation (become). + For example, when using sudo operations you must first disable 'requiretty' in the sudoers file for the target hosts, + which is why this feature is disabled by default. + env: + - name: ANSIBLE_PIPELINING + ini: + - section: defaults + key: pipelining + - section: connection + key: pipelining + type: boolean + vars: + - name: ansible_pipelining +""" diff --git a/lib/ansible/plugins/doc_fragments/constructed.py b/lib/ansible/plugins/doc_fragments/constructed.py new file mode 100644 index 0000000..7810acb --- /dev/null +++ b/lib/ansible/plugins/doc_fragments/constructed.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- + +# Copyright: (c) 2017, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +class ModuleDocFragment(object): + + DOCUMENTATION = r''' +options: + strict: + description: + - If C(yes) make invalid entries a fatal error, otherwise skip and continue. + - Since it is possible to use facts in the expressions they might not always be available + and we ignore those errors by default. + type: bool + default: no + compose: + description: Create vars from jinja2 expressions. + type: dict + default: {} + groups: + description: Add hosts to group based on Jinja2 conditionals. + type: dict + default: {} + keyed_groups: + description: Add hosts to group based on the values of a variable. + type: list + default: [] + elements: dict + suboptions: + parent_group: + type: str + description: parent group for keyed group + prefix: + type: str + description: A keyed group name will start with this prefix + default: '' + separator: + type: str + description: separator used to build the keyed group name + default: "_" + key: + type: str + description: + - The key from input dictionary used to generate groups + default_value: + description: + - The default value when the host variable's value is an empty string. + - This option is mutually exclusive with C(trailing_separator). + type: str + version_added: '2.12' + trailing_separator: + description: + - Set this option to I(False) to omit the C(separator) after the host variable when the value is an empty string. + - This option is mutually exclusive with C(default_value). + type: bool + default: True + version_added: '2.12' + use_extra_vars: + version_added: '2.11' + description: Merge extra vars into the available variables for composition (highest precedence). + type: bool + default: False + ini: + - section: inventory_plugins + key: use_extra_vars + env: + - name: ANSIBLE_INVENTORY_USE_EXTRA_VARS + leading_separator: + description: + - Use in conjunction with keyed_groups. + - By default, a keyed group that does not have a prefix or a separator provided will have a name that starts with an underscore. + - This is because the default prefix is "" and the default separator is "_". + - Set this option to False to omit the leading underscore (or other separator) if no prefix is given. + - If the group name is derived from a mapping the separator is still used to concatenate the items. + - To not use a separator in the group name at all, set the separator for the keyed group to an empty string instead. + type: boolean + default: True + version_added: '2.11' +''' diff --git a/lib/ansible/plugins/doc_fragments/decrypt.py b/lib/ansible/plugins/doc_fragments/decrypt.py new file mode 100644 index 0000000..ea7cf59 --- /dev/null +++ b/lib/ansible/plugins/doc_fragments/decrypt.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- + +# Copyright: (c) 2017, Brian Coca <bcoca@redhat.com> +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +class ModuleDocFragment(object): + + # Standard files documentation fragment + DOCUMENTATION = r''' +options: + decrypt: + description: + - This option controls the autodecryption of source files using vault. + type: bool + default: yes + version_added: '2.4' +''' diff --git a/lib/ansible/plugins/doc_fragments/default_callback.py b/lib/ansible/plugins/doc_fragments/default_callback.py new file mode 100644 index 0000000..5798334 --- /dev/null +++ b/lib/ansible/plugins/doc_fragments/default_callback.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- + +# Copyright: (c) 2017, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +class ModuleDocFragment(object): + + DOCUMENTATION = r''' + options: + display_skipped_hosts: + name: Show skipped hosts + description: "Toggle to control displaying skipped task/host results in a task" + type: bool + default: yes + env: + - name: ANSIBLE_DISPLAY_SKIPPED_HOSTS + ini: + - key: display_skipped_hosts + section: defaults + display_ok_hosts: + name: Show 'ok' hosts + description: "Toggle to control displaying 'ok' task/host results in a task" + type: bool + default: yes + env: + - name: ANSIBLE_DISPLAY_OK_HOSTS + ini: + - key: display_ok_hosts + section: defaults + version_added: '2.7' + display_failed_stderr: + name: Use STDERR for failed and unreachable tasks + description: "Toggle to control whether failed and unreachable tasks are displayed to STDERR (vs. STDOUT)" + type: bool + default: no + env: + - name: ANSIBLE_DISPLAY_FAILED_STDERR + ini: + - key: display_failed_stderr + section: defaults + version_added: '2.7' + show_custom_stats: + name: Show custom stats + description: 'This adds the custom stats set via the set_stats plugin to the play recap' + type: bool + default: no + env: + - name: ANSIBLE_SHOW_CUSTOM_STATS + ini: + - key: show_custom_stats + section: defaults + show_per_host_start: + name: Show per host task start + description: 'This adds output that shows when a task is started to execute for each host' + type: bool + default: no + env: + - name: ANSIBLE_SHOW_PER_HOST_START + ini: + - key: show_per_host_start + section: defaults + version_added: '2.9' + check_mode_markers: + name: Show markers when running in check mode + description: + - Toggle to control displaying markers when running in check mode. + - "The markers are C(DRY RUN) at the beginning and ending of playbook execution (when calling C(ansible-playbook --check)) + and C(CHECK MODE) as a suffix at every play and task that is run in check mode." + type: bool + default: no + version_added: '2.9' + env: + - name: ANSIBLE_CHECK_MODE_MARKERS + ini: + - key: check_mode_markers + section: defaults + show_task_path_on_failure: + name: Show file path on failed tasks + description: + When a task fails, display the path to the file containing the failed task and the line number. + This information is displayed automatically for every task when running with C(-vv) or greater verbosity. + type: bool + default: no + env: + - name: ANSIBLE_SHOW_TASK_PATH_ON_FAILURE + ini: + - key: show_task_path_on_failure + section: defaults + version_added: '2.11' +''' diff --git a/lib/ansible/plugins/doc_fragments/files.py b/lib/ansible/plugins/doc_fragments/files.py new file mode 100644 index 0000000..b87fd11 --- /dev/null +++ b/lib/ansible/plugins/doc_fragments/files.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- + +# Copyright: (c) 2014, Matt Martz <matt@sivel.net> +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +class ModuleDocFragment(object): + + # Standard files documentation fragment + + # Note: mode is overridden by the copy and template modules so if you change the description + # here, you should also change it there. + DOCUMENTATION = r''' +options: + mode: + description: + - The permissions the resulting filesystem object should have. + - For those used to I(/usr/bin/chmod) remember that modes are actually octal numbers. + You must either add a leading zero so that Ansible's YAML parser knows it is an octal number + (like C(0644) or C(01777)) or quote it (like C('644') or C('1777')) so Ansible receives + a string and can do its own conversion from string into number. + - Giving Ansible a number without following one of these rules will end up with a decimal + number which will have unexpected results. + - As of Ansible 1.8, the mode may be specified as a symbolic mode (for example, C(u+rwx) or + C(u=rw,g=r,o=r)). + - If C(mode) is not specified and the destination filesystem object B(does not) exist, the default C(umask) on the system will be used + when setting the mode for the newly created filesystem object. + - If C(mode) is not specified and the destination filesystem object B(does) exist, the mode of the existing filesystem object will be used. + - Specifying C(mode) is the best way to ensure filesystem objects are created with the correct permissions. + See CVE-2020-1736 for further details. + type: raw + owner: + description: + - Name of the user that should own the filesystem object, as would be fed to I(chown). + - When left unspecified, it uses the current user unless you are root, in which + case it can preserve the previous ownership. + - Specifying a numeric username will be assumed to be a user ID and not a username. Avoid numeric usernames to avoid this confusion. + + type: str + group: + description: + - Name of the group that should own the filesystem object, as would be fed to I(chown). + - When left unspecified, it uses the current group of the current user unless you are root, + in which case it can preserve the previous ownership. + type: str + seuser: + description: + - The user part of the SELinux filesystem object context. + - By default it uses the C(system) policy, where applicable. + - When set to C(_default), it will use the C(user) portion of the policy if available. + type: str + serole: + description: + - The role part of the SELinux filesystem object context. + - When set to C(_default), it will use the C(role) portion of the policy if available. + type: str + setype: + description: + - The type part of the SELinux filesystem object context. + - When set to C(_default), it will use the C(type) portion of the policy if available. + type: str + selevel: + description: + - The level part of the SELinux filesystem object context. + - This is the MLS/MCS attribute, sometimes known as the C(range). + - When set to C(_default), it will use the C(level) portion of the policy if available. + type: str + unsafe_writes: + description: + - Influence when to use atomic operation to prevent data corruption or inconsistent reads from the target filesystem object. + - By default this module uses atomic operations to prevent data corruption or inconsistent reads from the target filesystem objects, + but sometimes systems are configured or just broken in ways that prevent this. One example is docker mounted filesystem objects, + which cannot be updated atomically from inside the container and can only be written in an unsafe manner. + - This option allows Ansible to fall back to unsafe methods of updating filesystem objects when atomic operations fail + (however, it doesn't force Ansible to perform unsafe writes). + - IMPORTANT! Unsafe writes are subject to race conditions and can lead to data corruption. + type: bool + default: no + version_added: '2.2' + attributes: + description: + - The attributes the resulting filesystem object should have. + - To get supported flags look at the man page for I(chattr) on the target system. + - This string should contain the attributes in the same order as the one displayed by I(lsattr). + - The C(=) operator is assumed as default, otherwise C(+) or C(-) operators need to be included in the string. + type: str + aliases: [ attr ] + version_added: '2.3' +''' diff --git a/lib/ansible/plugins/doc_fragments/inventory_cache.py b/lib/ansible/plugins/doc_fragments/inventory_cache.py new file mode 100644 index 0000000..9326c3f --- /dev/null +++ b/lib/ansible/plugins/doc_fragments/inventory_cache.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- + +# Copyright: (c) 2017, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +class ModuleDocFragment(object): + + # inventory cache + DOCUMENTATION = r''' +options: + cache: + description: + - Toggle to enable/disable the caching of the inventory's source data, requires a cache plugin setup to work. + type: bool + default: no + env: + - name: ANSIBLE_INVENTORY_CACHE + ini: + - section: inventory + key: cache + cache_plugin: + description: + - Cache plugin to use for the inventory's source data. + type: str + default: memory + env: + - name: ANSIBLE_CACHE_PLUGIN + - name: ANSIBLE_INVENTORY_CACHE_PLUGIN + ini: + - section: defaults + key: fact_caching + - section: inventory + key: cache_plugin + cache_timeout: + description: + - Cache duration in seconds + default: 3600 + type: int + env: + - name: ANSIBLE_CACHE_PLUGIN_TIMEOUT + - name: ANSIBLE_INVENTORY_CACHE_TIMEOUT + ini: + - section: defaults + key: fact_caching_timeout + - section: inventory + key: cache_timeout + cache_connection: + description: + - Cache connection data or path, read cache plugin documentation for specifics. + type: str + env: + - name: ANSIBLE_CACHE_PLUGIN_CONNECTION + - name: ANSIBLE_INVENTORY_CACHE_CONNECTION + ini: + - section: defaults + key: fact_caching_connection + - section: inventory + key: cache_connection + cache_prefix: + description: + - Prefix to use for cache plugin files/tables + default: ansible_inventory_ + env: + - name: ANSIBLE_CACHE_PLUGIN_PREFIX + - name: ANSIBLE_INVENTORY_CACHE_PLUGIN_PREFIX + ini: + - section: default + key: fact_caching_prefix + deprecated: + alternatives: Use the 'defaults' section instead + why: Fixes typing error in INI section name + version: '2.16' + - section: defaults + key: fact_caching_prefix + - section: inventory + key: cache_prefix +''' diff --git a/lib/ansible/plugins/doc_fragments/result_format_callback.py b/lib/ansible/plugins/doc_fragments/result_format_callback.py new file mode 100644 index 0000000..1b71173 --- /dev/null +++ b/lib/ansible/plugins/doc_fragments/result_format_callback.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- + +# Copyright: (c) 2017, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +class ModuleDocFragment(object): + + DOCUMENTATION = r''' + options: + result_format: + name: Format of the task result + description: + - Define the task result format used in the callback output. + - These formats do not cause the callback to emit valid JSON or YAML formats. + - The output contains these formats interspersed with other non-machine parsable data. + type: str + default: json + env: + - name: ANSIBLE_CALLBACK_RESULT_FORMAT + ini: + - key: callback_result_format + section: defaults + choices: + - json + - yaml + version_added: '2.13' + pretty_results: + name: Configure output for readability + description: + - Configure the result format to be more readable + - When the result format is set to C(yaml) this option defaults to C(True), and defaults + to C(False) when configured to C(json). + - Setting this option to C(True) will force C(json) and C(yaml) results to always be pretty + printed regardless of verbosity. + - When set to C(True) and used with the C(yaml) result format, this option will + modify module responses in an attempt to produce a more human friendly output at the expense + of correctness, and should not be relied upon to aid in writing variable manipulations + or conditionals. For correctness, set this option to C(False) or set the result format to C(json). + type: bool + default: null + env: + - name: ANSIBLE_CALLBACK_FORMAT_PRETTY + ini: + - key: callback_format_pretty + section: defaults + version_added: '2.13' +''' diff --git a/lib/ansible/plugins/doc_fragments/return_common.py b/lib/ansible/plugins/doc_fragments/return_common.py new file mode 100644 index 0000000..6f54288 --- /dev/null +++ b/lib/ansible/plugins/doc_fragments/return_common.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- + +# Copyright: (c) 2016, Ansible, Inc +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +class ModuleDocFragment(object): + # Standard documentation fragment + RETURN = r''' +changed: + description: Whether the module affected changes on the target. + returned: always + type: bool + sample: false +failed: + description: Whether the module failed to execute. + returned: always + type: bool + sample: true +msg: + description: Human-readable message. + returned: as needed + type: str + sample: all ok +skipped: + description: Whether the module was skipped. + returned: always + type: bool + sample: false +results: + description: List of module results, + returned: when using a loop. + type: list + sample: [{changed: True, msg: 'first item changed'}, {changed: False, msg: 'second item ok'}] +exception: + description: Optional information from a handled error. + returned: on some errors + type: str + sample: Unknown error +''' diff --git a/lib/ansible/plugins/doc_fragments/shell_common.py b/lib/ansible/plugins/doc_fragments/shell_common.py new file mode 100644 index 0000000..fe1ae4e --- /dev/null +++ b/lib/ansible/plugins/doc_fragments/shell_common.py @@ -0,0 +1,98 @@ +# Copyright (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +class ModuleDocFragment(object): + + # common shelldocumentation fragment + DOCUMENTATION = """ +options: + remote_tmp: + description: + - Temporary directory to use on targets when executing tasks. + default: '~/.ansible/tmp' + env: [{name: ANSIBLE_REMOTE_TEMP}, {name: ANSIBLE_REMOTE_TMP}] + ini: + - section: defaults + key: remote_tmp + vars: + - name: ansible_remote_tmp + common_remote_group: + name: Enables changing the group ownership of temporary files and directories + default: null + description: + - Checked when Ansible needs to execute a module as a different user. + - If setfacl and chown both fail and do not let the different user access the module's files, they will be chgrp'd to this group. + - In order for this to work, the remote_user and become_user must share a common group and this setting must be set to that group. + env: [{name: ANSIBLE_COMMON_REMOTE_GROUP}] + vars: + - name: ansible_common_remote_group + ini: + - {key: common_remote_group, section: defaults} + version_added: "2.10" + system_tmpdirs: + description: + - "List of valid system temporary directories on the managed machine for Ansible to validate + C(remote_tmp) against, when specific permissions are needed. These must be world + readable, writable, and executable. This list should only contain directories which the + system administrator has pre-created with the proper ownership and permissions otherwise + security issues can arise." + - When C(remote_tmp) is required to be a system temp dir and it does not match any in the list, + the first one from the list will be used instead. + default: [ /var/tmp, /tmp ] + type: list + elements: string + env: [{name: ANSIBLE_SYSTEM_TMPDIRS}] + ini: + - section: defaults + key: system_tmpdirs + vars: + - name: ansible_system_tmpdirs + async_dir: + description: + - Directory in which ansible will keep async job information + default: '~/.ansible_async' + env: [{name: ANSIBLE_ASYNC_DIR}] + ini: + - section: defaults + key: async_dir + vars: + - name: ansible_async_dir + environment: + type: list + elements: dictionary + default: [{}] + description: + - List of dictionaries of environment variables and their values to use when executing commands. + keyword: + - name: environment + admin_users: + type: list + elements: string + default: ['root', 'toor'] + description: + - list of users to be expected to have admin privileges. This is used by the controller to + determine how to share temporary files between the remote user and the become user. + env: + - name: ANSIBLE_ADMIN_USERS + ini: + - section: defaults + key: admin_users + vars: + - name: ansible_admin_users + world_readable_temp: + version_added: '2.10' + default: False + description: + - This makes the temporary files created on the machine world-readable and will issue a warning instead of failing the task. + - It is useful when becoming an unprivileged user. + env: + - name: ANSIBLE_SHELL_ALLOW_WORLD_READABLE_TEMP + vars: + - name: ansible_shell_allow_world_readable_temp + ini: + - {key: allow_world_readable_tmpfiles, section: defaults} + type: boolean +""" diff --git a/lib/ansible/plugins/doc_fragments/shell_windows.py b/lib/ansible/plugins/doc_fragments/shell_windows.py new file mode 100644 index 0000000..ac52c60 --- /dev/null +++ b/lib/ansible/plugins/doc_fragments/shell_windows.py @@ -0,0 +1,51 @@ +# Copyright (c) 2019 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +class ModuleDocFragment(object): + + # Windows shell documentation fragment + # FIXME: set_module_language don't belong here but must be set so they don't fail when someone + # get_option('set_module_language') on this plugin + DOCUMENTATION = r""" +options: + async_dir: + description: + - Directory in which ansible will keep async job information. + - Before Ansible 2.8, this was set to C(remote_tmp + "\.ansible_async"). + default: '%USERPROFILE%\.ansible_async' + ini: + - section: powershell + key: async_dir + vars: + - name: ansible_async_dir + version_added: '2.8' + remote_tmp: + description: + - Temporary directory to use on targets when copying files to the host. + default: '%TEMP%' + ini: + - section: powershell + key: remote_tmp + vars: + - name: ansible_remote_tmp + set_module_language: + description: + - Controls if we set the locale for modules when executing on the + target. + - Windows only supports C(no) as an option. + type: bool + default: 'no' + choices: ['no', False] + environment: + description: + - List of dictionaries of environment variables and their values to use when + executing commands. + keyword: + - name: environment + type: list + elements: dictionary + default: [{}] +""" diff --git a/lib/ansible/plugins/doc_fragments/template_common.py b/lib/ansible/plugins/doc_fragments/template_common.py new file mode 100644 index 0000000..6276e84 --- /dev/null +++ b/lib/ansible/plugins/doc_fragments/template_common.py @@ -0,0 +1,121 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2019 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +class ModuleDocFragment(object): + + # Standard template documentation fragment, use by template and win_template. + DOCUMENTATION = r''' +description: +- Templates are processed by the L(Jinja2 templating language,http://jinja.pocoo.org/docs/). +- Documentation on the template formatting can be found in the + L(Template Designer Documentation,http://jinja.pocoo.org/docs/templates/). +- Additional variables listed below can be used in templates. +- C(ansible_managed) (configurable via the C(defaults) section of C(ansible.cfg)) contains a string which can be used to + describe the template name, host, modification time of the template file and the owner uid. +- C(template_host) contains the node name of the template's machine. +- C(template_uid) is the numeric user id of the owner. +- C(template_path) is the path of the template. +- C(template_fullpath) is the absolute path of the template. +- C(template_destpath) is the path of the template on the remote system (added in 2.8). +- C(template_run_date) is the date that the template was rendered. +options: + src: + description: + - Path of a Jinja2 formatted template on the Ansible controller. + - This can be a relative or an absolute path. + - The file must be encoded with C(utf-8) but I(output_encoding) can be used to control the encoding of the output + template. + type: path + required: yes + dest: + description: + - Location to render the template to on the remote machine. + type: path + required: yes + newline_sequence: + description: + - Specify the newline sequence to use for templating files. + type: str + choices: [ '\n', '\r', '\r\n' ] + default: '\n' + version_added: '2.4' + block_start_string: + description: + - The string marking the beginning of a block. + type: str + default: '{%' + version_added: '2.4' + block_end_string: + description: + - The string marking the end of a block. + type: str + default: '%}' + version_added: '2.4' + variable_start_string: + description: + - The string marking the beginning of a print statement. + type: str + default: '{{' + version_added: '2.4' + variable_end_string: + description: + - The string marking the end of a print statement. + type: str + default: '}}' + version_added: '2.4' + comment_start_string: + description: + - The string marking the beginning of a comment statement. + type: str + version_added: '2.12' + comment_end_string: + description: + - The string marking the end of a comment statement. + type: str + version_added: '2.12' + trim_blocks: + description: + - Determine when newlines should be removed from blocks. + - When set to C(yes) the first newline after a block is removed (block, not variable tag!). + type: bool + default: yes + version_added: '2.4' + lstrip_blocks: + description: + - Determine when leading spaces and tabs should be stripped. + - When set to C(yes) leading spaces and tabs are stripped from the start of a line to a block. + type: bool + default: no + version_added: '2.6' + force: + description: + - Determine when the file is being transferred if the destination already exists. + - When set to C(yes), replace the remote file when contents are different than the source. + - When set to C(no), the file will only be transferred if the destination does not exist. + type: bool + default: yes + output_encoding: + description: + - Overrides the encoding used to write the template file defined by C(dest). + - It defaults to C(utf-8), but any encoding supported by python can be used. + - The source template file must always be encoded using C(utf-8), for homogeneity. + type: str + default: utf-8 + version_added: '2.7' +notes: +- Including a string that uses a date in the template will result in the template being marked 'changed' each time. +- Since Ansible 0.9, templates are loaded with C(trim_blocks=True). +- > + Also, you can override jinja2 settings by adding a special header to template file. + i.e. C(#jinja2:variable_start_string:'[%', variable_end_string:'%]', trim_blocks: False) + which changes the variable interpolation markers to C([% var %]) instead of C({{ var }}). + This is the best way to prevent evaluation of things that look like, but should not be Jinja2. +- To find Byte Order Marks in files, use C(Format-Hex <file> -Count 16) on Windows, and use C(od -a -t x1 -N 16 <file>) + on Linux. +''' diff --git a/lib/ansible/plugins/doc_fragments/url.py b/lib/ansible/plugins/doc_fragments/url.py new file mode 100644 index 0000000..eb2b17f --- /dev/null +++ b/lib/ansible/plugins/doc_fragments/url.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- + +# Copyright: (c) 2018, John Barker <gundalow@redhat.com> +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +class ModuleDocFragment(object): + + # Standard files documentation fragment + DOCUMENTATION = r''' +options: + url: + description: + - HTTP, HTTPS, or FTP URL in the form (http|https|ftp)://[user[:pass]]@host.domain[:port]/path + type: str + force: + description: + - If C(yes) do not get a cached copy. + type: bool + default: no + http_agent: + description: + - Header to identify as, generally appears in web server logs. + type: str + default: ansible-httpget + use_proxy: + description: + - If C(no), it will not use a proxy, even if one is defined in an environment variable on the target hosts. + type: bool + default: yes + validate_certs: + description: + - If C(no), SSL certificates will not be validated. + - This should only be used on personally controlled sites using self-signed certificates. + type: bool + default: yes + url_username: + description: + - The username for use in HTTP basic authentication. + - This parameter can be used without I(url_password) for sites that allow empty passwords + type: str + url_password: + description: + - The password for use in HTTP basic authentication. + - If the I(url_username) parameter is not specified, the I(url_password) parameter will not be used. + type: str + force_basic_auth: + description: + - Credentials specified with I(url_username) and I(url_password) should be passed in HTTP Header. + type: bool + default: no + client_cert: + description: + - PEM formatted certificate chain file to be used for SSL client authentication. + - This file can also include the key as well, and if the key is included, C(client_key) is not required. + type: path + client_key: + description: + - PEM formatted file that contains your private key to be used for SSL client authentication. + - If C(client_cert) contains both the certificate and key, this option is not required. + type: path + use_gssapi: + description: + - Use GSSAPI to perform the authentication, typically this is for Kerberos or Kerberos through Negotiate + authentication. + - Requires the Python library L(gssapi,https://github.com/pythongssapi/python-gssapi) to be installed. + - Credentials for GSSAPI can be specified with I(url_username)/I(url_password) or with the GSSAPI env var + C(KRB5CCNAME) that specified a custom Kerberos credential cache. + - NTLM authentication is C(not) supported even if the GSSAPI mech for NTLM has been installed. + type: bool + default: no + version_added: '2.11' +''' diff --git a/lib/ansible/plugins/doc_fragments/url_windows.py b/lib/ansible/plugins/doc_fragments/url_windows.py new file mode 100644 index 0000000..286f4b4 --- /dev/null +++ b/lib/ansible/plugins/doc_fragments/url_windows.py @@ -0,0 +1,150 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2019 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +class ModuleDocFragment: + + # Common options for Ansible.ModuleUtils.WebRequest + DOCUMENTATION = r''' +options: + method: + description: + - The HTTP Method of the request. + type: str + follow_redirects: + description: + - Whether or the module should follow redirects. + - C(all) will follow all redirect. + - C(none) will not follow any redirect. + - C(safe) will follow only "safe" redirects, where "safe" means that the + client is only doing a C(GET) or C(HEAD) on the URI to which it is being + redirected. + - When following a redirected URL, the C(Authorization) header and any + credentials set will be dropped and not redirected. + choices: + - all + - none + - safe + default: safe + type: str + headers: + description: + - Extra headers to set on the request. + - This should be a dictionary where the key is the header name and the + value is the value for that header. + type: dict + http_agent: + description: + - Header to identify as, generally appears in web server logs. + - This is set to the C(User-Agent) header on a HTTP request. + default: ansible-httpget + type: str + maximum_redirection: + description: + - Specify how many times the module will redirect a connection to an + alternative URI before the connection fails. + - If set to C(0) or I(follow_redirects) is set to C(none), or C(safe) when + not doing a C(GET) or C(HEAD) it prevents all redirection. + default: 50 + type: int + timeout: + description: + - Specifies how long the request can be pending before it times out (in + seconds). + - Set to C(0) to specify an infinite timeout. + default: 30 + type: int + validate_certs: + description: + - If C(no), SSL certificates will not be validated. + - This should only be used on personally controlled sites using self-signed + certificates. + default: yes + type: bool + client_cert: + description: + - The path to the client certificate (.pfx) that is used for X509 + authentication. This path can either be the path to the C(pfx) on the + filesystem or the PowerShell certificate path + C(Cert:\CurrentUser\My\<thumbprint>). + - The WinRM connection must be authenticated with C(CredSSP) or C(become) + is used on the task if the certificate file is not password protected. + - Other authentication types can set I(client_cert_password) when the cert + is password protected. + type: str + client_cert_password: + description: + - The password for I(client_cert) if the cert is password protected. + type: str + force_basic_auth: + description: + - By default the authentication header is only sent when a webservice + responses to an initial request with a 401 status. Since some basic auth + services do not properly send a 401, logins will fail. + - This option forces the sending of the Basic authentication header upon + the original request. + default: no + type: bool + url_username: + description: + - The username to use for authentication. + type: str + url_password: + description: + - The password for I(url_username). + type: str + use_default_credential: + description: + - Uses the current user's credentials when authenticating with a server + protected with C(NTLM), C(Kerberos), or C(Negotiate) authentication. + - Sites that use C(Basic) auth will still require explicit credentials + through the I(url_username) and I(url_password) options. + - The module will only have access to the user's credentials if using + C(become) with a password, you are connecting with SSH using a password, + or connecting with WinRM using C(CredSSP) or C(Kerberos with delegation). + - If not using C(become) or a different auth method to the ones stated + above, there will be no default credentials available and no + authentication will occur. + default: no + type: bool + use_proxy: + description: + - If C(no), it will not use the proxy defined in IE for the current user. + default: yes + type: bool + proxy_url: + description: + - An explicit proxy to use for the request. + - By default, the request will use the IE defined proxy unless I(use_proxy) + is set to C(no). + type: str + proxy_username: + description: + - The username to use for proxy authentication. + type: str + proxy_password: + description: + - The password for I(proxy_username). + type: str + proxy_use_default_credential: + description: + - Uses the current user's credentials when authenticating with a proxy host + protected with C(NTLM), C(Kerberos), or C(Negotiate) authentication. + - Proxies that use C(Basic) auth will still require explicit credentials + through the I(proxy_username) and I(proxy_password) options. + - The module will only have access to the user's credentials if using + C(become) with a password, you are connecting with SSH using a password, + or connecting with WinRM using C(CredSSP) or C(Kerberos with delegation). + - If not using C(become) or a different auth method to the ones stated + above, there will be no default credentials available and no proxy + authentication will occur. + default: no + type: bool +seealso: +- module: community.windows.win_inet_proxy +''' diff --git a/lib/ansible/plugins/doc_fragments/validate.py b/lib/ansible/plugins/doc_fragments/validate.py new file mode 100644 index 0000000..ac66d25 --- /dev/null +++ b/lib/ansible/plugins/doc_fragments/validate.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- + +# Copyright: (c) 2015, Ansible, Inc +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +class ModuleDocFragment(object): + # Standard documentation fragment + DOCUMENTATION = r''' +options: + validate: + description: + - The validation command to run before copying the updated file into the final destination. + - A temporary file path is used to validate, passed in through '%s' which must be present as in the examples below. + - Also, the command is passed securely so shell features such as expansion and pipes will not work. + - For an example on how to handle more complex validation than what this + option provides, see R(handling complex validation,complex_configuration_validation). + type: str +''' diff --git a/lib/ansible/plugins/doc_fragments/vars_plugin_staging.py b/lib/ansible/plugins/doc_fragments/vars_plugin_staging.py new file mode 100644 index 0000000..b2da29c --- /dev/null +++ b/lib/ansible/plugins/doc_fragments/vars_plugin_staging.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- + +# Copyright: (c) 2019, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + + +class ModuleDocFragment(object): + + DOCUMENTATION = r''' +options: + stage: + description: + - Control when this vars plugin may be executed. + - Setting this option to C(all) will run the vars plugin after importing inventory and whenever it is demanded by a task. + - Setting this option to C(task) will only run the vars plugin whenever it is demanded by a task. + - Setting this option to C(inventory) will only run the vars plugin after parsing inventory. + - If this option is omitted, the global I(RUN_VARS_PLUGINS) configuration is used to determine when to execute the vars plugin. + choices: ['all', 'task', 'inventory'] + version_added: "2.10" + type: str +''' diff --git a/lib/ansible/plugins/filter/__init__.py b/lib/ansible/plugins/filter/__init__.py new file mode 100644 index 0000000..5ae10da --- /dev/null +++ b/lib/ansible/plugins/filter/__init__.py @@ -0,0 +1,14 @@ +# (c) Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible import constants as C +from ansible.plugins import AnsibleJinja2Plugin + + +class AnsibleJinja2Filter(AnsibleJinja2Plugin): + + def _no_options(self, *args, **kwargs): + raise NotImplementedError("Jinaj2 filter plugins do not support option functions, they use direct arguments instead.") diff --git a/lib/ansible/plugins/filter/b64decode.yml b/lib/ansible/plugins/filter/b64decode.yml new file mode 100644 index 0000000..30565fa --- /dev/null +++ b/lib/ansible/plugins/filter/b64decode.yml @@ -0,0 +1,29 @@ +DOCUMENTATION: + name: b64decode + author: ansible core team + version_added: 'historical' + short_description: Decode a base64 string + description: + - Base64 decoding function. + - The return value is a string. + - Trying to store a binary blob in a string most likely corrupts the binary. To base64 decode a binary blob, + use the ``base64`` command and pipe the encoded data through standard input. + For example, in the ansible.builtin.shell`` module, ``cmd="base64 --decode > myfile.bin" stdin="{{ encoded }}"``. + positional: _input + options: + _input: + description: A base64 string to decode. + type: string + required: true + +EXAMPLES: | + # b64 decode a string + lola: "{{ 'bG9sYQ==' | b64decode }}" + + # b64 decode the content of 'b64stuff' variable + stuff: "{{ b64stuff | b64encode }}" + +RETURN: + _value: + description: The contents of the base64 encoded string. + type: string diff --git a/lib/ansible/plugins/filter/b64encode.yml b/lib/ansible/plugins/filter/b64encode.yml new file mode 100644 index 0000000..14676e5 --- /dev/null +++ b/lib/ansible/plugins/filter/b64encode.yml @@ -0,0 +1,25 @@ +DOCUMENTATION: + name: b64encode + author: ansible core team + version_added: 'historical' + short_description: Encode a string as base64 + description: + - Base64 encoding function. + positional: _input + options: + _input: + description: A string to encode. + type: string + required: true + +EXAMPLES: | + # b64 encode a string + b64lola: "{{ 'lola'|b64encode }}" + + # b64 encode the content of 'stuff' variable + b64stuff: "{{ stuff|b64encode }}" + +RETURN: + _value: + description: A base64 encoded string. + type: string diff --git a/lib/ansible/plugins/filter/basename.yml b/lib/ansible/plugins/filter/basename.yml new file mode 100644 index 0000000..4e868df --- /dev/null +++ b/lib/ansible/plugins/filter/basename.yml @@ -0,0 +1,24 @@ +DOCUMENTATION: + name: basename + author: ansible core team + version_added: "historical" + short_description: get a path's base name + description: + - Returns the last name component of a path, what is left in the string that is not 'dirname'. + options: + _input: + description: A path. + type: path + required: true + seealso: + - plugin_type: filter + plugin: ansible.builtin.dirname +EXAMPLES: | + + # To get the last name of a file path, like 'foo.txt' out of '/etc/asdf/foo.txt' + {{ mypath | basename }} + +RETURN: + _value: + description: The base name from the path provided. + type: str diff --git a/lib/ansible/plugins/filter/bool.yml b/lib/ansible/plugins/filter/bool.yml new file mode 100644 index 0000000..86ba353 --- /dev/null +++ b/lib/ansible/plugins/filter/bool.yml @@ -0,0 +1,28 @@ +DOCUMENTATION: + name: bool + version_added: "historical" + short_description: cast into a boolean + description: + - Attempt to cast the input into a boolean (C(True) or C(False)) value. + positional: _input + options: + _input: + description: Data to cast. + type: raw + required: true + +EXAMPLES: | + + # simply encrypt my key in a vault + vars: + isbool: "{{ (a == b)|bool }} " + otherbool: "{{ anothervar|bool }} " + + # in a task + ... + when: some_string_value | bool + +RETURN: + _value: + description: The boolean resulting of casting the input expression into a C(True) or C(False) value. + type: bool diff --git a/lib/ansible/plugins/filter/checksum.yml b/lib/ansible/plugins/filter/checksum.yml new file mode 100644 index 0000000..2f8eadd --- /dev/null +++ b/lib/ansible/plugins/filter/checksum.yml @@ -0,0 +1,21 @@ +DOCUMENTATION: + name: checksum + version_added: "1.9" + short_description: checksum of input data + description: + - Returns a checksum (L(SHA-1, https://en.wikipedia.org/wiki/SHA-1)) hash of the input data. + positional: _input + options: + _input: + description: Data to checksum. + type: raw + required: true + +EXAMPLES: | + # csum => "109f4b3c50d7b0df729d299bc6f8e9ef9066971f" + csum: "{{ 'test2' | checksum }}" + +RETURN: + _value: + description: The checksum (SHA-1) of the input. + type: string diff --git a/lib/ansible/plugins/filter/combinations.yml b/lib/ansible/plugins/filter/combinations.yml new file mode 100644 index 0000000..a46e51e --- /dev/null +++ b/lib/ansible/plugins/filter/combinations.yml @@ -0,0 +1,26 @@ +DOCUMENTATION: + name: combinations + version_added: "historical" + short_description: combinations from the elements of a list + description: + - Create a list of combinations of sets from the elements of a list. + positional: _input, set_size + options: + _input: + description: Elements to combine. + type: list + required: true + set_size: + description: The size of the set for each combination. + type: int + required: true +EXAMPLES: | + + # combos_of_two => [ [ 1, 2 ], [ 1, 3 ], [ 1, 4 ], [ 1, 5 ], [ 2, 3 ], [ 2, 4 ], [ 2, 5 ], [ 3, 4 ], [ 3, 5 ], [ 4, 5 ] ] + combos_of_two: "{{ [1,2,3,4,5] | combinations(2) }}" + + +RETURN: + _value: + description: List of combination sets resulting from the supplied elements and set size. + type: list diff --git a/lib/ansible/plugins/filter/combine.yml b/lib/ansible/plugins/filter/combine.yml new file mode 100644 index 0000000..86788f3 --- /dev/null +++ b/lib/ansible/plugins/filter/combine.yml @@ -0,0 +1,44 @@ +DOCUMENTATION: + name: combine + version_added: "2.0" + short_description: combine two dictionaries + description: + - Create a dictionary (hash/associative array) as a result of merging existing dictionaries. + positional: _input, _dicts + options: + _input: + description: First dictionary to combine. + type: dict + required: true + _dicts: # TODO: this is really an *args so not list, but list ref + description: The list of dictionaries to combine. + type: list + elements: dictionary + required: true + recursive: + description: If C(True), merge elements recursively. + type: bool + default: false + list_merge: + description: Behavior when encountering list elements. + type: str + default: replace + choices: + replace: overwrite older entries with newer ones + keep: discard newer entries + append: append newer entries to the older ones + prepend: insert newer entries in front of the older ones + append_rp: append newer entries to the older ones, overwrite duplicates + prepend_rp: insert newer entries in front of the older ones, discard duplicates + +EXAMPLES: | + + # ab => {'a':1, 'b':3, 'c': 4} + ab: {{ {'a':1, 'b':2} | combine({'b':3, 'c':4}) }} + + many: "{{ dict1 | combine(dict2, dict3, dict4) }}" + +RETURN: + _value: + description: Resulting merge of supplied dictionaries. + type: dict diff --git a/lib/ansible/plugins/filter/comment.yml b/lib/ansible/plugins/filter/comment.yml new file mode 100644 index 0000000..95a4efb --- /dev/null +++ b/lib/ansible/plugins/filter/comment.yml @@ -0,0 +1,60 @@ +DOCUMENTATION: + name: comment + version_added: 'historical' + short_description: comment out a string + description: + - Use programming language conventions to turn the input string into an embeddable comment. + positional: _input, style + options: + _input: + description: String to comment. + type: string + required: true + style: + description: Comment style to use. + type: string + default: plain + choices: ['plain', 'decoration', 'erlang', 'c', 'cblock', 'xml'] + decoration: + description: Indicator for comment or intermediate comment depending on the style. + type: string + begining: + description: Indicator of the start of a comment block, only available for styles that support multiline comments. + type: string + end: + description: Indicator the end of a comment block, only available for styles that support multiline comments. + type: string + newline: + description: Indicator of comment end of line, only available for styles that support multiline comments. + type: string + default: '\n' + prefix: + description: Token to start each line inside a comment block, only available for styles that support multiline comments. + type: string + prefix_count: + description: Number of times to add a prefix at the start of a line, when a prefix exists and is usable. + type: int + default: 1 + postfix: + description: Indicator of the end of each line inside a comment block, only available for styles that support multiline comments. + type: string + protfix_count: + description: Number of times to add a postfix at the end of a line, when a prefix exists and is usable. + type: int + default: 1 + +EXAMPLES: | + + # commented => # + # # Plain style (default) + # # + commented: "{{ 'Plain style (default)' | comment }}" + + # not going to show that here ... + verycustom: "{{ "Custom style" | comment('plain', prefix='#######\n#', postfix='#\n#######\n ###\n #') }}" + + +RETURN: + _value: + description: The 'commented out' string. + type: string diff --git a/lib/ansible/plugins/filter/core.py b/lib/ansible/plugins/filter/core.py new file mode 100644 index 0000000..52a2cd1 --- /dev/null +++ b/lib/ansible/plugins/filter/core.py @@ -0,0 +1,658 @@ +# (c) 2012, Jeroen Hoekx <jeroen@hoekx.be> +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import base64 +import glob +import hashlib +import json +import ntpath +import os.path +import re +import shlex +import sys +import time +import uuid +import yaml +import datetime + +from collections.abc import Mapping +from functools import partial +from random import Random, SystemRandom, shuffle + +from jinja2.filters import pass_environment + +from ansible.errors import AnsibleError, AnsibleFilterError, AnsibleFilterTypeError +from ansible.module_utils.six import string_types, integer_types, reraise, text_type +from ansible.module_utils._text import to_bytes, to_native, to_text +from ansible.module_utils.common.collections import is_sequence +from ansible.module_utils.common.yaml import yaml_load, yaml_load_all +from ansible.parsing.ajson import AnsibleJSONEncoder +from ansible.parsing.yaml.dumper import AnsibleDumper +from ansible.template import recursive_check_defined +from ansible.utils.display import Display +from ansible.utils.encrypt import passlib_or_crypt +from ansible.utils.hashing import md5s, checksum_s +from ansible.utils.unicode import unicode_wrap +from ansible.utils.vars import merge_hash + +display = Display() + +UUID_NAMESPACE_ANSIBLE = uuid.UUID('361E6D51-FAEC-444A-9079-341386DA8E2E') + + +def to_yaml(a, *args, **kw): + '''Make verbose, human readable yaml''' + default_flow_style = kw.pop('default_flow_style', None) + try: + transformed = yaml.dump(a, Dumper=AnsibleDumper, allow_unicode=True, default_flow_style=default_flow_style, **kw) + except Exception as e: + raise AnsibleFilterError("to_yaml - %s" % to_native(e), orig_exc=e) + return to_text(transformed) + + +def to_nice_yaml(a, indent=4, *args, **kw): + '''Make verbose, human readable yaml''' + try: + transformed = yaml.dump(a, Dumper=AnsibleDumper, indent=indent, allow_unicode=True, default_flow_style=False, **kw) + except Exception as e: + raise AnsibleFilterError("to_nice_yaml - %s" % to_native(e), orig_exc=e) + return to_text(transformed) + + +def to_json(a, *args, **kw): + ''' Convert the value to JSON ''' + + # defaults for filters + if 'vault_to_text' not in kw: + kw['vault_to_text'] = True + if 'preprocess_unsafe' not in kw: + kw['preprocess_unsafe'] = False + + return json.dumps(a, cls=AnsibleJSONEncoder, *args, **kw) + + +def to_nice_json(a, indent=4, sort_keys=True, *args, **kw): + '''Make verbose, human readable JSON''' + return to_json(a, indent=indent, sort_keys=sort_keys, separators=(',', ': '), *args, **kw) + + +def to_bool(a): + ''' return a bool for the arg ''' + if a is None or isinstance(a, bool): + return a + if isinstance(a, string_types): + a = a.lower() + if a in ('yes', 'on', '1', 'true', 1): + return True + return False + + +def to_datetime(string, format="%Y-%m-%d %H:%M:%S"): + return datetime.datetime.strptime(string, format) + + +def strftime(string_format, second=None, utc=False): + ''' return a date string using string. See https://docs.python.org/3/library/time.html#time.strftime for format ''' + if utc: + timefn = time.gmtime + else: + timefn = time.localtime + if second is not None: + try: + second = float(second) + except Exception: + raise AnsibleFilterError('Invalid value for epoch value (%s)' % second) + return time.strftime(string_format, timefn(second)) + + +def quote(a): + ''' return its argument quoted for shell usage ''' + if a is None: + a = u'' + return shlex.quote(to_text(a)) + + +def fileglob(pathname): + ''' return list of matched regular files for glob ''' + return [g for g in glob.glob(pathname) if os.path.isfile(g)] + + +def regex_replace(value='', pattern='', replacement='', ignorecase=False, multiline=False): + ''' Perform a `re.sub` returning a string ''' + + value = to_text(value, errors='surrogate_or_strict', nonstring='simplerepr') + + flags = 0 + if ignorecase: + flags |= re.I + if multiline: + flags |= re.M + _re = re.compile(pattern, flags=flags) + return _re.sub(replacement, value) + + +def regex_findall(value, regex, multiline=False, ignorecase=False): + ''' Perform re.findall and return the list of matches ''' + + value = to_text(value, errors='surrogate_or_strict', nonstring='simplerepr') + + flags = 0 + if ignorecase: + flags |= re.I + if multiline: + flags |= re.M + return re.findall(regex, value, flags) + + +def regex_search(value, regex, *args, **kwargs): + ''' Perform re.search and return the list of matches or a backref ''' + + value = to_text(value, errors='surrogate_or_strict', nonstring='simplerepr') + + groups = list() + for arg in args: + if arg.startswith('\\g'): + match = re.match(r'\\g<(\S+)>', arg).group(1) + groups.append(match) + elif arg.startswith('\\'): + match = int(re.match(r'\\(\d+)', arg).group(1)) + groups.append(match) + else: + raise AnsibleFilterError('Unknown argument') + + flags = 0 + if kwargs.get('ignorecase'): + flags |= re.I + if kwargs.get('multiline'): + flags |= re.M + + match = re.search(regex, value, flags) + if match: + if not groups: + return match.group() + else: + items = list() + for item in groups: + items.append(match.group(item)) + return items + + +def ternary(value, true_val, false_val, none_val=None): + ''' value ? true_val : false_val ''' + if value is None and none_val is not None: + return none_val + elif bool(value): + return true_val + else: + return false_val + + +def regex_escape(string, re_type='python'): + string = to_text(string, errors='surrogate_or_strict', nonstring='simplerepr') + '''Escape all regular expressions special characters from STRING.''' + if re_type == 'python': + return re.escape(string) + elif re_type == 'posix_basic': + # list of BRE special chars: + # https://en.wikibooks.org/wiki/Regular_Expressions/POSIX_Basic_Regular_Expressions + return regex_replace(string, r'([].[^$*\\])', r'\\\1') + # TODO: implement posix_extended + # It's similar to, but different from python regex, which is similar to, + # but different from PCRE. It's possible that re.escape would work here. + # https://remram44.github.io/regex-cheatsheet/regex.html#programs + elif re_type == 'posix_extended': + raise AnsibleFilterError('Regex type (%s) not yet implemented' % re_type) + else: + raise AnsibleFilterError('Invalid regex type (%s)' % re_type) + + +def from_yaml(data): + if isinstance(data, string_types): + # The ``text_type`` call here strips any custom + # string wrapper class, so that CSafeLoader can + # read the data + return yaml_load(text_type(to_text(data, errors='surrogate_or_strict'))) + return data + + +def from_yaml_all(data): + if isinstance(data, string_types): + # The ``text_type`` call here strips any custom + # string wrapper class, so that CSafeLoader can + # read the data + return yaml_load_all(text_type(to_text(data, errors='surrogate_or_strict'))) + return data + + +@pass_environment +def rand(environment, end, start=None, step=None, seed=None): + if seed is None: + r = SystemRandom() + else: + r = Random(seed) + if isinstance(end, integer_types): + if not start: + start = 0 + if not step: + step = 1 + return r.randrange(start, end, step) + elif hasattr(end, '__iter__'): + if start or step: + raise AnsibleFilterError('start and step can only be used with integer values') + return r.choice(end) + else: + raise AnsibleFilterError('random can only be used on sequences and integers') + + +def randomize_list(mylist, seed=None): + try: + mylist = list(mylist) + if seed: + r = Random(seed) + r.shuffle(mylist) + else: + shuffle(mylist) + except Exception: + pass + return mylist + + +def get_hash(data, hashtype='sha1'): + try: + h = hashlib.new(hashtype) + except Exception as e: + # hash is not supported? + raise AnsibleFilterError(e) + + h.update(to_bytes(data, errors='surrogate_or_strict')) + return h.hexdigest() + + +def get_encrypted_password(password, hashtype='sha512', salt=None, salt_size=None, rounds=None, ident=None): + passlib_mapping = { + 'md5': 'md5_crypt', + 'blowfish': 'bcrypt', + 'sha256': 'sha256_crypt', + 'sha512': 'sha512_crypt', + } + + hashtype = passlib_mapping.get(hashtype, hashtype) + try: + return passlib_or_crypt(password, hashtype, salt=salt, salt_size=salt_size, rounds=rounds, ident=ident) + except AnsibleError as e: + reraise(AnsibleFilterError, AnsibleFilterError(to_native(e), orig_exc=e), sys.exc_info()[2]) + + +def to_uuid(string, namespace=UUID_NAMESPACE_ANSIBLE): + uuid_namespace = namespace + if not isinstance(uuid_namespace, uuid.UUID): + try: + uuid_namespace = uuid.UUID(namespace) + except (AttributeError, ValueError) as e: + raise AnsibleFilterError("Invalid value '%s' for 'namespace': %s" % (to_native(namespace), to_native(e))) + # uuid.uuid5() requires bytes on Python 2 and bytes or text or Python 3 + return to_text(uuid.uuid5(uuid_namespace, to_native(string, errors='surrogate_or_strict'))) + + +def mandatory(a, msg=None): + from jinja2.runtime import Undefined + + ''' Make a variable mandatory ''' + if isinstance(a, Undefined): + if a._undefined_name is not None: + name = "'%s' " % to_text(a._undefined_name) + else: + name = '' + + if msg is not None: + raise AnsibleFilterError(to_native(msg)) + else: + raise AnsibleFilterError("Mandatory variable %s not defined." % name) + + return a + + +def combine(*terms, **kwargs): + recursive = kwargs.pop('recursive', False) + list_merge = kwargs.pop('list_merge', 'replace') + if kwargs: + raise AnsibleFilterError("'recursive' and 'list_merge' are the only valid keyword arguments") + + # allow the user to do `[dict1, dict2, ...] | combine` + dictionaries = flatten(terms, levels=1) + + # recursively check that every elements are defined (for jinja2) + recursive_check_defined(dictionaries) + + if not dictionaries: + return {} + + if len(dictionaries) == 1: + return dictionaries[0] + + # merge all the dicts so that the dict at the end of the array have precedence + # over the dict at the beginning. + # we merge the dicts from the highest to the lowest priority because there is + # a huge probability that the lowest priority dict will be the biggest in size + # (as the low prio dict will hold the "default" values and the others will be "patches") + # and merge_hash create a copy of it's first argument. + # so high/right -> low/left is more efficient than low/left -> high/right + high_to_low_prio_dict_iterator = reversed(dictionaries) + result = next(high_to_low_prio_dict_iterator) + for dictionary in high_to_low_prio_dict_iterator: + result = merge_hash(dictionary, result, recursive, list_merge) + + return result + + +def comment(text, style='plain', **kw): + # Predefined comment types + comment_styles = { + 'plain': { + 'decoration': '# ' + }, + 'erlang': { + 'decoration': '% ' + }, + 'c': { + 'decoration': '// ' + }, + 'cblock': { + 'beginning': '/*', + 'decoration': ' * ', + 'end': ' */' + }, + 'xml': { + 'beginning': '<!--', + 'decoration': ' - ', + 'end': '-->' + } + } + + # Pointer to the right comment type + style_params = comment_styles[style] + + if 'decoration' in kw: + prepostfix = kw['decoration'] + else: + prepostfix = style_params['decoration'] + + # Default params + p = { + 'newline': '\n', + 'beginning': '', + 'prefix': (prepostfix).rstrip(), + 'prefix_count': 1, + 'decoration': '', + 'postfix': (prepostfix).rstrip(), + 'postfix_count': 1, + 'end': '' + } + + # Update default params + p.update(style_params) + p.update(kw) + + # Compose substrings for the final string + str_beginning = '' + if p['beginning']: + str_beginning = "%s%s" % (p['beginning'], p['newline']) + str_prefix = '' + if p['prefix']: + if p['prefix'] != p['newline']: + str_prefix = str( + "%s%s" % (p['prefix'], p['newline'])) * int(p['prefix_count']) + else: + str_prefix = str( + "%s" % (p['newline'])) * int(p['prefix_count']) + str_text = ("%s%s" % ( + p['decoration'], + # Prepend each line of the text with the decorator + text.replace( + p['newline'], "%s%s" % (p['newline'], p['decoration'])))).replace( + # Remove trailing spaces when only decorator is on the line + "%s%s" % (p['decoration'], p['newline']), + "%s%s" % (p['decoration'].rstrip(), p['newline'])) + str_postfix = p['newline'].join( + [''] + [p['postfix'] for x in range(p['postfix_count'])]) + str_end = '' + if p['end']: + str_end = "%s%s" % (p['newline'], p['end']) + + # Return the final string + return "%s%s%s%s%s" % ( + str_beginning, + str_prefix, + str_text, + str_postfix, + str_end) + + +@pass_environment +def extract(environment, item, container, morekeys=None): + if morekeys is None: + keys = [item] + elif isinstance(morekeys, list): + keys = [item] + morekeys + else: + keys = [item, morekeys] + + value = container + for key in keys: + value = environment.getitem(value, key) + + return value + + +def b64encode(string, encoding='utf-8'): + return to_text(base64.b64encode(to_bytes(string, encoding=encoding, errors='surrogate_or_strict'))) + + +def b64decode(string, encoding='utf-8'): + return to_text(base64.b64decode(to_bytes(string, errors='surrogate_or_strict')), encoding=encoding) + + +def flatten(mylist, levels=None, skip_nulls=True): + + ret = [] + for element in mylist: + if skip_nulls and element in (None, 'None', 'null'): + # ignore null items + continue + elif is_sequence(element): + if levels is None: + ret.extend(flatten(element, skip_nulls=skip_nulls)) + elif levels >= 1: + # decrement as we go down the stack + ret.extend(flatten(element, levels=(int(levels) - 1), skip_nulls=skip_nulls)) + else: + ret.append(element) + else: + ret.append(element) + + return ret + + +def subelements(obj, subelements, skip_missing=False): + '''Accepts a dict or list of dicts, and a dotted accessor and produces a product + of the element and the results of the dotted accessor + + >>> obj = [{"name": "alice", "groups": ["wheel"], "authorized": ["/tmp/alice/onekey.pub"]}] + >>> subelements(obj, 'groups') + [({'name': 'alice', 'groups': ['wheel'], 'authorized': ['/tmp/alice/onekey.pub']}, 'wheel')] + + ''' + if isinstance(obj, dict): + element_list = list(obj.values()) + elif isinstance(obj, list): + element_list = obj[:] + else: + raise AnsibleFilterError('obj must be a list of dicts or a nested dict') + + if isinstance(subelements, list): + subelement_list = subelements[:] + elif isinstance(subelements, string_types): + subelement_list = subelements.split('.') + else: + raise AnsibleFilterTypeError('subelements must be a list or a string') + + results = [] + + for element in element_list: + values = element + for subelement in subelement_list: + try: + values = values[subelement] + except KeyError: + if skip_missing: + values = [] + break + raise AnsibleFilterError("could not find %r key in iterated item %r" % (subelement, values)) + except TypeError: + raise AnsibleFilterTypeError("the key %s should point to a dictionary, got '%s'" % (subelement, values)) + if not isinstance(values, list): + raise AnsibleFilterTypeError("the key %r should point to a list, got %r" % (subelement, values)) + + for value in values: + results.append((element, value)) + + return results + + +def dict_to_list_of_dict_key_value_elements(mydict, key_name='key', value_name='value'): + ''' takes a dictionary and transforms it into a list of dictionaries, + with each having a 'key' and 'value' keys that correspond to the keys and values of the original ''' + + if not isinstance(mydict, Mapping): + raise AnsibleFilterTypeError("dict2items requires a dictionary, got %s instead." % type(mydict)) + + ret = [] + for key in mydict: + ret.append({key_name: key, value_name: mydict[key]}) + return ret + + +def list_of_dict_key_value_elements_to_dict(mylist, key_name='key', value_name='value'): + ''' takes a list of dicts with each having a 'key' and 'value' keys, and transforms the list into a dictionary, + effectively as the reverse of dict2items ''' + + if not is_sequence(mylist): + raise AnsibleFilterTypeError("items2dict requires a list, got %s instead." % type(mylist)) + + try: + return dict((item[key_name], item[value_name]) for item in mylist) + except KeyError: + raise AnsibleFilterTypeError( + "items2dict requires each dictionary in the list to contain the keys '%s' and '%s', got %s instead." + % (key_name, value_name, mylist) + ) + except TypeError: + raise AnsibleFilterTypeError("items2dict requires a list of dictionaries, got %s instead." % mylist) + + +def path_join(paths): + ''' takes a sequence or a string, and return a concatenation + of the different members ''' + if isinstance(paths, string_types): + return os.path.join(paths) + elif is_sequence(paths): + return os.path.join(*paths) + else: + raise AnsibleFilterTypeError("|path_join expects string or sequence, got %s instead." % type(paths)) + + +class FilterModule(object): + ''' Ansible core jinja2 filters ''' + + def filters(self): + return { + # base 64 + 'b64decode': b64decode, + 'b64encode': b64encode, + + # uuid + 'to_uuid': to_uuid, + + # json + 'to_json': to_json, + 'to_nice_json': to_nice_json, + 'from_json': json.loads, + + # yaml + 'to_yaml': to_yaml, + 'to_nice_yaml': to_nice_yaml, + 'from_yaml': from_yaml, + 'from_yaml_all': from_yaml_all, + + # path + 'basename': partial(unicode_wrap, os.path.basename), + 'dirname': partial(unicode_wrap, os.path.dirname), + 'expanduser': partial(unicode_wrap, os.path.expanduser), + 'expandvars': partial(unicode_wrap, os.path.expandvars), + 'path_join': path_join, + 'realpath': partial(unicode_wrap, os.path.realpath), + 'relpath': partial(unicode_wrap, os.path.relpath), + 'splitext': partial(unicode_wrap, os.path.splitext), + 'win_basename': partial(unicode_wrap, ntpath.basename), + 'win_dirname': partial(unicode_wrap, ntpath.dirname), + 'win_splitdrive': partial(unicode_wrap, ntpath.splitdrive), + + # file glob + 'fileglob': fileglob, + + # types + 'bool': to_bool, + 'to_datetime': to_datetime, + + # date formatting + 'strftime': strftime, + + # quote string for shell usage + 'quote': quote, + + # hash filters + # md5 hex digest of string + 'md5': md5s, + # sha1 hex digest of string + 'sha1': checksum_s, + # checksum of string as used by ansible for checksumming files + 'checksum': checksum_s, + # generic hashing + 'password_hash': get_encrypted_password, + 'hash': get_hash, + + # regex + 'regex_replace': regex_replace, + 'regex_escape': regex_escape, + 'regex_search': regex_search, + 'regex_findall': regex_findall, + + # ? : ; + 'ternary': ternary, + + # random stuff + 'random': rand, + 'shuffle': randomize_list, + + # undefined + 'mandatory': mandatory, + + # comment-style decoration + 'comment': comment, + + # debug + 'type_debug': lambda o: o.__class__.__name__, + + # Data structures + 'combine': combine, + 'extract': extract, + 'flatten': flatten, + 'dict2items': dict_to_list_of_dict_key_value_elements, + 'items2dict': list_of_dict_key_value_elements_to_dict, + 'subelements': subelements, + 'split': partial(unicode_wrap, text_type.split), + } diff --git a/lib/ansible/plugins/filter/dict2items.yml b/lib/ansible/plugins/filter/dict2items.yml new file mode 100644 index 0000000..aa51826 --- /dev/null +++ b/lib/ansible/plugins/filter/dict2items.yml @@ -0,0 +1,45 @@ +DOCUMENTATION: + name: dict2items + author: Ansible core team + version_added: "2.6" + short_description: Convert a dictionary into an itemized list of dictionaries + positional: _input, key_name, value_name + description: + - Takes a dictionary and transforms it into a list of dictionaries, with each having a + C(key) and C(value) keys that correspond to the keys and values of the original. + options: + _input: + description: + - The dictionary to transform + type: dict + required: true + key_name: + description: The name of the property on the item representing the dictionary's keys. + type: str + default: key + version_added: "2.8" + value_name: + description: The name of the property on the item representing the dictionary's values. + type: str + default: value + version_added: "2.8" + seealso: + - plugin_type: filter + plugin: ansible.builtin.items2dict + +EXAMPLES: | + + # items => [ { "key": "a", "value": 1 }, { "key": "b", "value": 2 } ] + items: "{{ {'a': 1, 'b': 2}| dict2items}}" + + vars: + files: + users: /etc/passwd + groups: /etc/group + files_dicts: "{{ files | dict2items(key_name='file', value_name='path') }}" + +RETURN: + _value: + description: A list of dictionaries. + type: list + elements: dict diff --git a/lib/ansible/plugins/filter/difference.yml b/lib/ansible/plugins/filter/difference.yml new file mode 100644 index 0000000..decc811 --- /dev/null +++ b/lib/ansible/plugins/filter/difference.yml @@ -0,0 +1,35 @@ +DOCUMENTATION: + name: difference + author: Brian Coca (@bcoca) + version_added: "1.4" + short_description: the difference of one list from another + description: + - Provide a unique list of all the elements of the first list that do not appear in the second one. + options: + _input: + description: A list. + type: list + required: true + _second_list: + description: A list. + type: list + required: true + seealso: + - plugin_type: filter + plugin: ansible.builtin.intersect + - plugin_type: filter + plugin: ansible.builtin.symmetric_difference + - plugin_type: filter + plugin: ansible.builtin.union + - plugin_type: filter + plugin: ansible.builtin.unique +EXAMPLES: | + # return the elements of list1 not in list2 + # list1: [1, 2, 5, 1, 3, 4, 10] + # list2: [1, 2, 3, 4, 5, 11, 99] + {{ list1 | difference(list2) }} + # => [10] +RETURN: + _value: + description: A unique list of the elements from the first list that do not appear on the second. + type: list diff --git a/lib/ansible/plugins/filter/dirname.yml b/lib/ansible/plugins/filter/dirname.yml new file mode 100644 index 0000000..52f7d5d --- /dev/null +++ b/lib/ansible/plugins/filter/dirname.yml @@ -0,0 +1,24 @@ +DOCUMENTATION: + name: dirname + author: ansible core team + version_added: "historical" + short_description: get a path's directory name + description: + - Returns the 'head' component of a path, basically everything that is not the 'basename'. + options: + _input: + description: A path. + type: path + required: true + seealso: + - plugin: ansible.builtin.basename + plugin_type: filter +EXAMPLES: | + + # To get the dir name of a file path, like '/etc/asdf' out of '/etc/asdf/foo.txt' + {{ mypath | dirname }} + +RETURN: + _value: + description: The directory portion of the original path. + type: path diff --git a/lib/ansible/plugins/filter/encryption.py b/lib/ansible/plugins/filter/encryption.py new file mode 100644 index 0000000..b6f4961 --- /dev/null +++ b/lib/ansible/plugins/filter/encryption.py @@ -0,0 +1,82 @@ +# Copyright: (c) 2021, Ansible Project + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from jinja2.runtime import Undefined +from jinja2.exceptions import UndefinedError + +from ansible.errors import AnsibleFilterError, AnsibleFilterTypeError +from ansible.module_utils._text import to_native, to_bytes +from ansible.module_utils.six import string_types, binary_type +from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode +from ansible.parsing.vault import is_encrypted, VaultSecret, VaultLib +from ansible.utils.display import Display + +display = Display() + + +def do_vault(data, secret, salt=None, vaultid='filter_default', wrap_object=False): + + if not isinstance(secret, (string_types, binary_type, Undefined)): + raise AnsibleFilterTypeError("Secret passed is required to be a string, instead we got: %s" % type(secret)) + + if not isinstance(data, (string_types, binary_type, Undefined)): + raise AnsibleFilterTypeError("Can only vault strings, instead we got: %s" % type(data)) + + vault = '' + vs = VaultSecret(to_bytes(secret)) + vl = VaultLib() + try: + vault = vl.encrypt(to_bytes(data), vs, vaultid, salt) + except UndefinedError: + raise + except Exception as e: + raise AnsibleFilterError("Unable to encrypt: %s" % to_native(e), orig_exc=e) + + if wrap_object: + vault = AnsibleVaultEncryptedUnicode(vault) + else: + vault = to_native(vault) + + return vault + + +def do_unvault(vault, secret, vaultid='filter_default'): + + if not isinstance(secret, (string_types, binary_type, Undefined)): + raise AnsibleFilterTypeError("Secret passed is required to be as string, instead we got: %s" % type(secret)) + + if not isinstance(vault, (string_types, binary_type, AnsibleVaultEncryptedUnicode, Undefined)): + raise AnsibleFilterTypeError("Vault should be in the form of a string, instead we got: %s" % type(vault)) + + data = '' + vs = VaultSecret(to_bytes(secret)) + vl = VaultLib([(vaultid, vs)]) + if isinstance(vault, AnsibleVaultEncryptedUnicode): + vault.vault = vl + data = vault.data + elif is_encrypted(vault): + try: + data = vl.decrypt(vault) + except UndefinedError: + raise + except Exception as e: + raise AnsibleFilterError("Unable to decrypt: %s" % to_native(e), orig_exc=e) + else: + data = vault + + return to_native(data) + + +class FilterModule(object): + ''' Ansible vault jinja2 filters ''' + + def filters(self): + filters = { + 'vault': do_vault, + 'unvault': do_unvault, + } + + return filters diff --git a/lib/ansible/plugins/filter/expanduser.yml b/lib/ansible/plugins/filter/expanduser.yml new file mode 100644 index 0000000..2aff468 --- /dev/null +++ b/lib/ansible/plugins/filter/expanduser.yml @@ -0,0 +1,21 @@ +DOCUMENTATION: + name: basename + author: ansible core team + version_added: "1.5" + short_description: Returns a path with C(~) translation. + description: + - Translates C(~) in a path to the proper user's home directory. + options: + _input: + description: A string that contains a path. + type: path + required: true +EXAMPLES: | + + # To get '/home/myuser/stuff.txt' from '~/stuff.txt'. + {{ mypath | expanduser }} + +RETURN: + _value: + description: The translated path. + type: path diff --git a/lib/ansible/plugins/filter/expandvars.yml b/lib/ansible/plugins/filter/expandvars.yml new file mode 100644 index 0000000..02c201e --- /dev/null +++ b/lib/ansible/plugins/filter/expandvars.yml @@ -0,0 +1,21 @@ +DOCUMENTATION: + name: expandvars + author: ansible core team + version_added: "1.5" + short_description: expand environment variables + description: + - Will do a shell-like substitution of environment variables on the provided input. + options: + _input: + description: A string that contains environment variables. + type: str + required: true +EXAMPLES: | + + # To get '/home/myuser/stuff.txt' from '$HOME/stuff.txt' + {{ mypath | expandvars }} + +RETURN: + _value: + description: The string with translated environment variable values. + type: str diff --git a/lib/ansible/plugins/filter/extract.yml b/lib/ansible/plugins/filter/extract.yml new file mode 100644 index 0000000..2b4989d --- /dev/null +++ b/lib/ansible/plugins/filter/extract.yml @@ -0,0 +1,39 @@ +DOCUMENTATION: + name: extract + version_added: "2.1" + short_description: extract a value based on an index or key + description: + - Extract a value from a list or dictionary based on an index/key. + - User must ensure that index or key used matches the type of container. + - Equivalent of using C(list[index]) and C(dictionary[key]) but useful as a filter to combine with C(map). + positional: _input, container, morekeys + options: + _input: + description: Index or key to extract. + type: raw + required: true + contianer: + description: Dictionary or list from which to extract a value. + type: raw + required: true + morekeys: + description: Indicies or keys to extract from the initial result (subkeys/subindices). + type: list + elements: dictionary + required: true + +EXAMPLES: | + + # extracted => 'b', same as ['a', 'b', 'c'][1] + extracted: "{{ 1 | extract(['a', 'b', 'c']) }}" + + # extracted_key => '2', same as {'a': 1, 'b': 2, 'c': 3}['b'] + extracted_key: "{{ 'b' | extract({'a': 1, 'b': 2, 'c': 3}) }}" + + # extracted_key_r => '2', same as [{'a': 1, 'b': 2, 'c': 3}, {'x': 9, 'y': 10}][0]['b'] + extracted_key_r: "{{ 0 | extract([{'a': 1, 'b': 2, 'c': 3}, {'x': 9, 'y': 10}], morekeys='b') }}" + +RETURN: + _value: + description: Resulting merge of supplied dictionaries. + type: dict diff --git a/lib/ansible/plugins/filter/fileglob.yml b/lib/ansible/plugins/filter/fileglob.yml new file mode 100644 index 0000000..69e8a9b --- /dev/null +++ b/lib/ansible/plugins/filter/fileglob.yml @@ -0,0 +1,22 @@ +DOCUMENTATION: + name: fileglob + short_description: explode a path glob to matching files + description: + - Return a list of files that matches the supplied path glob pattern. + - Filters run on the controller, so the files are matched from the controller's file system. + positional: _input + options: + _input: + description: Path glob pattern. + type: string + required: true + +EXAMPLES: | + # found = ['/etc/hosts', '/etc/hasts'] + found: "{{ '/etc/h?sts' | fileglob }}" + +RETURN: + _value: + description: List of files matched. + type: list + elements: string diff --git a/lib/ansible/plugins/filter/flatten.yml b/lib/ansible/plugins/filter/flatten.yml new file mode 100644 index 0000000..b909c3d --- /dev/null +++ b/lib/ansible/plugins/filter/flatten.yml @@ -0,0 +1,32 @@ +DOCUMENTATION: + name: flatten + version_added: "2.5" + short_description: flatten lists within a list + description: + - For a given list, take any elements that are lists and insert their elements into the parent list directly. + positional: _input, levels, skip_nulls + options: + _input: + description: First dictionary to combine. + type: dict + required: true + levels: + description: Number of recursive list depths to flatten. + type: int + skip_nulls: + description: Skip C(null)/C(None) elements when inserting into the top list. + type: bool + default: true + +EXAMPLES: | + + # [1,2,3,4,5,6] + flat: "{{ [1 , 2, [3, [4, 5]], 6] | flatten }}" + + # [1,2,3,[4,5],6] + flatone: "{{ [1, 2, [3, [4, 5]], 6] | flatten(1) }}" + +RETURN: + _value: + description: The flattened list. + type: list diff --git a/lib/ansible/plugins/filter/from_json.yml b/lib/ansible/plugins/filter/from_json.yml new file mode 100644 index 0000000..4edc2bd --- /dev/null +++ b/lib/ansible/plugins/filter/from_json.yml @@ -0,0 +1,25 @@ +DOCUMENTATION: + name: from_json + version_added: 'historical' + short_description: Convert JSON string into variable structure + description: + - Converts a JSON string representation into an equivalent structured Ansible variable. + - Ansible automatically converts JSON strings into variable structures in most contexts, use this plugin in contexts where automatic conversion does not happen. + notes: + - This filter functions as a wrapper to the Python C(json.loads) function. + options: + _input: + description: A JSON string. + type: string + required: true +EXAMPLES: | + # variable from string variable containing a JSON document + {{ docker_config | from_json }} + + # variable from string JSON document + {{ '{"a": true, "b": 54, "c": [1,2,3]}' | from_json }} + +RETURN: + _value: + description: The variable resulting from deserialization of the JSON document. + type: raw diff --git a/lib/ansible/plugins/filter/from_yaml.yml b/lib/ansible/plugins/filter/from_yaml.yml new file mode 100644 index 0000000..e9b1599 --- /dev/null +++ b/lib/ansible/plugins/filter/from_yaml.yml @@ -0,0 +1,25 @@ +DOCUMENTATION: + name: from_yaml + version_added: 'historical' + short_description: Convert YAML string into variable structure + description: + - Converts a YAML string representation into an equivalent structured Ansible variable. + - Ansible automatically converts YAML strings into variable structures in most contexts, use this plugin in contexts where automatic conversion does not happen. + notes: + - This filter functions as a wrapper to the L(Python pyyaml library, https://pypi.org/project/PyYAML/)'s C(yaml.safe_load) function. + options: + _input: + description: A YAML string. + type: string + required: true +EXAMPLES: | + # variable from string variable containing a YAML document + {{ github_workflow | from_yaml}} + + # variable from string JSON document + {{ '{"a": true, "b": 54, "c": [1,2,3]}' | from_yaml }} + +RETURN: + _value: + description: The variable resulting from deserializing the YAML document. + type: raw diff --git a/lib/ansible/plugins/filter/from_yaml_all.yml b/lib/ansible/plugins/filter/from_yaml_all.yml new file mode 100644 index 0000000..b179f1c --- /dev/null +++ b/lib/ansible/plugins/filter/from_yaml_all.yml @@ -0,0 +1,28 @@ +DOCUMENTATION: + name: from_yaml_all + version_added: 'historical' + short_description: Convert a series of YAML documents into a variable structure + description: + - Converts a YAML documents in a string representation into an equivalent structured Ansible variable. + - Ansible internally auto-converts YAML strings into variable structures in most contexts, but by default does not handle 'multi document' YAML files or strings. + - If multiple YAML documents are not supplied, this is the equivalend of using C(from_yaml). + notes: + - This filter functions as a wrapper to the Python C(yaml.safe_load_all) function, part of the L(pyyaml Python library, https://pypi.org/project/PyYAML/). + - Possible conflicts in variable names from the mulitple documents are resolved directly by the pyyaml library. + options: + _input: + description: A YAML string. + type: string + required: true + +EXAMPLES: | + # variable from string variable containing YAML documents + {{ multidoc_yaml_string | from_yaml_all }} + + # variable from multidocument YAML string + {{ '---\n{"a": true, "b": 54, "c": [1,2,3]}\n...\n---{"x": 1}\n...\n' | from_yaml_all}} + +RETURN: + _value: + description: The variable resulting from deserializing the YAML documents. + type: raw diff --git a/lib/ansible/plugins/filter/hash.yml b/lib/ansible/plugins/filter/hash.yml new file mode 100644 index 0000000..0f5f315 --- /dev/null +++ b/lib/ansible/plugins/filter/hash.yml @@ -0,0 +1,28 @@ +DOCUMENTATION: + name: checksum + version_added: "1.9" + short_description: hash of input data + description: + - Returns a configurable hash of the input data. Uses L(SHA-1, https://en.wikipedia.org/wiki/SHA-1) by default. + positional: _input + options: + _input: + description: Data to checksum. + type: raw + required: true + hashtype: + description: + - Type of algorithm to produce the hash. + - The list of available choices depends on the installed Python's hashlib. + type: string + default: sha1 +EXAMPLES: | + # sha1_hash => "109f4b3c50d7b0df729d299bc6f8e9ef9066971f" + sha1_hash: {{ 'test2' | hash('sha1') }} + # md5 => "5a105e8b9d40e1329780d62ea2265d8a" + md5: {{ 'test2' | hash('md5') }} + +RETURN: + _value: + description: The checksum of the input, as configured in I(hashtype). + type: string diff --git a/lib/ansible/plugins/filter/human_readable.yml b/lib/ansible/plugins/filter/human_readable.yml new file mode 100644 index 0000000..e3028ac --- /dev/null +++ b/lib/ansible/plugins/filter/human_readable.yml @@ -0,0 +1,35 @@ +DOCUMENTATION: + name: human_redable + version_added: "historical" + short_description: Make bytes/bits human readable + description: + - Convert byte or bit figures to more human readable formats. + positional: _input, isbits, unit + options: + _input: + description: Number of bytes, or bits. Depends on I(isbits). + type: int + required: true + isbits: + description: Whether the input is bits, instead of bytes. + type: bool + default: false + unit: + description: Unit to force output into. If none specified the largest unit arrived at will be used. + type: str + choices: [ 'Y', 'Z', 'E', 'P', 'T', 'G', 'M', 'K', 'B'] +EXAMPLES: | + + # size => "1.15 GB" + size: "{{ 1232345345 | human_readable }}" + + # size => "1.15 Gb" + size_bits: "{{ 1232345345 | human_readable(true) }}" + + # size => "1175.26 MB" + size_MB: "{{ 1232345345 | human_readable(unit='M') }}" + +RETURN: + _value: + description: Human readable byte or bit size. + type: str diff --git a/lib/ansible/plugins/filter/human_to_bytes.yml b/lib/ansible/plugins/filter/human_to_bytes.yml new file mode 100644 index 0000000..f03deed --- /dev/null +++ b/lib/ansible/plugins/filter/human_to_bytes.yml @@ -0,0 +1,34 @@ +DOCUMENTATION: + name: human_to_bytes + version_added: "historical" + short_description: Get bytes from string + description: + - Convert a human readable byte or bit string into a number bytes. + positional: _input, default_unit, isbits + options: + _input: + description: Human readable description of a number of bytes. + type: int + required: true + default_unit: + description: Unit to assume when input does not specify it. + type: str + choices: ['Y', 'Z', 'E', 'P', 'T', 'G', 'M', 'K', 'B'] + isbits: + description: If C(True), force to interpret only bit input; if C(False), force bytes. Otherwise use the notation to guess. + type: bool +EXAMPLES: | + + # size => 1234803098 + size: '{{ "1.15 GB" | human_to_bytes }}' + + # size => 1234803098 + size: '{{ "1.15" | human_to_bytes(deafult_unit="G") }}' + + # this is an error, wants bits, got bytes + ERROR: '{{ "1.15 GB" | human_to_bytes(isbits=true) }}' + +RETURN: + _value: + description: Integer representing the bytes from the input. + type: int diff --git a/lib/ansible/plugins/filter/intersect.yml b/lib/ansible/plugins/filter/intersect.yml new file mode 100644 index 0000000..d811eca --- /dev/null +++ b/lib/ansible/plugins/filter/intersect.yml @@ -0,0 +1,35 @@ +DOCUMENTATION: + name: intersect + author: Brian Coca (@bcoca) + version_added: "1.4" + short_description: intersection of lists + description: + - Provide a list with the common elements from other lists. + options: + _input: + description: A list. + type: list + required: true + _second_list: + description: A list. + type: list + required: true + seealso: + - plugin_type: filter + plugin: ansible.builtin.difference + - plugin_type: filter + plugin: ansible.builtin.symmetric_difference + - plugin_type: filter + plugin: ansible.builtin.unique + - plugin_type: filter + plugin: ansible.builtin.union +EXAMPLES: | + # return only the common elements of list1 and list2 + # list1: [1, 2, 5, 3, 4, 10] + # list2: [1, 2, 3, 4, 5, 11, 99] + {{ list1 | intersect(list2) }} + # => [1, 2, 5, 3, 4] +RETURN: + _value: + description: A list with unique elements common to both lists, also known as a set. + type: list diff --git a/lib/ansible/plugins/filter/items2dict.yml b/lib/ansible/plugins/filter/items2dict.yml new file mode 100644 index 0000000..1352c67 --- /dev/null +++ b/lib/ansible/plugins/filter/items2dict.yml @@ -0,0 +1,48 @@ +DOCUMENTATION: + name: items2dict + author: Ansible core team + version_added: "2.7" + short_description: Consolidate a list of itemized dictionaries into a dictionary + positional: _input, key_name, value_name + description: + - Takes a list of dicts with each having a C(key) and C(value) keys, and transforms the list into a dictionary, + effectively as the reverse of R(dict2items,ansible_collections.ansible.builtin.dict2items_filter). + options: + _input: + description: + - A list of dictionaries. + - Every dictionary must have keys C(key) and C(value). + type: list + elements: dict + required: true + key_name: + description: The name of the key in the element dictionaries that holds the key to use at destination. + type: str + default: key + value_name: + description: The name of the key in the element dictionaries that holds the value to use at destination. + type: str + default: value + seealso: + - plugin_type: filter + plugin: ansible.builtin.dict2items + +EXAMPLES: | + # mydict => { "hi": "bye", "ciao": "ciao" } + mydict: {{ [{'key': 'hi', 'value': 'bye'}, {'key': 'ciao', 'value': 'ciao'} ]| items2dict}} + + # The output is a dictionary with two key/value pairs: + # Application: payment + # Environment: dev + vars: + tags: + - key: Application + value: payment + - key: Environment + value: dev + consolidated: "{{ tags | items2dict }}" + +RETURN: + _value: + description: Dictionary with the consolidated key/values. + type: dict diff --git a/lib/ansible/plugins/filter/log.yml b/lib/ansible/plugins/filter/log.yml new file mode 100644 index 0000000..c7bb704 --- /dev/null +++ b/lib/ansible/plugins/filter/log.yml @@ -0,0 +1,33 @@ +DOCUMENTATION: + name: log + version_added: "1.9" + short_description: log of (math operation) + description: + - Math operation that returns the L(logarithm, https://en.wikipedia.org/wiki/Logarithm) to base N of the input number. + - By default, computes the L(natural logarithm, https://en.wikipedia.org/wiki/Natural_logarithm). + notes: + - This is a passthrough to Python's C(math.log). + positional: _input, base + options: + _input: + description: Number to operate on. + type: float + required: true + base: + description: Which base to use. Defaults to L(Euler's number, https://en.wikipedia.org/wiki/Euler%27s_number). + type: float + default: 2.718281828459045 + +EXAMPLES: | + + # 1.2920296742201791 + eightlogfive: "{{ 8 | log(5) }}" + + # 0.9030899869919435 + eightlog10: "{{ 8 | log() }}" + + +RETURN: + _value: + description: Resulting number. + type: float diff --git a/lib/ansible/plugins/filter/mandatory.yml b/lib/ansible/plugins/filter/mandatory.yml new file mode 100644 index 0000000..5addf15 --- /dev/null +++ b/lib/ansible/plugins/filter/mandatory.yml @@ -0,0 +1,21 @@ +DOCUMENTATION: + name: mandatory + version_added: "historical" + short_description: make a variable's existance mandatory + description: + - Depending on context undefined variables can be ignored or skipped, this ensures they force an error. + positional: _input + options: + _input: + description: Mandatory expression. + type: raw + required: true +EXAMPLES: | + + # results in a Filter Error + {{ notdefined | mandatory }} + +RETURN: + _value: + description: The input if defined, otherwise an error. + type: raw diff --git a/lib/ansible/plugins/filter/mathstuff.py b/lib/ansible/plugins/filter/mathstuff.py new file mode 100644 index 0000000..d4b6af7 --- /dev/null +++ b/lib/ansible/plugins/filter/mathstuff.py @@ -0,0 +1,252 @@ +# Copyright 2014, Brian Coca <bcoca@ansible.com> +# Copyright 2017, Ken Celenza <ken@networktocode.com> +# Copyright 2017, Jason Edelman <jason@networktocode.com> +# Copyright 2017, Ansible Project +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import itertools +import math + +from collections.abc import Hashable, Mapping, Iterable + +from jinja2.filters import pass_environment + +from ansible.errors import AnsibleFilterError, AnsibleFilterTypeError +from ansible.module_utils.common.text import formatters +from ansible.module_utils.six import binary_type, text_type +from ansible.module_utils._text import to_native, to_text +from ansible.utils.display import Display + +try: + from jinja2.filters import do_unique + HAS_UNIQUE = True +except ImportError: + HAS_UNIQUE = False + + +display = Display() + + +@pass_environment +# Use case_sensitive=None as a sentinel value, so we raise an error only when +# explicitly set and cannot be handle (by Jinja2 w/o 'unique' or fallback version) +def unique(environment, a, case_sensitive=None, attribute=None): + + def _do_fail(e): + if case_sensitive is False or attribute: + raise AnsibleFilterError("Jinja2's unique filter failed and we cannot fall back to Ansible's version " + "as it does not support the parameters supplied", orig_exc=e) + + error = e = None + try: + if HAS_UNIQUE: + c = list(do_unique(environment, a, case_sensitive=bool(case_sensitive), attribute=attribute)) + except TypeError as e: + error = e + _do_fail(e) + except Exception as e: + error = e + _do_fail(e) + display.warning('Falling back to Ansible unique filter as Jinja2 one failed: %s' % to_text(e)) + + if not HAS_UNIQUE or error: + + # handle Jinja2 specific attributes when using Ansible's version + if case_sensitive is False or attribute: + raise AnsibleFilterError("Ansible's unique filter does not support case_sensitive=False nor attribute parameters, " + "you need a newer version of Jinja2 that provides their version of the filter.") + + c = [] + for x in a: + if x not in c: + c.append(x) + + return c + + +@pass_environment +def intersect(environment, a, b): + if isinstance(a, Hashable) and isinstance(b, Hashable): + c = set(a) & set(b) + else: + c = unique(environment, [x for x in a if x in b], True) + return c + + +@pass_environment +def difference(environment, a, b): + if isinstance(a, Hashable) and isinstance(b, Hashable): + c = set(a) - set(b) + else: + c = unique(environment, [x for x in a if x not in b], True) + return c + + +@pass_environment +def symmetric_difference(environment, a, b): + if isinstance(a, Hashable) and isinstance(b, Hashable): + c = set(a) ^ set(b) + else: + isect = intersect(environment, a, b) + c = [x for x in union(environment, a, b) if x not in isect] + return c + + +@pass_environment +def union(environment, a, b): + if isinstance(a, Hashable) and isinstance(b, Hashable): + c = set(a) | set(b) + else: + c = unique(environment, a + b, True) + return c + + +def logarithm(x, base=math.e): + try: + if base == 10: + return math.log10(x) + else: + return math.log(x, base) + except TypeError as e: + raise AnsibleFilterTypeError('log() can only be used on numbers: %s' % to_native(e)) + + +def power(x, y): + try: + return math.pow(x, y) + except TypeError as e: + raise AnsibleFilterTypeError('pow() can only be used on numbers: %s' % to_native(e)) + + +def inversepower(x, base=2): + try: + if base == 2: + return math.sqrt(x) + else: + return math.pow(x, 1.0 / float(base)) + except (ValueError, TypeError) as e: + raise AnsibleFilterTypeError('root() can only be used on numbers: %s' % to_native(e)) + + +def human_readable(size, isbits=False, unit=None): + ''' Return a human readable string ''' + try: + return formatters.bytes_to_human(size, isbits, unit) + except TypeError as e: + raise AnsibleFilterTypeError("human_readable() failed on bad input: %s" % to_native(e)) + except Exception: + raise AnsibleFilterError("human_readable() can't interpret following string: %s" % size) + + +def human_to_bytes(size, default_unit=None, isbits=False): + ''' Return bytes count from a human readable string ''' + try: + return formatters.human_to_bytes(size, default_unit, isbits) + except TypeError as e: + raise AnsibleFilterTypeError("human_to_bytes() failed on bad input: %s" % to_native(e)) + except Exception: + raise AnsibleFilterError("human_to_bytes() can't interpret following string: %s" % size) + + +def rekey_on_member(data, key, duplicates='error'): + """ + Rekey a dict of dicts on another member + + May also create a dict from a list of dicts. + + duplicates can be one of ``error`` or ``overwrite`` to specify whether to error out if the key + value would be duplicated or to overwrite previous entries if that's the case. + """ + if duplicates not in ('error', 'overwrite'): + raise AnsibleFilterError("duplicates parameter to rekey_on_member has unknown value: {0}".format(duplicates)) + + new_obj = {} + + # Ensure the positional args are defined - raise jinja2.exceptions.UndefinedError if not + bool(data) and bool(key) + + if isinstance(data, Mapping): + iterate_over = data.values() + elif isinstance(data, Iterable) and not isinstance(data, (text_type, binary_type)): + iterate_over = data + else: + raise AnsibleFilterTypeError("Type is not a valid list, set, or dict") + + for item in iterate_over: + if not isinstance(item, Mapping): + raise AnsibleFilterTypeError("List item is not a valid dict") + + try: + key_elem = item[key] + except KeyError: + raise AnsibleFilterError("Key {0} was not found".format(key)) + except TypeError as e: + raise AnsibleFilterTypeError(to_native(e)) + except Exception as e: + raise AnsibleFilterError(to_native(e)) + + # Note: if new_obj[key_elem] exists it will always be a non-empty dict (it will at + # minimum contain {key: key_elem} + if new_obj.get(key_elem, None): + if duplicates == 'error': + raise AnsibleFilterError("Key {0} is not unique, cannot correctly turn into dict".format(key_elem)) + elif duplicates == 'overwrite': + new_obj[key_elem] = item + else: + new_obj[key_elem] = item + + return new_obj + + +class FilterModule(object): + ''' Ansible math jinja2 filters ''' + + def filters(self): + filters = { + # exponents and logarithms + 'log': logarithm, + 'pow': power, + 'root': inversepower, + + # set theory + 'unique': unique, + 'intersect': intersect, + 'difference': difference, + 'symmetric_difference': symmetric_difference, + 'union': union, + + # combinatorial + 'product': itertools.product, + 'permutations': itertools.permutations, + 'combinations': itertools.combinations, + + # computer theory + 'human_readable': human_readable, + 'human_to_bytes': human_to_bytes, + 'rekey_on_member': rekey_on_member, + + # zip + 'zip': zip, + 'zip_longest': itertools.zip_longest, + + } + + return filters diff --git a/lib/ansible/plugins/filter/md5.yml b/lib/ansible/plugins/filter/md5.yml new file mode 100644 index 0000000..c97870d --- /dev/null +++ b/lib/ansible/plugins/filter/md5.yml @@ -0,0 +1,24 @@ +DOCUMENTATION: + name: md5 + version_added: "historical" + short_description: MD5 hash of input data + description: + - Returns an L(MD5 hash, https://en.wikipedia.org/wiki/MD5) of the input data + positional: _input + notes: + - This requires the MD5 algorithm to be available on the system, security contexts like FIPS might prevent this. + - MD5 has long been deemed insecure and is not recommended for security related uses. + options: + _input: + description: data to hash + type: raw + required: true + +EXAMPLES: | + # md5hash => "ae2b1fca515949e5d54fb22b8ed95575" + md5hash: "{{ 'testing' | md5 }}" + +RETURN: + _value: + description: The MD5 hash of the input. + type: string diff --git a/lib/ansible/plugins/filter/password_hash.yml b/lib/ansible/plugins/filter/password_hash.yml new file mode 100644 index 0000000..d12efb4 --- /dev/null +++ b/lib/ansible/plugins/filter/password_hash.yml @@ -0,0 +1,37 @@ +DOCUMENTATION: + name: password_hash + version_added: "historical" + short_description: convert input password into password_hash + description: + - Returns a password_hash of a secret. + positional: _input + notes: + - Algorithms available might be restricted by the system. + options: + _input: + description: Secret to hash. + type: string + required: true + hashtype: + description: Hashing algorithm to use. + type: string + default: sha512 + choices: [ md5, blowfish, sha256, sha512 ] + salt: + description: Secret string that is used for the hashing, if none is provided a random one can be generated. + type: int + rounds: + description: Number of encryption rounds, default varies by algorithm used. + type: int + ident: + description: Algorithm identifier. + type: string + +EXAMPLES: | + # pwdhash => "$6$/bQCntzQ7VrgVcFa$VaMkmevkY1dqrx8neaenUDlVU.6L/.ojRbrnI4ID.yBHU6XON1cB422scCiXfUL5wRucMdLgJU0Fn38uoeBni/" + pwdhash: "{{ 'testing' | password_hash }}" + +RETURN: + _value: + description: The resulting password hash. + type: string diff --git a/lib/ansible/plugins/filter/path_join.yml b/lib/ansible/plugins/filter/path_join.yml new file mode 100644 index 0000000..d50deaa --- /dev/null +++ b/lib/ansible/plugins/filter/path_join.yml @@ -0,0 +1,30 @@ +DOCUMENTATION: + name: path_join + author: Anthony Bourguignon (@Toniob) + version_added: "2.10" + short_description: Join one or more path components + positional: _input + description: + - Returns a path obtained by joining one or more path components. + options: + _input: + description: A path, or a list of paths. + type: list + elements: str + required: true + +EXAMPLES: | + + # If path == 'foo/bar' and file == 'baz.txt', the result is '/etc/foo/bar/subdir/baz.txt' + {{ ('/etc', path, 'subdir', file) | path_join }} + + # equivalent to '/etc/subdir/{{filename}}' + wheremyfile: "{{ ['/etc', 'subdir', filename] | path_join }}" + + # trustme => '/etc/apt/trusted.d/mykey.gpgp' + trustme: "{{ ['/etc', 'apt', 'trusted.d', 'mykey.gpg'] | path_join }}" + +RETURN: + _value: + description: The concatenated path. + type: str diff --git a/lib/ansible/plugins/filter/permutations.yml b/lib/ansible/plugins/filter/permutations.yml new file mode 100644 index 0000000..6e0202b --- /dev/null +++ b/lib/ansible/plugins/filter/permutations.yml @@ -0,0 +1,26 @@ +DOCUMENTATION: + name: permutations + version_added: "historical" + short_description: permutations from the elements of a list + description: + - Create a list of the permutations of lists from the elements of a list. + - Unlike combinations, in permutations order is significant. + positional: _input, list_size + options: + _input: + description: Elements to base the permutations on. + type: list + required: true + list_size: + description: The size of the list for each permutation. + type: int + required: true + +EXAMPLES: | + # ptrs_of_two => [ [ 1, 2 ], [ 1, 3 ], [ 1, 4 ], [ 1, 5 ], [ 2, 1 ], [ 2, 3 ], [ 2, 4 ], [ 2, 5 ], [ 3, 1 ], [ 3, 2 ], [ 3, 4 ], [ 3, 5 ], [ 4, 1 ], [ 4, 2 ], [ 4, 3 ], [ 4, 5 ], [ 5, 1 ], [ 5, 2 ], [ 5, 3 ], [ 5, 4 ] ] + prts_of_two: "{{ [1,2,3,4,5] | permutations(2) }}" + +RETURN: + _value: + description: List of permutations lists resulting from the supplied elements and list size. + type: list diff --git a/lib/ansible/plugins/filter/pow.yml b/lib/ansible/plugins/filter/pow.yml new file mode 100644 index 0000000..da2fa42 --- /dev/null +++ b/lib/ansible/plugins/filter/pow.yml @@ -0,0 +1,34 @@ +DOCUMENTATION: + name: pow + version_added: "1.9" + short_description: power of (math operation) + description: + - Math operation that returns the Nth power of inputed number, C(X ^ N). + notes: + - This is a passthrough to Python's C(math.pow). + positional: _input, _power + options: + _input: + description: The base. + type: float + required: true + _power: + description: Which power (exponent) to use. + type: float + required: true + +EXAMPLES: | + + # => 32768 + eight_power_five: "{{ 8 | pow(5) }}" + + # 4 + square_of_2: "{{ 2 | pow(2) }}" + + # me ^ 3 + cube_me: "{{ me | pow(3) }}" + +RETURN: + _value: + description: Resulting number. + type: float diff --git a/lib/ansible/plugins/filter/product.yml b/lib/ansible/plugins/filter/product.yml new file mode 100644 index 0000000..5035522 --- /dev/null +++ b/lib/ansible/plugins/filter/product.yml @@ -0,0 +1,42 @@ +DOCUMENTATION: + name: product + version_added: "historical" + short_description: cartesian product of lists + description: + - Combines two lists into one with each element being the product of the elements of the input lists. + - Creates 'nested loops'. Looping over C(listA) and C(listB) is the same as looping over C(listA | product(listB)). + notes: + - This is a passthrough to Python's C(itertools.product) + positional: _input, _additional_lists, repeat + options: + _input: + description: First list. + type: list + required: true + _additional_lists: #TODO: *args, N possible additional lists + description: Additional list for the product. + type: list + required: false + repeat: + description: Number of times to repeat the product against itself. + default: 1 + type: int +EXAMPLES: | + + # product => [ [ 1, "a" ], [ 1, "b" ], [ 1, "c" ], [ 2, "a" ], [ 2, "b" ], [ 2, "c" ], [ 3, "a" ], [ 3, "b" ], [ 3, "c" ], [ 4, "a" ], [ 4, "b" ], [ 4, "c" ], [ 5, "a" ], [ 5, "b" ], [ 5, "c" ] ] + product: "{{ [1,2,3,4,5] | product(['a', 'b', 'c']) }}" + + # repeat_original => [ [ 1, 1 ], [ 1, 2 ], [ 2, 1 ], [ 2, 2 ] ] + repeat_original: "{{ [1,2] | product(repeat=2) }}" + + # repeat_product => [ [ 1, "a", 1, "a" ], [ 1, "a", 1, "b" ], [ 1, "a", 2, "a" ], [ 1, "a", 2, "b" ], [ 1, "b", 1, "a" ], [ 1, "b", 1, "b" ], [ 1, "b", 2, "a" ], [ 1, "b", 2, "b" ], [ 2, "a", 1, "a" ], [ 2, "a", 1, "b" ], [ 2, "a", 2, "a" ], [ 2, "a", 2, "b" ], [ 2, "b", 1, "a" ], [ 2, "b", 1, "b" ], [ 2, "b", 2, "a" ], [ 2, "b", 2, "b" ] ] + repeat_product: "{{ [1,2] | product(['a', 'b'], repeat=2) }}" + + # domains => [ 'example.com', 'ansible.com', 'redhat.com' ] + domains: "{{ [ 'example', 'ansible', 'redhat'] | product(['com']) | map('join', '.') }}" + +RETURN: + _value: + description: List of lists of combined elements from the input lists. + type: list + elements: list diff --git a/lib/ansible/plugins/filter/quote.yml b/lib/ansible/plugins/filter/quote.yml new file mode 100644 index 0000000..2d621ed --- /dev/null +++ b/lib/ansible/plugins/filter/quote.yml @@ -0,0 +1,23 @@ +DOCUMENTATION: + name: quote + version_added: "2.10" + short_description: shell quoting + description: + - Quote a string to safely use as in a POSIX shell. + notes: + - This is a passthrough to Python's C(shlex.quote). + positional: _input + options: + _input: + description: String to quote. + type: str + required: true + +EXAMPLES: | + - name: Run a shell command + shell: echo {{ string_value | quote }} + +RETURN: + _value: + description: Quoted string. + type: str diff --git a/lib/ansible/plugins/filter/random.yml b/lib/ansible/plugins/filter/random.yml new file mode 100644 index 0000000..b72dbb2 --- /dev/null +++ b/lib/ansible/plugins/filter/random.yml @@ -0,0 +1,35 @@ +DOCUMENTATION: + name: random + version_added: "2.6" + short_description: random number or list item + description: + - Use the input to either select a random element of a list or generate a random number. + positional: _input, start, step, seed + options: + _input: + description: A number or list/sequence, if it is a number it is the top bound for random number generation, if it is a sequence or list, the source of the random element selected. + type: raw + required: true + start: + description: Bottom bound for the random number/element generated. + type: int + step: + description: Subsets the defined range by only using this value to select the increments of it between start and end. + type: int + default: 1 + seed: + description: If specified use a pseudo random selection instead (repeatable). + type: str + +EXAMPLES: | + + # can be any item from the list + random_item: "{{ ['a','b','c'] | random }}" + + # cron line, select random minute repeatable for each host + "{{ 60 | random(seed=inventory_hostname) }} * * * * root /script/from/cron" + +RETURN: + _value: + description: Random number or list element. + type: raw diff --git a/lib/ansible/plugins/filter/realpath.yml b/lib/ansible/plugins/filter/realpath.yml new file mode 100644 index 0000000..12687b6 --- /dev/null +++ b/lib/ansible/plugins/filter/realpath.yml @@ -0,0 +1,21 @@ +DOCUMENTATION: + name: realpath + author: darkone23 (@darkone23) + version_added: "1.8" + short_description: Turn path into real path + description: + - Resolves/follows symliknks to return the 'real path' from a given path. + - Filters alwasy run on controller so this path is resolved using the controller's filesystem. + options: + _input: + description: A path. + type: path + required: true +EXAMPLES: | + + realpath: {{ '/path/to/synlink' | realpath }} + +RETURN: + _value: + description: The canonical path. + type: path diff --git a/lib/ansible/plugins/filter/regex_escape.yml b/lib/ansible/plugins/filter/regex_escape.yml new file mode 100644 index 0000000..7819909 --- /dev/null +++ b/lib/ansible/plugins/filter/regex_escape.yml @@ -0,0 +1,29 @@ +DOCUMENTATION: + name: regex_escape + version_added: "2.8" + short_description: escape regex chars + description: + - Escape special characters in a string for use in a regular expression. + positional: _input, re_type + notes: + - posix_extended is not implemented yet + options: + _input: + description: String to escape. + type: str + required: true + re_type: + description: Which type of escaping to use. + type: str + default: python + choices: [python, posix_basic] + +EXAMPLES: | + + # safe_for_regex => '\^f\.\*o\(\.\*\)\$' + safe_for_regex: "{{ '^f.*o(.*)$' | regex_escape() }}" + +RETURN: + _value: + description: Escaped string. + type: str diff --git a/lib/ansible/plugins/filter/regex_findall.yml b/lib/ansible/plugins/filter/regex_findall.yml new file mode 100644 index 0000000..707d6fa --- /dev/null +++ b/lib/ansible/plugins/filter/regex_findall.yml @@ -0,0 +1,37 @@ +DOCUMENTATION: + name: regex_findall + version_added: "2.0" + short_description: extract all regex matches from string + description: + - Search in a string or extract all the parts of a string matching a regular expression. + positional: _input, _regex + options: + _input: + description: String to match against. + type: str + required: true + _regex: + description: Regular expression string that defines the match. + type: str + multiline: + description: Search across line endings if C(True), do not if otherwise. + type: bool + default: no + ignorecase: + description: Force the search to be case insensitive if C(True), case sensitive otherwise. + type: bool + default: no + +EXAMPLES: | + + # all_pirates => ['CAR', 'tar', 'bar'] + all_pirates: "{{ 'CAR\ntar\nfoo\nbar\n' | regex_findall('^.ar$', multiline=True, ignorecase=True) }}" + + # get_ips => ['8.8.8.8', '8.8.4.4'] + get_ips: "{{ 'Some DNS servers are 8.8.8.8 and 8.8.4.4' | regex_findall('\\b(?:[0-9]{1,3}\\.){3}[0-9]{1,3}\\b') }}" + +RETURN: + _value: + description: List of matched strings. + type: list + elements: str diff --git a/lib/ansible/plugins/filter/regex_replace.yml b/lib/ansible/plugins/filter/regex_replace.yml new file mode 100644 index 0000000..0277b56 --- /dev/null +++ b/lib/ansible/plugins/filter/regex_replace.yml @@ -0,0 +1,46 @@ +DOCUMENTATION: + name: regex_replace + version_added: "2.0" + short_description: replace a string via regex + description: + - Replace a substring defined by a regular expression with another defined by another regular expression based on the first match. + notes: + - Maps to Python's C(re.replace). + positional: _input, _regex_match, _regex_replace + options: + _input: + description: String to match against. + type: str + required: true + _regex_match: + description: Regular expression string that defines the match. + type: int + required: true + _regex_replace: + description: Regular expression string that defines the replacement. + type: int + required: true + multiline: + description: Search across line endings if C(True), do not if otherwise. + type: bool + default: no + ignorecase: + description: Force the search to be case insensitive if C(True), case sensitive otherwise. + type: bool + default: no + +EXAMPLES: | + + # whatami => 'able' + whatami: "{{ 'ansible' | regex_replace('^a.*i(.*)$', 'a\\1') }}" + + # commalocal => 'localhost, 80' + commalocal: "{{ 'localhost:80' | regex_replace('^(?P<host>.+):(?P<port>\\d+)$', '\\g<host>, \\g<port>') }}" + + # piratecomment => '#CAR\n#tar\nfoo\n#bar\n' + piratecomment: "{{ 'CAR\ntar\nfoo\nbar\n' | regex_replace('^(.ar)$', '#\\1', multiline=True, ignorecase=True) }}" + +RETURN: + _value: + description: String with substitution (or original if no match). + type: str diff --git a/lib/ansible/plugins/filter/regex_search.yml b/lib/ansible/plugins/filter/regex_search.yml new file mode 100644 index 0000000..c61efb7 --- /dev/null +++ b/lib/ansible/plugins/filter/regex_search.yml @@ -0,0 +1,38 @@ +DOCUMENTATION: + name: regex_search + version_added: "2.0" + short_description: extract regex match from string + description: + - Search in a string to extract the part that matches the regular expression. + notes: + - Maps to Python's C(re.search). + positional: _input, _regex + options: + _input: + description: String to match against. + type: str + required: true + _regex: + description: Regular expression string that defines the match. + type: str + multiline: + description: Search across line endings if C(True), do not if otherwise. + type: bool + default: no + ignorecase: + description: Force the search to be case insensitive if C(True), case sensitive otherwise. + type: bool + default: no + +EXAMPLES: | + + # db => 'database42' + db: "{{ 'server1/database42' | regex_search('database[0-9]+') }}" + + # drinkat => 'BAR' + drinkat: "{{ 'foo\nBAR' | regex_search('^bar', multiline=True, ignorecase=True) }}" + +RETURN: + _value: + description: Matched string or empty string if no match. + type: str diff --git a/lib/ansible/plugins/filter/rekey_on_member.yml b/lib/ansible/plugins/filter/rekey_on_member.yml new file mode 100644 index 0000000..d7470ab --- /dev/null +++ b/lib/ansible/plugins/filter/rekey_on_member.yml @@ -0,0 +1,30 @@ +DOCUMENTATION: + name: rekey_on_member + version_added: "2.13" + short_description: Rekey a list of dicts into a dict using a member + positional: _input, '_key', duplicates + description: Iterate over several iterables in parallel, producing tuples with an item from each one. + options: + _input: + description: Original dictionary. + type: dict + required: yes + _key: + description: The key to rekey. + type: str + required: yes + duplicates: + description: How to handle duplicates. + type: str + default: error + choices: [overwrite, error] + +EXAMPLES: | + + # mydict => {'eigrp': {'state': 'enabled', 'proto': 'eigrp'}, 'ospf': {'state': 'enabled', 'proto': 'ospf'}} + mydict: '{{ [{"proto": "eigrp", "state": "enabled"}, {"proto": "ospf", "state": "enabled"}] | rekey_on_member("proto") }}' + +RETURN: + _value: + description: The resulting dictionary. + type: dict diff --git a/lib/ansible/plugins/filter/relpath.yml b/lib/ansible/plugins/filter/relpath.yml new file mode 100644 index 0000000..47611c7 --- /dev/null +++ b/lib/ansible/plugins/filter/relpath.yml @@ -0,0 +1,28 @@ +DOCUMENTATION: + name: relpath + author: Jakub Jirutka (@jirutka) + version_added: "1.7" + short_description: Make a path relative + positional: _input, start + description: + - Converts the given path to a relative path from the I(start), + or relative to the directory given in I(start). + options: + _input: + description: A path. + type: str + required: true + start: + description: The directory the path should be relative to. If not supplied the current working directory will be used. + type: str + +EXAMPLES: | + + # foobar => ../test/me.txt + testing: "{{ '/tmp/test/me.txt' | relpath('/tmp/other/') }}" + otherrelpath: "{{ mypath | relpath(mydir) }}" + +RETURN: + _value: + description: The relative path. + type: str diff --git a/lib/ansible/plugins/filter/root.yml b/lib/ansible/plugins/filter/root.yml new file mode 100644 index 0000000..4f52590 --- /dev/null +++ b/lib/ansible/plugins/filter/root.yml @@ -0,0 +1,32 @@ +DOCUMENTATION: + name: root + version_added: "1.9" + short_description: root of (math operation) + description: + - Math operation that returns the Nth root of inputed number C(X ^^ N). + positional: _input, base + options: + _input: + description: Number to operate on. + type: float + required: true + base: + description: Which root to take. + type: float + default: 2 + +EXAMPLES: | + + # => 8 + fiveroot: "{{ 32768 | root (5) }}" + + # 2 + sqrt_of_2: "{{ 4 | root }}" + + # me ^^ 3 + cuberoot_me: "{{ me | root(3) }}" + +RETURN: + _value: + description: Resulting number. + type: float diff --git a/lib/ansible/plugins/filter/sha1.yml b/lib/ansible/plugins/filter/sha1.yml new file mode 100644 index 0000000..f80803b --- /dev/null +++ b/lib/ansible/plugins/filter/sha1.yml @@ -0,0 +1,24 @@ +DOCUMENTATION: + name: sha1 + version_added: "historical" + short_description: SHA-1 hash of input data + description: + - Returns a L(SHA-1 hash, https://en.wikipedia.org/wiki/SHA-1) of the input data. + positional: _input + notes: + - This requires the SHA-1 algorithm to be available on the system, security contexts like FIPS might prevent this. + - SHA-1 has been deemed insecure and is not recommended for security related uses. + options: + _input: + description: Data to hash. + type: raw + required: true + +EXAMPLES: | + # sha1hash => "dc724af18fbdd4e59189f5fe768a5f8311527050" + sha1hash: "{{ 'testing' | sha1 }}" + +RETURN: + _value: + description: The SHA-1 hash of the input. + type: string diff --git a/lib/ansible/plugins/filter/shuffle.yml b/lib/ansible/plugins/filter/shuffle.yml new file mode 100644 index 0000000..a7c3e7e --- /dev/null +++ b/lib/ansible/plugins/filter/shuffle.yml @@ -0,0 +1,27 @@ +DOCUMENTATION: + name: shuffle + version_added: "2.6" + short_description: randomize a list + description: + - Take the elements of the input list and return in a random order. + positional: _input + options: + _input: + description: A number or list to randomize. + type: list + elements: any + required: true + seed: + description: If specified use a pseudo random selection instead (repeatable). + type: str + +EXAMPLES: | + + randomized_list: "{{ ['a','b','c'] | shuffle}}" + per_host_repeatable: "{{ ['a','b','c'] | shuffle(seed=inventory_hostname) }}" + +RETURN: + _value: + description: Random number or list element. + type: list + elements: any diff --git a/lib/ansible/plugins/filter/split.yml b/lib/ansible/plugins/filter/split.yml new file mode 100644 index 0000000..3e7b59e --- /dev/null +++ b/lib/ansible/plugins/filter/split.yml @@ -0,0 +1,32 @@ +DOCUMENTATION: + name: split + version_added: "historical" + short_description: split a string into a list + description: + - Using Python's text object method C(split) we turn strings into lists via a 'spliting character'. + notes: + - This is a passthrough to Python's C(str.split). + positional: _input, _split_string + options: + _input: + description: A string to split. + type: str + required: true + _split_string: + description: A string on which to split the original. + type: str + default: ' ' + +EXAMPLES: | + + # listjojo => [ "jojo", "is", "a" ] + listjojo: "{{ 'jojo is a' | split }}" + + # listjojocomma => [ "jojo is", "a" ] + listjojocomma: "{{ 'jojo is, a' | split(',' }}" + +RETURN: + _value: + description: List of substrings split from the original. + type: list + elements: str diff --git a/lib/ansible/plugins/filter/splitext.yml b/lib/ansible/plugins/filter/splitext.yml new file mode 100644 index 0000000..ea9cbce --- /dev/null +++ b/lib/ansible/plugins/filter/splitext.yml @@ -0,0 +1,30 @@ +DOCUMENTATION: + name: splitext + author: Matt Martz (@sivel) + version_added: "2.0" + short_description: split a path into root and file extension + positional: _input + description: + - Returns a list of two, with the elements consisting of filename root and extension. + options: + _input: + description: A path. + type: str + required: true + +EXAMPLES: | + + # gobble => [ '/etc/make', 'conf' ] + gobble: "{{ '/etc/make.conf' | splitext }}" + + # file_n_ext => [ 'ansible', 'cfg' ] + file_n_ext: "{{ 'ansible.cfg' | splitext }}" + + # hoax => ['/etc/hoasdf', ''] + hoax: '{{ "/etc//hoasdf/"|splitext }}' + +RETURN: + _value: + description: A list consisting of root of the path and the extension. + type: list + elements: str diff --git a/lib/ansible/plugins/filter/strftime.yml b/lib/ansible/plugins/filter/strftime.yml new file mode 100644 index 0000000..6cb8874 --- /dev/null +++ b/lib/ansible/plugins/filter/strftime.yml @@ -0,0 +1,45 @@ +DOCUMENTATION: + name: strftime + version_added: "2.4" + short_description: date formating + description: + - Using Python's C(strftime) function, take a data formating string and a date/time to create a formated date. + notes: + - This is a passthrough to Python's C(stftime). + positional: _input, second, utc + options: + _input: + description: + - A formating string following C(stftime) conventions. + - See L(the Python documentation, https://docs.python.org/3/library/datetime.html#strftime-strptime-behavior) for a reference. + type: str + required: true + second: + description: Datetime in seconds from C(epoch) to format, if not supplied C(gmttime/localtime) will be used. + type: int + utc: + description: Whether time supplied is in UTC. + type: bool + default: false + +EXAMPLES: | + # Display year-month-day + {{ '%Y-%m-%d' | strftime }} + # => "2021-03-19" + + # Display hour:min:sec + {{ '%H:%M:%S' | strftime }} + # => "21:51:04" + + # Use ansible_date_time.epoch fact + {{ '%Y-%m-%d %H:%M:%S' | strftime(ansible_date_time.epoch) }} + # => "2021-03-19 21:54:09" + + # Use arbitrary epoch value + {{ '%Y-%m-%d' | strftime(0) }} # => 1970-01-01 + {{ '%Y-%m-%d' | strftime(1441357287) }} # => 2015-09-04 + +RETURN: + _value: + description: A formatted date/time string. + type: str diff --git a/lib/ansible/plugins/filter/subelements.yml b/lib/ansible/plugins/filter/subelements.yml new file mode 100644 index 0000000..a2d1a94 --- /dev/null +++ b/lib/ansible/plugins/filter/subelements.yml @@ -0,0 +1,38 @@ +DOCUMENTATION: + name: subelements + version_added: "2.7" + short_description: retuns a product of a list and it's elements + positional: _input, _subelement, skip_missing + description: + - This produces a product of an object and the subelement values of that object, similar to the subelements lookup. This lets you specify individual subelements to use in a template I(_input). + options: + _input: + description: Original list. + type: list + elements: any + required: yes + _subelement: + description: Label of property to extract from original list items. + type: str + required: yes + skip_missing: + description: If C(True), ignore missing subelements, otherwise missing subelements generate an error. + type: bool + default: no + +EXAMPLES: | + # data + users: + - groups: [1,2,3] + name: lola + - name: fernando + groups: [2,3,4] + + # user_w_groups =>[ { "groups": [ 1, 2, 3 ], "name": "lola" }, 1 ], [ { "groups": [ 1, 2, 3 ], "name": "lola" }, 2 ], [ { "groups": [ 1, 2, 3 ], "name": "lola" }, 3 ], [ { "groups": [ 2, 3, 4 ], "name": "fernando" }, 2 ], [ { "groups": [ 2, 3, 4 ], "name": "fernando" }, 3 ], [ { "groups": [ 2, 3, 4 ], "name": "fernando" }, 4 ] ] + users_w_groups: {{ users | subelements('groups', skip_missing=True) }} + +RETURN: + _value: + description: List made of original list and product of the subelement list. + type: list + elements: any diff --git a/lib/ansible/plugins/filter/symmetric_difference.yml b/lib/ansible/plugins/filter/symmetric_difference.yml new file mode 100644 index 0000000..de4f3c6 --- /dev/null +++ b/lib/ansible/plugins/filter/symmetric_difference.yml @@ -0,0 +1,35 @@ +DOCUMENTATION: + name: symmetric_difference + author: Brian Coca (@bcoca) + version_added: "1.4" + short_description: different items from two lists + description: + - Provide a unique list of all the elements unique to each list. + options: + _input: + description: A list. + type: list + required: true + _second_list: + description: A list. + type: list + required: true + seealso: + - plugin_type: filter + plugin: ansible.builtin.difference + - plugin_type: filter + plugin: ansible.builtin.intersect + - plugin_type: filter + plugin: ansible.builtin.union + - plugin_type: filter + plugin: ansible.builtin.unique +EXAMPLES: | + # return the elements of list1 not in list2 and the elements in list2 not in list1 + # list1: [1, 2, 5, 1, 3, 4, 10] + # list2: [1, 2, 3, 4, 5, 11, 99] + {{ list1 | symmetric_difference(list2) }} + # => [10, 11, 99] +RETURN: + _value: + description: A unique list of the elements from two lists that are unique to each one. + type: list diff --git a/lib/ansible/plugins/filter/ternary.yml b/lib/ansible/plugins/filter/ternary.yml new file mode 100644 index 0000000..50ff767 --- /dev/null +++ b/lib/ansible/plugins/filter/ternary.yml @@ -0,0 +1,44 @@ +DOCUMENTATION: + name: ternary + author: Brian Coca (@bcoca) + version_added: '1.9' + short_description: Ternary operation filter + description: + - Return the first value if the input is C(True), the second if C(False). + positional: true_val, false_val + options: + _input: + description: A boolean expression, must evaluate to C(True) or C(False). + type: bool + required: true + true_val: + description: Value to return if the input is C(True). + type: any + required: true + false_val: + description: Value to return if the input is C(False). + type: any + none_val: + description: Value to return if the input is C(None). If not set, C(None) will be treated as C(False). + type: any + version_added: '2.8' + notes: + - Vars as values are evaluated even when not returned. This is due to them being evaluated before being passed into the filter. + +EXAMPLES: | + # set first 10 volumes rw, rest as dp + volume_mode: "{{ (item|int < 11)|ternary('rw', 'dp') }}" + + # choose correct vpc subnet id, note that vars as values are evaluated even if not returned + vpc_subnet_id: "{{ (ec2_subnet_type == 'public') | ternary(ec2_vpc_public_subnet_id, ec2_vpc_private_subnet_id) }}" + + - name: service-foo, use systemd module unless upstart is present, then use old service module + service: + state: restarted + enabled: yes + use: "{{ (ansible_service_mgr == 'upstart') | ternary('service', 'systemd') }}" + +RETURN: + _value: + description: The value indicated by the input. + type: any diff --git a/lib/ansible/plugins/filter/to_datetime.yml b/lib/ansible/plugins/filter/to_datetime.yml new file mode 100644 index 0000000..dbd476a --- /dev/null +++ b/lib/ansible/plugins/filter/to_datetime.yml @@ -0,0 +1,35 @@ +DOCUMENTATION: + name: to_datetime + version_added: "2.4" + short_description: Get C(datetime) from string + description: + - Using the input string attempt to create a matching Python C(datetime) object. + notes: + - For a full list of format codes for working with Python date format strings, see + L(the Python documentation, https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior). + positional: _input + options: + _input: + description: A string containing date time information. + type: str + required: true + format: + description: C(strformat) formatted string that describes the expected format of the input string. + type: str + +EXAMPLES: | + + # Get total amount of seconds between two dates. Default date format is %Y-%m-%d %H:%M:%S but you can pass your own format + secsdiff: '{{ (("2016-08-14 20:00:12" | to_datetime) - ("2015-12-25" | to_datetime("%Y-%m-%d"))).total_seconds() }}' + + # Get remaining seconds after delta has been calculated. NOTE: This does NOT convert years, days, hours, and so on to seconds. For that, use total_seconds() + {{ (("2016-08-14 20:00:12" | to_datetime) - ("2016-08-14 18:00:00" | to_datetime)).seconds }} + # This expression evaluates to "12" and not "132". Delta is 2 hours, 12 seconds + + # get amount of days between two dates. This returns only number of days and discards remaining hours, minutes, and seconds + {{ (("2016-08-14 20:00:12" | to_datetime) - ("2015-12-25" | to_datetime('%Y-%m-%d'))).days }} + +RETURN: + _value: + description: C(datetime) object from the represented value. + type: raw diff --git a/lib/ansible/plugins/filter/to_json.yml b/lib/ansible/plugins/filter/to_json.yml new file mode 100644 index 0000000..6f32d7c --- /dev/null +++ b/lib/ansible/plugins/filter/to_json.yml @@ -0,0 +1,69 @@ +DOCUMENTATION: + name: to_json + author: core team + version_added: 'historical' + short_description: Convert variable to JSON string + description: + - Converts an Ansible variable into a JSON string representation. + - This filter functions as a wrapper to the Python C(json.dumps) function. + - Ansible internally auto-converts JSON strings into variable structures so this plugin is used to force it into a JSON string. + options: + _input: + description: A variable or expression that returns a data structure. + type: raw + required: true + vault_to_text: + description: Toggle to either unvault a vault or create the JSON version of a vaulted object. + type: bool + default: True + version_added: '2.9' + preprocess_unsafe: + description: Toggle to represent unsafe values directly in JSON or create a unsafe object in JSON. + type: bool + default: True + version_added: '2.9' + allow_nan: + description: When C(False), strict adherence to float value limits of the JSON specifications, so C(nan), C(inf) and C(-inf) values will produce errors. + When C(True), JavaScript equivalents will be used (C(NaN), C(Infinity), C(-Infinity)). + default: True + type: bool + check_circular: + description: Controls the usage of the internal circular reference detection, if off can result in overflow errors. + default: True + type: bool + ensure_ascii: + description: Escapes all non ASCII characters. + default: True + type: bool + indent: + description: Number of spaces to indent Python structures, mainly used for display to humans. + default: 0 + type: integer + separators: + description: The C(item) and C(key) separator to be used in the serialized output, + default may change depending on I(indent) and Python version. + default: "(', ', ': ')" + type: tuple + skipkeys: + description: If C(True), keys that are not basic Python types will be skipped. + default: False + type: bool + sort_keys: + description: Affects sorting of dictionary keys. + default: False + type: bool + notes: + - Both I(vault_to_text) and I(preprocess_unsafe) defaulted to C(False) between Ansible 2.9 and 2.12. + - 'These parameters to C(json.dumps) will be ignored, as they are overriden internally: I(cls), I(default)' + +EXAMPLES: | + # dump variable in a template to create a JSON document + {{ docker_config|to_json }} + + # same as above but 'prettier' (equivalent to to_nice_json filter) + {{ docker_config|to_json(indent=4, sort_keys=True) }} + +RETURN: + _value: + description: The JSON serialized string representing the variable structure inputted. + type: string diff --git a/lib/ansible/plugins/filter/to_nice_json.yml b/lib/ansible/plugins/filter/to_nice_json.yml new file mode 100644 index 0000000..bedc18b --- /dev/null +++ b/lib/ansible/plugins/filter/to_nice_json.yml @@ -0,0 +1,54 @@ +DOCUMENTATION: + name: to_nice_json + author: core team + version_added: 'historical' + short_description: Convert variable to 'nicely formatted' JSON string + description: + - Converts an Ansible variable into a 'nicely formatted' JSON string representation + - This filter functions as a wrapper to the Python C(json.dumps) function. + - Ansible automatically converts JSON strings into variable structures so this plugin is used to forcibly retain a JSON string. + options: + _input: + description: A variable or expression that returns a data structure. + type: raw + required: true + vault_to_text: + description: Toggle to either unvault a vault or create the JSON version of a vaulted object. + type: bool + default: True + version_added: '2.9' + preprocess_unsafe: + description: Toggle to represent unsafe values directly in JSON or create a unsafe object in JSON. + type: bool + default: True + version_added: '2.9' + allow_nan: + description: When C(False), strict adherence to float value limits of the JSON specification, so C(nan), C(inf) and C(-inf) values will produce errors. + When C(True), JavaScript equivalents will be used (C(NaN), C(Infinity), C(-Infinity)). + default: True + type: bool + check_circular: + description: Controls the usage of the internal circular reference detection, if off can result in overflow errors. + default: True + type: bool + ensure_ascii: + description: Escapes all non ASCII characters. + default: True + type: bool + skipkeys: + description: If C(True), keys that are not basic Python types will be skipped. + default: False + type: bool + notes: + - Both I(vault_to_text) and I(preprocess_unsafe) defaulted to C(False) between Ansible 2.9 and 2.12. + - 'These parameters to C(json.dumps) will be ignored, they are overriden for internal use: I(cls), I(default), I(indent), I(separators), I(sort_keys).' + +EXAMPLES: | + # dump variable in a template to create a nicely formatted JSON document + {{ docker_config|to_nice_json }} + + +RETURN: + _value: + description: The 'nicely formatted' JSON serialized string representing the variable structure inputted. + type: string diff --git a/lib/ansible/plugins/filter/to_nice_yaml.yml b/lib/ansible/plugins/filter/to_nice_yaml.yml new file mode 100644 index 0000000..4677a86 --- /dev/null +++ b/lib/ansible/plugins/filter/to_nice_yaml.yml @@ -0,0 +1,39 @@ +DOCUMENTATION: + name: to_yaml + author: core team + version_added: 'historical' + short_description: Convert variable to YAML string + description: + - Converts an Ansible variable into a YAML string representation. + - This filter functions as a wrapper to the L(Python PyYAML library, https://pypi.org/project/PyYAML/)'s C(yaml.dump) function. + - Ansible internally auto-converts YAML strings into variable structures so this plugin is used to force it into a YAML string. + positional: _input + options: + _input: + description: A variable or expression that returns a data structure. + type: raw + required: true + indent: + description: Number of spaces to indent Python structures, mainly used for display to humans. + type: integer + sort_keys: + description: Affects sorting of dictionary keys. + default: True + type: bool + #allow_unicode: + # description: + # type: bool + # default: true + #default_style=None, canonical=None, width=None, line_break=None, encoding=None, explicit_start=None, explicit_end=None, version=None, tags=None + notes: + - More options may be available, see L(PyYAML documentation, https://pyyaml.org/wiki/PyYAMLDocumentation) for details. + - 'These parameters to C(yaml.dump) will be ignored, as they are overriden internally: I(default_flow_style)' + +EXAMPLES: | + # dump variable in a template to create a YAML document + {{ github_workflow | to_nice_yaml }} + +RETURN: + _value: + description: The YAML serialized string representing the variable structure inputted. + type: string diff --git a/lib/ansible/plugins/filter/to_uuid.yml b/lib/ansible/plugins/filter/to_uuid.yml new file mode 100644 index 0000000..266bf05 --- /dev/null +++ b/lib/ansible/plugins/filter/to_uuid.yml @@ -0,0 +1,30 @@ +DOCUMENTATION: + name: to_uuid + version_added: "2.9" + short_description: namespaced UUID generator + description: + - Use to generate namespeced Universal Unique ID. + positional: _input, namespace + options: + _input: + description: String to use as base fo the UUID. + type: str + required: true + namespace: + description: UUID namespace to use. + type: str + default: 361E6D51-FAEC-444A-9079-341386DA8E2E + +EXAMPLES: | + + # To create a namespaced UUIDv5 + uuid: "{{ string | to_uuid(namespace='11111111-2222-3333-4444-555555555555') }}" + + + # To create a namespaced UUIDv5 using the default Ansible namespace '361E6D51-FAEC-444A-9079-341386DA8E2E' + uuid: "{{ string | to_uuid }}" + +RETURN: + _value: + description: Generated UUID. + type: string diff --git a/lib/ansible/plugins/filter/to_yaml.yml b/lib/ansible/plugins/filter/to_yaml.yml new file mode 100644 index 0000000..2e7be60 --- /dev/null +++ b/lib/ansible/plugins/filter/to_yaml.yml @@ -0,0 +1,52 @@ +DOCUMENTATION: + name: to_yaml + author: core team + version_added: 'historical' + short_description: Convert variable to YAML string + description: + - Converts an Ansible variable into a YAML string representation. + - This filter functions as a wrapper to the L(Python PyYAML library, https://pypi.org/project/PyYAML/)'s C(yaml.dump) function. + - Ansible automatically converts YAML strings into variable structures so this plugin is used to forcibly retain a YAML string. + positional: _input + options: + _input: + description: A variable or expression that returns a data structure. + type: raw + required: true + indent: + description: Number of spaces to indent Python structures, mainly used for display to humans. + type: integer + sort_keys: + description: Affects sorting of dictionary keys. + default: True + type: bool + notes: + - More options may be available, see L(PyYAML documentation, https://pyyaml.org/wiki/PyYAMLDocumentation) for details. + + # TODO: find docs for these + #allow_unicode: + # description: + # type: bool + # default: true + #default_flow_style + #default_style + #canonical=None, + #width=None, + #line_break=None, + #encoding=None, + #explicit_start=None, + #explicit_end=None, + #version=None, + #tags=None + +EXAMPLES: | + # dump variable in a template to create a YAML document + {{ github_workflow |to_yaml}} + + # same as above but 'prettier' (equivalent to to_nice_yaml filter) + {{ docker_config|to_json(indent=4) }} + +RETURN: + _value: + description: The YAML serialized string representing the variable structure inputted. + type: string diff --git a/lib/ansible/plugins/filter/type_debug.yml b/lib/ansible/plugins/filter/type_debug.yml new file mode 100644 index 0000000..73f7946 --- /dev/null +++ b/lib/ansible/plugins/filter/type_debug.yml @@ -0,0 +1,20 @@ +DOCUMENTATION: + name: type_debug + author: Adrian Likins (@alikins) + version_added: "2.3" + short_description: show input data type + description: + - Returns the equivalent of Python's C(type) function. + options: + _input: + description: Variable or expression of which you want to determine type. + type: any + required: true +EXAMPLES: | + # get type of 'myvar' + {{ myvar | type_debug }} + +RETURN: + _value: + description: The Python 'type' of the I(_input) provided. + type: string diff --git a/lib/ansible/plugins/filter/union.yml b/lib/ansible/plugins/filter/union.yml new file mode 100644 index 0000000..d737900 --- /dev/null +++ b/lib/ansible/plugins/filter/union.yml @@ -0,0 +1,35 @@ +DOCUMENTATION: + name: union + author: Brian Coca (@bcoca) + version_added: "1.4" + short_description: union of lists + description: + - Provide a unique list of all the elements of two lists. + options: + _input: + description: A list. + type: list + required: true + _second_list: + description: A list. + type: list + required: true + seealso: + - plugin_type: filter + plugin: ansible.builtin.difference + - plugin_type: filter + plugin: ansible.builtin.intersect + - plugin_type: filter + plugin: ansible.builtin.symmetric_difference + - plugin_type: filter + plugin: ansible.builtin.unique +EXAMPLES: | + # return the unique elements of list1 added to list2 + # list1: [1, 2, 5, 1, 3, 4, 10] + # list2: [1, 2, 3, 4, 5, 11, 99] + {{ list1 | union(list2) }} + # => [1, 2, 5, 1, 3, 4, 10, 11, 99] +RETURN: + _value: + description: A unique list of all the elements from both lists. + type: list diff --git a/lib/ansible/plugins/filter/unique.yml b/lib/ansible/plugins/filter/unique.yml new file mode 100644 index 0000000..c627816 --- /dev/null +++ b/lib/ansible/plugins/filter/unique.yml @@ -0,0 +1,30 @@ +DOCUMENTATION: + name: unique + author: Brian Coca (@bcoca) + version_added: "1.4" + short_description: set of unique items of a list + description: + - Creates a list of unique elements (a set) from the provided input list. + options: + _input: + description: A list. + type: list + required: true + seealso: + - plugin_type: filter + plugin: ansible.builtin.difference + - plugin_type: filter + plugin: ansible.builtin.intersect + - plugin_type: filter + plugin: ansible.builtin.symmetric_difference + - plugin_type: filter + plugin: ansible.builtin.union +EXAMPLES: | + # return only the unique elements of list1 + # list1: [1, 2, 5, 1, 3, 4, 10] + {{ list1 | unique }} + # => [1, 2, 5, 3, 4, 10] +RETURN: + _value: + description: A list with unique elements, also known as a set. + type: list diff --git a/lib/ansible/plugins/filter/unvault.yml b/lib/ansible/plugins/filter/unvault.yml new file mode 100644 index 0000000..96a82ca --- /dev/null +++ b/lib/ansible/plugins/filter/unvault.yml @@ -0,0 +1,36 @@ +DOCUMENTATION: + name: unvault + author: Brian Coca (@bcoca) + version_added: "2.12" + short_description: Open an Ansible Vault + description: + - Retrieve your information from an encrypted Ansible Vault. + positional: secret + options: + _input: + description: Vault string, or an C(AnsibleVaultEncryptedUnicode) string object. + type: string + required: true + secret: + description: Vault secret, the key that lets you open the vault. + type: string + required: true + vault_id: + description: Secret identifier, used internally to try to best match a secret when multiple are provided. + type: string + default: 'filter_default' + +EXAMPLES: | + # simply decrypt my key from a vault + vars: + mykey: "{{ myvaultedkey|unvault(passphrase) }} " + + - name: save templated unvaulted data + template: src=dump_template_data.j2 dest=/some/key/clear.txt + vars: + template_data: '{{ secretdata|uvault(vaultsecret) }}' + +RETURN: + _value: + description: The string that was contained in the vault. + type: string diff --git a/lib/ansible/plugins/filter/urldecode.yml b/lib/ansible/plugins/filter/urldecode.yml new file mode 100644 index 0000000..dd76937 --- /dev/null +++ b/lib/ansible/plugins/filter/urldecode.yml @@ -0,0 +1,48 @@ +DOCUMENTATION: + name: urlsplit + version_added: "2.4" + short_description: get components from URL + description: + - Split a URL into its component parts. + positional: _input, query + options: + _input: + description: URL string to split. + type: str + required: true + query: + description: Specify a single component to return. + type: str + choices: ["fragment", "hostname", "netloc", "password", "path", "port", "query", "scheme", "username"] + +RETURN: + _value: + description: + - A dictionary with components as keyword and their value. + - If I(query) is provided, a string or integer will be returned instead, depending on I(query). + type: any + +EXAMPLES: | + + {{ "http://user:password@www.acme.com:9000/dir/index.html?query=term#fragment" | urlsplit }} + # => + # { + # "fragment": "fragment", + # "hostname": "www.acme.com", + # "netloc": "user:password@www.acme.com:9000", + # "password": "password", + # "path": "/dir/index.html", + # "port": 9000, + # "query": "query=term", + # "scheme": "http", + # "username": "user" + # } + + {{ "http://user:password@www.acme.com:9000/dir/index.html?query=term#fragment" | urlsplit('hostname') }} + # => 'www.acme.com' + + {{ "http://user:password@www.acme.com:9000/dir/index.html?query=term#fragment" | urlsplit('query') }} + # => 'query=term' + + {{ "http://user:password@www.acme.com:9000/dir/index.html?query=term#fragment" | urlsplit('path') }} + # => '/dir/index.html' diff --git a/lib/ansible/plugins/filter/urls.py b/lib/ansible/plugins/filter/urls.py new file mode 100644 index 0000000..fb7abc6 --- /dev/null +++ b/lib/ansible/plugins/filter/urls.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- + +# Copyright: (c) 2012, Dag Wieers (@dagwieers) <dag@wieers.com> +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + +from functools import partial + +from urllib.parse import unquote_plus + + +class FilterModule(object): + ''' Ansible core jinja2 filters ''' + + def filters(self): + return { + 'urldecode': partial(unquote_plus), + } diff --git a/lib/ansible/plugins/filter/urlsplit.py b/lib/ansible/plugins/filter/urlsplit.py new file mode 100644 index 0000000..cce54bb --- /dev/null +++ b/lib/ansible/plugins/filter/urlsplit.py @@ -0,0 +1,87 @@ +# Copyright (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = r''' + name: urlsplit + version_added: "2.4" + short_description: get components from URL + description: + - Split a URL into its component parts. + positional: _input, query + options: + _input: + description: URL string to split. + type: str + required: true + query: + description: Specify a single component to return. + type: str + choices: ["fragment", "hostname", "netloc", "password", "path", "port", "query", "scheme", "username"] +''' + +EXAMPLES = r''' + + parts: '{{ "http://user:password@www.acme.com:9000/dir/index.html?query=term#fragment" | urlsplit }}' + # => + # { + # "fragment": "fragment", + # "hostname": "www.acme.com", + # "netloc": "user:password@www.acme.com:9000", + # "password": "password", + # "path": "/dir/index.html", + # "port": 9000, + # "query": "query=term", + # "scheme": "http", + # "username": "user" + # } + + hostname: '{{ "http://user:password@www.acme.com:9000/dir/index.html?query=term#fragment" | urlsplit("hostname") }}' + # => 'www.acme.com' + + query: '{{ "http://user:password@www.acme.com:9000/dir/index.html?query=term#fragment" | urlsplit("query") }}' + # => 'query=term' + + path: '{{ "http://user:password@www.acme.com:9000/dir/index.html?query=term#fragment" | urlsplit("path") }}' + # => '/dir/index.html' +''' + +RETURN = r''' + _value: + description: + - A dictionary with components as keyword and their value. + - If I(query) is provided, a string or integer will be returned instead, depending on I(query). + type: any +''' + +from urllib.parse import urlsplit + +from ansible.errors import AnsibleFilterError +from ansible.utils import helpers + + +def split_url(value, query='', alias='urlsplit'): + + results = helpers.object_to_dict(urlsplit(value), exclude=['count', 'index', 'geturl', 'encode']) + + # If a query is supplied, make sure it's valid then return the results. + # If no option is supplied, return the entire dictionary. + if query: + if query not in results: + raise AnsibleFilterError(alias + ': unknown URL component: %s' % query) + return results[query] + else: + return results + + +# ---- Ansible filters ---- +class FilterModule(object): + ''' URI filter ''' + + def filters(self): + return { + 'urlsplit': split_url + } diff --git a/lib/ansible/plugins/filter/vault.yml b/lib/ansible/plugins/filter/vault.yml new file mode 100644 index 0000000..1ad541e --- /dev/null +++ b/lib/ansible/plugins/filter/vault.yml @@ -0,0 +1,48 @@ +DOCUMENTATION: + name: vault + author: Brian Coca (@bcoca) + version_added: "2.12" + short_description: vault your secrets + description: + - Put your information into an encrypted Ansible Vault. + positional: secret + options: + _input: + description: Data to vault. + type: string + required: true + secret: + description: Vault secret, the key that lets you open the vault. + type: string + required: true + salt: + description: + - Encryption salt, will be random if not provided. + - While providing one makes the resulting encrypted string reproducible, it can lower the security of the vault. + type: string + vault_id: + description: Secret identifier, used internally to try to best match a secret when multiple are provided. + type: string + default: 'filter_default' + wrap_object: + description: + - This toggle can force the return of an C(AnsibleVaultEncryptedUnicode) string object, when C(False), you get a simple string. + - Mostly useful when combining with the C(to_yaml) filter to output the 'inline vault' format. + type: bool + default: False + +EXAMPLES: | + # simply encrypt my key in a vault + vars: + myvaultedkey: "{{ keyrawdata|vault(passphrase) }} " + + - name: save templated vaulted data + template: src=dump_template_data.j2 dest=/some/key/vault.txt + vars: + mysalt: '{{2**256|random(seed=inventory_hostname)}}' + template_data: '{{ secretdata|vault(vaultsecret, salt=mysalt) }}' + +RETURN: + _value: + description: The vault string that contains the secret data (or C(AnsibleVaultEncryptedUnicode) string object). + type: string diff --git a/lib/ansible/plugins/filter/win_basename.yml b/lib/ansible/plugins/filter/win_basename.yml new file mode 100644 index 0000000..f89baa5 --- /dev/null +++ b/lib/ansible/plugins/filter/win_basename.yml @@ -0,0 +1,24 @@ +DOCUMENTATION: + name: win_basename + author: ansible core team + version_added: "2.0" + short_description: Get a Windows path's base name + description: + - Returns the last name component of a Windows path, what is left in the string that is not 'win_dirname'. + options: + _input: + description: A Windows path. + type: str + required: true + seealso: + - plugin_type: filter + plugin: ansible.builtin.win_dirname +EXAMPLES: | + + # To get the last name of a file Windows path, like 'foo.txt' out of 'C:\Users\asdf\foo.txt' + {{ mypath | win_basename }} + +RETURN: + _value: + description: The base name from the Windows path provided. + type: str diff --git a/lib/ansible/plugins/filter/win_dirname.yml b/lib/ansible/plugins/filter/win_dirname.yml new file mode 100644 index 0000000..dbc85c7 --- /dev/null +++ b/lib/ansible/plugins/filter/win_dirname.yml @@ -0,0 +1,24 @@ +DOCUMENTATION: + name: win_dirname + author: ansible core team + version_added: "2.0" + short_description: Get a Windows path's directory + description: + - Returns the directory component of a Windows path, what is left in the string that is not 'win_basename'. + options: + _input: + description: A Windows path. + type: str + required: true + seealso: + - plugin_type: filter + plugin: ansible.builtin.win_basename +EXAMPLES: | + + # To get the last name of a file Windows path, like 'C:\users\asdf' out of 'C:\Users\asdf\foo.txt' + {{ mypath | win_dirname }} + +RETURN: + _value: + description: The directory from the Windows path provided. + type: str diff --git a/lib/ansible/plugins/filter/win_splitdrive.yml b/lib/ansible/plugins/filter/win_splitdrive.yml new file mode 100644 index 0000000..828d1dd --- /dev/null +++ b/lib/ansible/plugins/filter/win_splitdrive.yml @@ -0,0 +1,29 @@ +DOCUMENTATION: + name: win_splitdrive + author: ansible core team + version_added: "2.0" + short_description: Split a Windows path by the drive letter + description: + - Returns a list with the first component being the drive letter and the second, the rest of the path. + options: + _input: + description: A Windows path. + type: str + required: true + +EXAMPLES: | + + # To get the last name of a file Windows path, like ['C', '\Users\asdf\foo.txt'] out of 'C:\Users\asdf\foo.txt' + {{ mypath | win_splitdrive }} + + # just the drive letter + {{ mypath | win_splitdrive | first }} + + # path w/o drive letter + {{ mypath | win_splitdrive | last }} + +RETURN: + _value: + description: List in which the first element is the drive letter and the second the rest of the path. + type: list + elements: str diff --git a/lib/ansible/plugins/filter/zip.yml b/lib/ansible/plugins/filter/zip.yml new file mode 100644 index 0000000..20d7a9b --- /dev/null +++ b/lib/ansible/plugins/filter/zip.yml @@ -0,0 +1,43 @@ +DOCUMENTATION: + name: zip + version_added: "2.3" + short_description: combine list elements + positional: _input, _additional_lists + description: Iterate over several iterables in parallel, producing tuples with an item from each one. + notes: + - This is mostly a passhtrough to Python's C(zip) function. + options: + _input: + description: Original list. + type: list + elements: any + required: yes + _additional_lists: + description: Additional list(s). + type: list + elements: any + required: yes + strict: + description: If C(True) return an error on mismatching list length, otherwise shortest list determines output. + type: bool + default: no + +EXAMPLES: | + + # two => [[1, "a"], [2, "b"], [3, "c"], [4, "d"], [5, "e"], [6, "f"]] + two: "{{ [1,2,3,4,5,6] | zip(['a','b','c','d','e','f']) }}" + + # three => [ [ 1, "a", "d" ], [ 2, "b", "e" ], [ 3, "c", "f" ] ] + three: "{{ [1,2,3] | zip(['a','b','c'], ['d','e','f']) }}" + + # shorter => [[1, "a"], [2, "b"], [3, "c"]] + shorter: "{{ [1,2,3] | zip(['a','b','c','d','e','f']) }}" + + # compose dict from lists of keys and values + mydcit: "{{ dict(keys_list | zip(values_list)) }}" + +RETURN: + _value: + description: List of lists made of elements matching the positions of the input lists. + type: list + elements: list diff --git a/lib/ansible/plugins/filter/zip_longest.yml b/lib/ansible/plugins/filter/zip_longest.yml new file mode 100644 index 0000000..db351b4 --- /dev/null +++ b/lib/ansible/plugins/filter/zip_longest.yml @@ -0,0 +1,36 @@ +DOCUMENTATION: + name: zip_longest + version_added: "2.3" + short_description: combine list elements, with filler + positional: _input, _additional_lists + description: + - Make an iterator that aggregates elements from each of the iterables. + If the iterables are of uneven length, missing values are filled-in with I(fillvalue). + Iteration continues until the longest iterable is exhausted. + notes: + - This is mostly a passhtrough to Python's C(itertools.zip_longest) function + options: + _input: + description: Original list. + type: list + elements: any + required: yes + _additional_lists: + description: Additional list(s). + type: list + elements: any + required: yes + fillvalue: + description: Filler value to add to output when one of the lists does not contain enough elements to match the others. + type: any + +EXAMPLES: | + + # X_fill => [[1, "a", 21], [2, "b", 22], [3, "c", 23], ["X", "d", "X"], ["X", "e", "X"], ["X", "f", "X"]] + X_fill: "{{ [1,2,3] | zip_longest(['a','b','c','d','e','f'], [21, 22, 23], fillvalue='X') }}" + +RETURN: + _value: + description: List of lists made of elements matching the positions of the input lists. + type: list + elements: list diff --git a/lib/ansible/plugins/httpapi/__init__.py b/lib/ansible/plugins/httpapi/__init__.py new file mode 100644 index 0000000..0773921 --- /dev/null +++ b/lib/ansible/plugins/httpapi/__init__.py @@ -0,0 +1,87 @@ +# (c) 2018 Red Hat Inc. +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from abc import abstractmethod + +from ansible.plugins import AnsiblePlugin + + +class HttpApiBase(AnsiblePlugin): + def __init__(self, connection): + super(HttpApiBase, self).__init__() + + self.connection = connection + self._become = False + self._become_pass = '' + + def set_become(self, become_context): + self._become = become_context.become + self._become_pass = getattr(become_context, 'become_pass') or '' + + def login(self, username, password): + """Call a defined login endpoint to receive an authentication token. + + This should only be implemented if the API has a single endpoint which + can turn HTTP basic auth into a token which can be reused for the rest + of the calls for the session. + """ + pass + + def logout(self): + """ Call to implement session logout. + + Method to clear session gracefully e.g. tokens granted in login + need to be revoked. + """ + pass + + def update_auth(self, response, response_text): + """Return per-request auth token. + + The response should be a dictionary that can be plugged into the + headers of a request. The default implementation uses cookie data. + If no authentication data is found, return None + """ + cookie = response.info().get('Set-Cookie') + if cookie: + return {'Cookie': cookie} + + return None + + def handle_httperror(self, exc): + """Overridable method for dealing with HTTP codes. + + This method will attempt to handle known cases of HTTP status codes. + If your API uses status codes to convey information in a regular way, + you can override this method to handle it appropriately. + + :returns: + * True if the code has been handled in a way that the request + may be resent without changes. + * False if the error cannot be handled or recovered from by the + plugin. This will result in the HTTPError being raised as an + exception for the caller to deal with as appropriate (most likely + by failing). + * Any other value returned is taken as a valid response from the + server without making another request. In many cases, this can just + be the original exception. + """ + if exc.code == 401: + if self.connection._auth: + # Stored auth appears to be invalid, clear and retry + self.connection._auth = None + self.login(self.connection.get_option('remote_user'), self.connection.get_option('password')) + return True + else: + # Unauthorized and there's no token. Return an error + return False + + return exc + + @abstractmethod + def send_request(self, data, **message_kwargs): + """Prepares and sends request(s) to device.""" + pass diff --git a/lib/ansible/plugins/inventory/__init__.py b/lib/ansible/plugins/inventory/__init__.py new file mode 100644 index 0000000..c0b4264 --- /dev/null +++ b/lib/ansible/plugins/inventory/__init__.py @@ -0,0 +1,463 @@ +# (c) 2017, Red Hat, inc +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <https://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import hashlib +import os +import string + +from collections.abc import Mapping + +from ansible.errors import AnsibleError, AnsibleParserError +from ansible.inventory.group import to_safe_group_name as original_safe +from ansible.parsing.utils.addresses import parse_address +from ansible.plugins import AnsiblePlugin +from ansible.plugins.cache import CachePluginAdjudicator as CacheObject +from ansible.module_utils._text import to_bytes, to_native +from ansible.module_utils.parsing.convert_bool import boolean +from ansible.module_utils.six import string_types +from ansible.template import Templar +from ansible.utils.display import Display +from ansible.utils.vars import combine_vars, load_extra_vars + +display = Display() + + +# Helper methods +def to_safe_group_name(name): + # placeholder for backwards compat + return original_safe(name, force=True, silent=True) + + +def detect_range(line=None): + ''' + A helper function that checks a given host line to see if it contains + a range pattern described in the docstring above. + + Returns True if the given line contains a pattern, else False. + ''' + return '[' in line + + +def expand_hostname_range(line=None): + ''' + A helper function that expands a given line that contains a pattern + specified in top docstring, and returns a list that consists of the + expanded version. + + The '[' and ']' characters are used to maintain the pseudo-code + appearance. They are replaced in this function with '|' to ease + string splitting. + + References: https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html#hosts-and-groups + ''' + all_hosts = [] + if line: + # A hostname such as db[1:6]-node is considered to consists + # three parts: + # head: 'db' + # nrange: [1:6]; range() is a built-in. Can't use the name + # tail: '-node' + + # Add support for multiple ranges in a host so: + # db[01:10:3]node-[01:10] + # - to do this we split off at the first [...] set, getting the list + # of hosts and then repeat until none left. + # - also add an optional third parameter which contains the step. (Default: 1) + # so range can be [01:10:2] -> 01 03 05 07 09 + + (head, nrange, tail) = line.replace('[', '|', 1).replace(']', '|', 1).split('|') + bounds = nrange.split(":") + if len(bounds) != 2 and len(bounds) != 3: + raise AnsibleError("host range must be begin:end or begin:end:step") + beg = bounds[0] + end = bounds[1] + if len(bounds) == 2: + step = 1 + else: + step = bounds[2] + if not beg: + beg = "0" + if not end: + raise AnsibleError("host range must specify end value") + if beg[0] == '0' and len(beg) > 1: + rlen = len(beg) # range length formatting hint + if rlen != len(end): + raise AnsibleError("host range must specify equal-length begin and end formats") + + def fill(x): + return str(x).zfill(rlen) # range sequence + + else: + fill = str + + try: + i_beg = string.ascii_letters.index(beg) + i_end = string.ascii_letters.index(end) + if i_beg > i_end: + raise AnsibleError("host range must have begin <= end") + seq = list(string.ascii_letters[i_beg:i_end + 1:int(step)]) + except ValueError: # not an alpha range + seq = range(int(beg), int(end) + 1, int(step)) + + for rseq in seq: + hname = ''.join((head, fill(rseq), tail)) + + if detect_range(hname): + all_hosts.extend(expand_hostname_range(hname)) + else: + all_hosts.append(hname) + + return all_hosts + + +def get_cache_plugin(plugin_name, **kwargs): + try: + cache = CacheObject(plugin_name, **kwargs) + except AnsibleError as e: + if 'fact_caching_connection' in to_native(e): + raise AnsibleError("error, '%s' inventory cache plugin requires the one of the following to be set " + "to a writeable directory path:\nansible.cfg:\n[default]: fact_caching_connection,\n" + "[inventory]: cache_connection;\nEnvironment:\nANSIBLE_INVENTORY_CACHE_CONNECTION,\n" + "ANSIBLE_CACHE_PLUGIN_CONNECTION." % plugin_name) + else: + raise e + + if plugin_name != 'memory' and kwargs and not getattr(cache._plugin, '_options', None): + raise AnsibleError('Unable to use cache plugin {0} for inventory. Cache options were provided but may not reconcile ' + 'correctly unless set via set_options. Refer to the porting guide if the plugin derives user settings ' + 'from ansible.constants.'.format(plugin_name)) + return cache + + +class BaseInventoryPlugin(AnsiblePlugin): + """ Parses an Inventory Source""" + + TYPE = 'generator' + + # 3rd party plugins redefine this to + # use custom group name sanitization + # since constructed features enforce + # it by default. + _sanitize_group_name = staticmethod(to_safe_group_name) + + def __init__(self): + + super(BaseInventoryPlugin, self).__init__() + + self._options = {} + self.inventory = None + self.display = display + self._vars = {} + + def parse(self, inventory, loader, path, cache=True): + ''' Populates inventory from the given data. Raises an error on any parse failure + :arg inventory: a copy of the previously accumulated inventory data, + to be updated with any new data this plugin provides. + The inventory can be empty if no other source/plugin ran successfully. + :arg loader: a reference to the DataLoader, which can read in YAML and JSON files, + it also has Vault support to automatically decrypt files. + :arg path: the string that represents the 'inventory source', + normally a path to a configuration file for this inventory, + but it can also be a raw string for this plugin to consume + :arg cache: a boolean that indicates if the plugin should use the cache or not + you can ignore if this plugin does not implement caching. + ''' + + self.loader = loader + self.inventory = inventory + self.templar = Templar(loader=loader) + self._vars = load_extra_vars(loader) + + def verify_file(self, path): + ''' Verify if file is usable by this plugin, base does minimal accessibility check + :arg path: a string that was passed as an inventory source, + it normally is a path to a config file, but this is not a requirement, + it can also be parsed itself as the inventory data to process. + So only call this base class if you expect it to be a file. + ''' + + valid = False + b_path = to_bytes(path, errors='surrogate_or_strict') + if (os.path.exists(b_path) and os.access(b_path, os.R_OK)): + valid = True + else: + self.display.vvv('Skipping due to inventory source not existing or not being readable by the current user') + return valid + + def _populate_host_vars(self, hosts, variables, group=None, port=None): + if not isinstance(variables, Mapping): + raise AnsibleParserError("Invalid data from file, expected dictionary and got:\n\n%s" % to_native(variables)) + + for host in hosts: + self.inventory.add_host(host, group=group, port=port) + for k in variables: + self.inventory.set_variable(host, k, variables[k]) + + def _read_config_data(self, path): + ''' validate config and set options as appropriate + :arg path: path to common yaml format config file for this plugin + ''' + + config = {} + try: + # avoid loader cache so meta: refresh_inventory can pick up config changes + # if we read more than once, fs cache should be good enough + config = self.loader.load_from_file(path, cache=False) + except Exception as e: + raise AnsibleParserError(to_native(e)) + + # a plugin can be loaded via many different names with redirection- if so, we want to accept any of those names + valid_names = getattr(self, '_redirected_names') or [self.NAME] + + if not config: + # no data + raise AnsibleParserError("%s is empty" % (to_native(path))) + elif config.get('plugin') not in valid_names: + # this is not my config file + raise AnsibleParserError("Incorrect plugin name in file: %s" % config.get('plugin', 'none found')) + elif not isinstance(config, Mapping): + # configs are dictionaries + raise AnsibleParserError('inventory source has invalid structure, it should be a dictionary, got: %s' % type(config)) + + self.set_options(direct=config, var_options=self._vars) + if 'cache' in self._options and self.get_option('cache'): + cache_option_keys = [('_uri', 'cache_connection'), ('_timeout', 'cache_timeout'), ('_prefix', 'cache_prefix')] + cache_options = dict((opt[0], self.get_option(opt[1])) for opt in cache_option_keys if self.get_option(opt[1]) is not None) + self._cache = get_cache_plugin(self.get_option('cache_plugin'), **cache_options) + + return config + + def _consume_options(self, data): + ''' update existing options from alternate configuration sources not normally used by Ansible. + Many API libraries already have existing configuration sources, this allows plugin author to leverage them. + :arg data: key/value pairs that correspond to configuration options for this plugin + ''' + + for k in self._options: + if k in data: + self._options[k] = data.pop(k) + + def _expand_hostpattern(self, hostpattern): + ''' + Takes a single host pattern and returns a list of hostnames and an + optional port number that applies to all of them. + ''' + # Can the given hostpattern be parsed as a host with an optional port + # specification? + + try: + (pattern, port) = parse_address(hostpattern, allow_ranges=True) + except Exception: + # not a recognizable host pattern + pattern = hostpattern + port = None + + # Once we have separated the pattern, we expand it into list of one or + # more hostnames, depending on whether it contains any [x:y] ranges. + + if detect_range(pattern): + hostnames = expand_hostname_range(pattern) + else: + hostnames = [pattern] + + return (hostnames, port) + + +class BaseFileInventoryPlugin(BaseInventoryPlugin): + """ Parses a File based Inventory Source""" + + TYPE = 'storage' + + def __init__(self): + + super(BaseFileInventoryPlugin, self).__init__() + + +class Cacheable(object): + + _cache = CacheObject() + + @property + def cache(self): + return self._cache + + def load_cache_plugin(self): + plugin_name = self.get_option('cache_plugin') + cache_option_keys = [('_uri', 'cache_connection'), ('_timeout', 'cache_timeout'), ('_prefix', 'cache_prefix')] + cache_options = dict((opt[0], self.get_option(opt[1])) for opt in cache_option_keys if self.get_option(opt[1]) is not None) + self._cache = get_cache_plugin(plugin_name, **cache_options) + + def get_cache_key(self, path): + return "{0}_{1}".format(self.NAME, self._get_cache_prefix(path)) + + def _get_cache_prefix(self, path): + ''' create predictable unique prefix for plugin/inventory ''' + + m = hashlib.sha1() + m.update(to_bytes(self.NAME, errors='surrogate_or_strict')) + d1 = m.hexdigest() + + n = hashlib.sha1() + n.update(to_bytes(path, errors='surrogate_or_strict')) + d2 = n.hexdigest() + + return 's_'.join([d1[:5], d2[:5]]) + + def clear_cache(self): + self._cache.flush() + + def update_cache_if_changed(self): + self._cache.update_cache_if_changed() + + def set_cache_plugin(self): + self._cache.set_cache() + + +class Constructable(object): + + def _compose(self, template, variables, disable_lookups=True): + ''' helper method for plugins to compose variables for Ansible based on jinja2 expression and inventory vars''' + t = self.templar + + try: + use_extra = self.get_option('use_extra_vars') + except Exception: + use_extra = False + + if use_extra: + t.available_variables = combine_vars(variables, self._vars) + else: + t.available_variables = variables + + return t.template('%s%s%s' % (t.environment.variable_start_string, template, t.environment.variable_end_string), + disable_lookups=disable_lookups) + + def _set_composite_vars(self, compose, variables, host, strict=False): + ''' loops over compose entries to create vars for hosts ''' + if compose and isinstance(compose, dict): + for varname in compose: + try: + composite = self._compose(compose[varname], variables) + except Exception as e: + if strict: + raise AnsibleError("Could not set %s for host %s: %s" % (varname, host, to_native(e))) + continue + self.inventory.set_variable(host, varname, composite) + + def _add_host_to_composed_groups(self, groups, variables, host, strict=False, fetch_hostvars=True): + ''' helper to create complex groups for plugins based on jinja2 conditionals, hosts that meet the conditional are added to group''' + # process each 'group entry' + if groups and isinstance(groups, dict): + if fetch_hostvars: + variables = combine_vars(variables, self.inventory.get_host(host).get_vars()) + self.templar.available_variables = variables + for group_name in groups: + conditional = "{%% if %s %%} True {%% else %%} False {%% endif %%}" % groups[group_name] + group_name = self._sanitize_group_name(group_name) + try: + result = boolean(self.templar.template(conditional)) + except Exception as e: + if strict: + raise AnsibleParserError("Could not add host %s to group %s: %s" % (host, group_name, to_native(e))) + continue + + if result: + # ensure group exists, use sanitized name + group_name = self.inventory.add_group(group_name) + # add host to group + self.inventory.add_child(group_name, host) + + def _add_host_to_keyed_groups(self, keys, variables, host, strict=False, fetch_hostvars=True): + ''' helper to create groups for plugins based on variable values and add the corresponding hosts to it''' + if keys and isinstance(keys, list): + for keyed in keys: + if keyed and isinstance(keyed, dict): + + if fetch_hostvars: + variables = combine_vars(variables, self.inventory.get_host(host).get_vars()) + try: + key = self._compose(keyed.get('key'), variables) + except Exception as e: + if strict: + raise AnsibleParserError("Could not generate group for host %s from %s entry: %s" % (host, keyed.get('key'), to_native(e))) + continue + default_value_name = keyed.get('default_value', None) + trailing_separator = keyed.get('trailing_separator') + if trailing_separator is not None and default_value_name is not None: + raise AnsibleParserError("parameters are mutually exclusive for keyed groups: default_value|trailing_separator") + if key or (key == '' and default_value_name is not None): + prefix = keyed.get('prefix', '') + sep = keyed.get('separator', '_') + raw_parent_name = keyed.get('parent_group', None) + if raw_parent_name: + try: + raw_parent_name = self.templar.template(raw_parent_name) + except AnsibleError as e: + if strict: + raise AnsibleParserError("Could not generate parent group %s for group %s: %s" % (raw_parent_name, key, to_native(e))) + continue + + new_raw_group_names = [] + if isinstance(key, string_types): + # if key is empty, 'default_value' will be used as group name + if key == '' and default_value_name is not None: + new_raw_group_names.append(default_value_name) + else: + new_raw_group_names.append(key) + elif isinstance(key, list): + for name in key: + # if list item is empty, 'default_value' will be used as group name + if name == '' and default_value_name is not None: + new_raw_group_names.append(default_value_name) + else: + new_raw_group_names.append(name) + elif isinstance(key, Mapping): + for (gname, gval) in key.items(): + bare_name = '%s%s%s' % (gname, sep, gval) + if gval == '': + # key's value is empty + if default_value_name is not None: + bare_name = '%s%s%s' % (gname, sep, default_value_name) + elif trailing_separator is False: + bare_name = gname + new_raw_group_names.append(bare_name) + else: + raise AnsibleParserError("Invalid group name format, expected a string or a list of them or dictionary, got: %s" % type(key)) + + for bare_name in new_raw_group_names: + if prefix == '' and self.get_option('leading_separator') is False: + sep = '' + gname = self._sanitize_group_name('%s%s%s' % (prefix, sep, bare_name)) + result_gname = self.inventory.add_group(gname) + self.inventory.add_host(host, result_gname) + + if raw_parent_name: + parent_name = self._sanitize_group_name(raw_parent_name) + self.inventory.add_group(parent_name) + self.inventory.add_child(parent_name, result_gname) + + else: + # exclude case of empty list and dictionary, because these are valid constructions + # simply no groups need to be constructed, but are still falsy + if strict and key not in ([], {}): + raise AnsibleParserError("No key or key resulted empty for %s in host %s, invalid entry" % (keyed.get('key'), host)) + else: + raise AnsibleParserError("Invalid keyed group entry, it must be a dictionary: %s " % keyed) diff --git a/lib/ansible/plugins/inventory/advanced_host_list.py b/lib/ansible/plugins/inventory/advanced_host_list.py new file mode 100644 index 0000000..1b5d868 --- /dev/null +++ b/lib/ansible/plugins/inventory/advanced_host_list.py @@ -0,0 +1,63 @@ +# Copyright (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' + name: advanced_host_list + version_added: "2.4" + short_description: Parses a 'host list' with ranges + description: + - Parses a host list string as a comma separated values of hosts and supports host ranges. + - This plugin only applies to inventory sources that are not paths and contain at least one comma. +''' + +EXAMPLES = ''' + # simple range + # ansible -i 'host[1:10],' -m ping + + # still supports w/o ranges also + # ansible-playbook -i 'localhost,' play.yml +''' + +import os + +from ansible.errors import AnsibleError, AnsibleParserError +from ansible.module_utils._text import to_bytes, to_native, to_text +from ansible.plugins.inventory import BaseInventoryPlugin + + +class InventoryModule(BaseInventoryPlugin): + + NAME = 'advanced_host_list' + + def verify_file(self, host_list): + + valid = False + b_path = to_bytes(host_list, errors='surrogate_or_strict') + if not os.path.exists(b_path) and ',' in host_list: + valid = True + return valid + + def parse(self, inventory, loader, host_list, cache=True): + ''' parses the inventory file ''' + + super(InventoryModule, self).parse(inventory, loader, host_list) + + try: + for h in host_list.split(','): + h = h.strip() + if h: + try: + (hostnames, port) = self._expand_hostpattern(h) + except AnsibleError as e: + self.display.vvv("Unable to parse address from hostname, leaving unchanged: %s" % to_text(e)) + hostnames = [h] + port = None + + for host in hostnames: + if host not in self.inventory.hosts: + self.inventory.add_host(host, group='ungrouped', port=port) + except Exception as e: + raise AnsibleParserError("Invalid data from string, could not parse: %s" % to_native(e)) diff --git a/lib/ansible/plugins/inventory/auto.py b/lib/ansible/plugins/inventory/auto.py new file mode 100644 index 0000000..45941ca --- /dev/null +++ b/lib/ansible/plugins/inventory/auto.py @@ -0,0 +1,63 @@ +# Copyright (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' + name: auto + author: + - Matt Davis (@nitzmahone) + version_added: "2.5" + short_description: Loads and executes an inventory plugin specified in a YAML config + description: + - By enabling the C(auto) inventory plugin, any YAML inventory config file with a + C(plugin) key at its root will automatically cause the named plugin to be loaded and executed with that + config. This effectively provides automatic enabling of all installed/accessible inventory plugins. + - To disable this behavior, remove C(auto) from the C(INVENTORY_ENABLED) config element. +''' + +EXAMPLES = ''' +# This plugin is not intended for direct use; it is a fallback mechanism for automatic enabling of +# all installed inventory plugins. +''' + +from ansible.errors import AnsibleParserError +from ansible.plugins.inventory import BaseInventoryPlugin +from ansible.plugins.loader import inventory_loader + + +class InventoryModule(BaseInventoryPlugin): + + NAME = 'auto' + + def verify_file(self, path): + if not path.endswith('.yml') and not path.endswith('.yaml'): + return False + return super(InventoryModule, self).verify_file(path) + + def parse(self, inventory, loader, path, cache=True): + config_data = loader.load_from_file(path, cache=False) + + try: + plugin_name = config_data.get('plugin', None) + except AttributeError: + plugin_name = None + + if not plugin_name: + raise AnsibleParserError("no root 'plugin' key found, '{0}' is not a valid YAML inventory plugin config file".format(path)) + + plugin = inventory_loader.get(plugin_name) + + if not plugin: + raise AnsibleParserError("inventory config '{0}' specifies unknown plugin '{1}'".format(path, plugin_name)) + + if not plugin.verify_file(path): + raise AnsibleParserError("inventory source '{0}' could not be verified by inventory plugin '{1}'".format(path, plugin_name)) + + self.display.v("Using inventory plugin '{0}' to process inventory source '{1}'".format(plugin._load_name, path)) + plugin.parse(inventory, loader, path, cache=cache) + try: + plugin.update_cache_if_changed() + except AttributeError: + pass diff --git a/lib/ansible/plugins/inventory/constructed.py b/lib/ansible/plugins/inventory/constructed.py new file mode 100644 index 0000000..dd630c6 --- /dev/null +++ b/lib/ansible/plugins/inventory/constructed.py @@ -0,0 +1,177 @@ +# Copyright (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' + name: constructed + version_added: "2.4" + short_description: Uses Jinja2 to construct vars and groups based on existing inventory. + description: + - Uses a YAML configuration file with a valid YAML or C(.config) extension to define var expressions and group conditionals + - The Jinja2 conditionals that qualify a host for membership. + - The Jinja2 expressions are calculated and assigned to the variables + - Only variables already available from previous inventories or the fact cache can be used for templating. + - When I(strict) is False, failed expressions will be ignored (assumes vars were missing). + options: + plugin: + description: token that ensures this is a source file for the 'constructed' plugin. + required: True + choices: ['ansible.builtin.constructed', 'constructed'] + use_vars_plugins: + description: + - Normally, for performance reasons, vars plugins get executed after the inventory sources complete the base inventory, + this option allows for getting vars related to hosts/groups from those plugins. + - The host_group_vars (enabled by default) 'vars plugin' is the one responsible for reading host_vars/ and group_vars/ directories. + - This will execute all vars plugins, even those that are not supposed to execute at the 'inventory' stage. + See vars plugins docs for details on 'stage'. + required: false + default: false + type: boolean + version_added: '2.11' + extends_documentation_fragment: + - constructed +''' + +EXAMPLES = r''' + # inventory.config file in YAML format + plugin: ansible.builtin.constructed + strict: False + compose: + var_sum: var1 + var2 + + # this variable will only be set if I have a persistent fact cache enabled (and have non expired facts) + # `strict: False` will skip this instead of producing an error if it is missing facts. + server_type: "ansible_hostname | regex_replace ('(.{6})(.{2}).*', '\\2')" + groups: + # simple name matching + webservers: inventory_hostname.startswith('web') + + # using ec2 'tags' (assumes aws inventory) + development: "'devel' in (ec2_tags|list)" + + # using other host properties populated in inventory + private_only: not (public_dns_name is defined or ip_address is defined) + + # complex group membership + multi_group: (group_names | intersect(['alpha', 'beta', 'omega'])) | length >= 2 + + keyed_groups: + # this creates a group per distro (distro_CentOS, distro_Debian) and assigns the hosts that have matching values to it, + # using the default separator "_" + - prefix: distro + key: ansible_distribution + + # the following examples assume the first inventory is from the `aws_ec2` plugin + # this creates a group per ec2 architecture and assign hosts to the matching ones (arch_x86_64, arch_sparc, etc) + - prefix: arch + key: architecture + + # this creates a group per ec2 region like "us_west_1" + - prefix: "" + separator: "" + key: placement.region + + # this creates a common parent group for all ec2 availability zones + - key: placement.availability_zone + parent_group: all_ec2_zones +''' + +import os + +from ansible import constants as C +from ansible.errors import AnsibleParserError, AnsibleOptionsError +from ansible.inventory.helpers import get_group_vars +from ansible.plugins.inventory import BaseInventoryPlugin, Constructable +from ansible.module_utils._text import to_native +from ansible.utils.vars import combine_vars +from ansible.vars.fact_cache import FactCache +from ansible.vars.plugins import get_vars_from_inventory_sources + + +class InventoryModule(BaseInventoryPlugin, Constructable): + """ constructs groups and vars using Jinja2 template expressions """ + + NAME = 'constructed' + + def __init__(self): + + super(InventoryModule, self).__init__() + + self._cache = FactCache() + + def verify_file(self, path): + + valid = False + if super(InventoryModule, self).verify_file(path): + file_name, ext = os.path.splitext(path) + + if not ext or ext in ['.config'] + C.YAML_FILENAME_EXTENSIONS: + valid = True + + return valid + + def get_all_host_vars(self, host, loader, sources): + ''' requires host object ''' + return combine_vars(self.host_groupvars(host, loader, sources), self.host_vars(host, loader, sources)) + + def host_groupvars(self, host, loader, sources): + ''' requires host object ''' + gvars = get_group_vars(host.get_groups()) + + if self.get_option('use_vars_plugins'): + gvars = combine_vars(gvars, get_vars_from_inventory_sources(loader, sources, host.get_groups(), 'all')) + + return gvars + + def host_vars(self, host, loader, sources): + ''' requires host object ''' + hvars = host.get_vars() + + if self.get_option('use_vars_plugins'): + hvars = combine_vars(hvars, get_vars_from_inventory_sources(loader, sources, [host], 'all')) + + return hvars + + def parse(self, inventory, loader, path, cache=False): + ''' parses the inventory file ''' + + super(InventoryModule, self).parse(inventory, loader, path, cache=cache) + + self._read_config_data(path) + + sources = [] + try: + sources = inventory.processed_sources + except AttributeError: + if self.get_option('use_vars_plugins'): + raise AnsibleOptionsError("The option use_vars_plugins requires ansible >= 2.11.") + + strict = self.get_option('strict') + fact_cache = FactCache() + try: + # Go over hosts (less var copies) + for host in inventory.hosts: + + # get available variables to templar + hostvars = self.get_all_host_vars(inventory.hosts[host], loader, sources) + if host in fact_cache: # adds facts if cache is active + hostvars = combine_vars(hostvars, fact_cache[host]) + + # create composite vars + self._set_composite_vars(self.get_option('compose'), hostvars, host, strict=strict) + + # refetch host vars in case new ones have been created above + hostvars = self.get_all_host_vars(inventory.hosts[host], loader, sources) + if host in self._cache: # adds facts if cache is active + hostvars = combine_vars(hostvars, self._cache[host]) + + # constructed groups based on conditionals + self._add_host_to_composed_groups(self.get_option('groups'), hostvars, host, strict=strict, fetch_hostvars=False) + + # constructed groups based variable values + self._add_host_to_keyed_groups(self.get_option('keyed_groups'), hostvars, host, strict=strict, fetch_hostvars=False) + + except Exception as e: + raise AnsibleParserError("failed to parse %s: %s " % (to_native(path), to_native(e)), orig_exc=e) diff --git a/lib/ansible/plugins/inventory/generator.py b/lib/ansible/plugins/inventory/generator.py new file mode 100644 index 0000000..1955f36 --- /dev/null +++ b/lib/ansible/plugins/inventory/generator.py @@ -0,0 +1,135 @@ +# Copyright (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' + name: generator + version_added: "2.6" + short_description: Uses Jinja2 to construct hosts and groups from patterns + description: + - Uses a YAML configuration file with a valid YAML or C(.config) extension to define var expressions and group conditionals + - Create a template pattern that describes each host, and then use independent configuration layers + - Every element of every layer is combined to create a host for every layer combination + - Parent groups can be defined with reference to hosts and other groups using the same template variables + options: + plugin: + description: token that ensures this is a source file for the 'generator' plugin. + required: True + choices: ['ansible.builtin.generator', 'generator'] + hosts: + description: + - The C(name) key is a template used to generate + hostnames based on the C(layers) option. Each variable in the name is expanded to create a + cartesian product of all possible layer combinations. + - The C(parents) are a list of parent groups that the host belongs to. Each C(parent) item + contains a C(name) key, again expanded from the template, and an optional C(parents) key + that lists its parents. + - Parents can also contain C(vars), which is a dictionary of vars that + is then always set for that variable. This can provide easy access to the group name. E.g + set an C(application) variable that is set to the value of the C(application) layer name. + layers: + description: + - A dictionary of layers, with the key being the layer name, used as a variable name in the C(host) + C(name) and C(parents) keys. Each layer value is a list of possible values for that layer. +''' + +EXAMPLES = ''' + # inventory.config file in YAML format + # remember to enable this inventory plugin in the ansible.cfg before using + # View the output using `ansible-inventory -i inventory.config --list` + plugin: ansible.builtin.generator + hosts: + name: "{{ operation }}_{{ application }}_{{ environment }}_runner" + parents: + - name: "{{ operation }}_{{ application }}_{{ environment }}" + parents: + - name: "{{ operation }}_{{ application }}" + parents: + - name: "{{ operation }}" + - name: "{{ application }}" + - name: "{{ application }}_{{ environment }}" + parents: + - name: "{{ application }}" + vars: + application: "{{ application }}" + - name: "{{ environment }}" + vars: + environment: "{{ environment }}" + - name: runner + layers: + operation: + - build + - launch + environment: + - dev + - test + - prod + application: + - web + - api +''' + +import os + +from itertools import product + +from ansible import constants as C +from ansible.errors import AnsibleParserError +from ansible.plugins.inventory import BaseInventoryPlugin + + +class InventoryModule(BaseInventoryPlugin): + """ constructs groups and vars using Jinja2 template expressions """ + + NAME = 'generator' + + def __init__(self): + + super(InventoryModule, self).__init__() + + def verify_file(self, path): + + valid = False + if super(InventoryModule, self).verify_file(path): + file_name, ext = os.path.splitext(path) + + if not ext or ext in ['.config'] + C.YAML_FILENAME_EXTENSIONS: + valid = True + + return valid + + def template(self, pattern, variables): + self.templar.available_variables = variables + return self.templar.do_template(pattern) + + def add_parents(self, inventory, child, parents, template_vars): + for parent in parents: + try: + groupname = self.template(parent['name'], template_vars) + except (AttributeError, ValueError): + raise AnsibleParserError("Element %s has a parent with no name element" % child['name']) + if groupname not in inventory.groups: + inventory.add_group(groupname) + group = inventory.groups[groupname] + for (k, v) in parent.get('vars', {}).items(): + group.set_variable(k, self.template(v, template_vars)) + inventory.add_child(groupname, child) + self.add_parents(inventory, groupname, parent.get('parents', []), template_vars) + + def parse(self, inventory, loader, path, cache=False): + ''' parses the inventory file ''' + + super(InventoryModule, self).parse(inventory, loader, path, cache=cache) + + config = self._read_config_data(path) + + template_inputs = product(*config['layers'].values()) + for item in template_inputs: + template_vars = dict() + for i, key in enumerate(config['layers'].keys()): + template_vars[key] = item[i] + host = self.template(config['hosts']['name'], template_vars) + inventory.add_host(host) + self.add_parents(inventory, host, config['hosts'].get('parents', []), template_vars) diff --git a/lib/ansible/plugins/inventory/host_list.py b/lib/ansible/plugins/inventory/host_list.py new file mode 100644 index 0000000..eee8516 --- /dev/null +++ b/lib/ansible/plugins/inventory/host_list.py @@ -0,0 +1,66 @@ +# Copyright (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = r''' + name: host_list + version_added: "2.4" + short_description: Parses a 'host list' string + description: + - Parses a host list string as a comma separated values of hosts + - This plugin only applies to inventory strings that are not paths and contain a comma. +''' + +EXAMPLES = r''' + # define 2 hosts in command line + # ansible -i '10.10.2.6, 10.10.2.4' -m ping all + + # DNS resolvable names + # ansible -i 'host1.example.com, host2' -m user -a 'name=me state=absent' all + + # just use localhost + # ansible-playbook -i 'localhost,' play.yml -c local +''' + +import os + +from ansible.errors import AnsibleError, AnsibleParserError +from ansible.module_utils._text import to_bytes, to_native, to_text +from ansible.parsing.utils.addresses import parse_address +from ansible.plugins.inventory import BaseInventoryPlugin + + +class InventoryModule(BaseInventoryPlugin): + + NAME = 'host_list' + + def verify_file(self, host_list): + + valid = False + b_path = to_bytes(host_list, errors='surrogate_or_strict') + if not os.path.exists(b_path) and ',' in host_list: + valid = True + return valid + + def parse(self, inventory, loader, host_list, cache=True): + ''' parses the inventory file ''' + + super(InventoryModule, self).parse(inventory, loader, host_list) + + try: + for h in host_list.split(','): + h = h.strip() + if h: + try: + (host, port) = parse_address(h, allow_ranges=False) + except AnsibleError as e: + self.display.vvv("Unable to parse address from hostname, leaving unchanged: %s" % to_text(e)) + host = h + port = None + + if host not in self.inventory.hosts: + self.inventory.add_host(host, group='ungrouped', port=port) + except Exception as e: + raise AnsibleParserError("Invalid data from string, could not parse: %s" % to_native(e)) diff --git a/lib/ansible/plugins/inventory/ini.py b/lib/ansible/plugins/inventory/ini.py new file mode 100644 index 0000000..b9955cd --- /dev/null +++ b/lib/ansible/plugins/inventory/ini.py @@ -0,0 +1,393 @@ +# Copyright (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' + name: ini + version_added: "2.4" + short_description: Uses an Ansible INI file as inventory source. + description: + - INI file based inventory, sections are groups or group related with special C(:modifiers). + - Entries in sections C([group_1]) are hosts, members of the group. + - Hosts can have variables defined inline as key/value pairs separated by C(=). + - The C(children) modifier indicates that the section contains groups. + - The C(vars) modifier indicates that the section contains variables assigned to members of the group. + - Anything found outside a section is considered an 'ungrouped' host. + - Values passed in the INI format using the C(key=value) syntax are interpreted differently depending on where they are declared within your inventory. + - When declared inline with the host, INI values are processed by Python's ast.literal_eval function + (U(https://docs.python.org/3/library/ast.html#ast.literal_eval)) and interpreted as Python literal structures + (strings, numbers, tuples, lists, dicts, booleans, None). If you want a number to be treated as a string, you must quote it. + Host lines accept multiple C(key=value) parameters per line. + Therefore they need a way to indicate that a space is part of a value rather than a separator. + - When declared in a C(:vars) section, INI values are interpreted as strings. For example C(var=FALSE) would create a string equal to C(FALSE). + Unlike host lines, C(:vars) sections accept only a single entry per line, so everything after the C(=) must be the value for the entry. + - Do not rely on types set during definition, always make sure you specify type with a filter when needed when consuming the variable. + - See the Examples for proper quoting to prevent changes to variable type. + notes: + - Enabled in configuration by default. + - Consider switching to YAML format for inventory sources to avoid confusion on the actual type of a variable. + The YAML inventory plugin processes variable values consistently and correctly. +''' + +EXAMPLES = '''# fmt: ini +# Example 1 +[web] +host1 +host2 ansible_port=222 # defined inline, interpreted as an integer + +[web:vars] +http_port=8080 # all members of 'web' will inherit these +myvar=23 # defined in a :vars section, interpreted as a string + +[web:children] # child groups will automatically add their hosts to parent group +apache +nginx + +[apache] +tomcat1 +tomcat2 myvar=34 # host specific vars override group vars +tomcat3 mysecret="'03#pa33w0rd'" # proper quoting to prevent value changes + +[nginx] +jenkins1 + +[nginx:vars] +has_java = True # vars in child groups override same in parent + +[all:vars] +has_java = False # 'all' is 'top' parent + +# Example 2 +host1 # this is 'ungrouped' + +# both hosts have same IP but diff ports, also 'ungrouped' +host2 ansible_host=127.0.0.1 ansible_port=44 +host3 ansible_host=127.0.0.1 ansible_port=45 + +[g1] +host4 + +[g2] +host4 # same host as above, but member of 2 groups, will inherit vars from both + # inventory hostnames are unique +''' + +import ast +import re + +from ansible.inventory.group import to_safe_group_name +from ansible.plugins.inventory import BaseFileInventoryPlugin + +from ansible.errors import AnsibleError, AnsibleParserError +from ansible.module_utils._text import to_bytes, to_text +from ansible.utils.shlex import shlex_split + + +class InventoryModule(BaseFileInventoryPlugin): + """ + Takes an INI-format inventory file and builds a list of groups and subgroups + with their associated hosts and variable settings. + """ + NAME = 'ini' + _COMMENT_MARKERS = frozenset((u';', u'#')) + b_COMMENT_MARKERS = frozenset((b';', b'#')) + + def __init__(self): + + super(InventoryModule, self).__init__() + + self.patterns = {} + self._filename = None + + def parse(self, inventory, loader, path, cache=True): + + super(InventoryModule, self).parse(inventory, loader, path) + + self._filename = path + + try: + # Read in the hosts, groups, and variables defined in the inventory file. + if self.loader: + (b_data, private) = self.loader._get_file_contents(path) + else: + b_path = to_bytes(path, errors='surrogate_or_strict') + with open(b_path, 'rb') as fh: + b_data = fh.read() + + try: + # Faster to do to_text once on a long string than many + # times on smaller strings + data = to_text(b_data, errors='surrogate_or_strict').splitlines() + except UnicodeError: + # Handle non-utf8 in comment lines: https://github.com/ansible/ansible/issues/17593 + data = [] + for line in b_data.splitlines(): + if line and line[0] in self.b_COMMENT_MARKERS: + # Replace is okay for comment lines + # data.append(to_text(line, errors='surrogate_then_replace')) + # Currently we only need these lines for accurate lineno in errors + data.append(u'') + else: + # Non-comment lines still have to be valid uf-8 + data.append(to_text(line, errors='surrogate_or_strict')) + + self._parse(path, data) + except Exception as e: + raise AnsibleParserError(e) + + def _raise_error(self, message): + raise AnsibleError("%s:%d: " % (self._filename, self.lineno) + message) + + def _parse(self, path, lines): + ''' + Populates self.groups from the given array of lines. Raises an error on + any parse failure. + ''' + + self._compile_patterns() + + # We behave as though the first line of the inventory is '[ungrouped]', + # and begin to look for host definitions. We make a single pass through + # each line of the inventory, building up self.groups and adding hosts, + # subgroups, and setting variables as we go. + + pending_declarations = {} + groupname = 'ungrouped' + state = 'hosts' + self.lineno = 0 + for line in lines: + self.lineno += 1 + + line = line.strip() + # Skip empty lines and comments + if not line or line[0] in self._COMMENT_MARKERS: + continue + + # Is this a [section] header? That tells us what group we're parsing + # definitions for, and what kind of definitions to expect. + + m = self.patterns['section'].match(line) + if m: + (groupname, state) = m.groups() + + groupname = to_safe_group_name(groupname) + + state = state or 'hosts' + if state not in ['hosts', 'children', 'vars']: + title = ":".join(m.groups()) + self._raise_error("Section [%s] has unknown type: %s" % (title, state)) + + # If we haven't seen this group before, we add a new Group. + if groupname not in self.inventory.groups: + # Either [groupname] or [groupname:children] is sufficient to declare a group, + # but [groupname:vars] is allowed only if the # group is declared elsewhere. + # We add the group anyway, but make a note in pending_declarations to check at the end. + # + # It's possible that a group is previously pending due to being defined as a child + # group, in that case we simply pass so that the logic below to process pending + # declarations will take the appropriate action for a pending child group instead of + # incorrectly handling it as a var state pending declaration + if state == 'vars' and groupname not in pending_declarations: + pending_declarations[groupname] = dict(line=self.lineno, state=state, name=groupname) + + self.inventory.add_group(groupname) + + # When we see a declaration that we've been waiting for, we process and delete. + if groupname in pending_declarations and state != 'vars': + if pending_declarations[groupname]['state'] == 'children': + self._add_pending_children(groupname, pending_declarations) + elif pending_declarations[groupname]['state'] == 'vars': + del pending_declarations[groupname] + + continue + elif line.startswith('[') and line.endswith(']'): + self._raise_error("Invalid section entry: '%s'. Please make sure that there are no spaces" % line + " " + + "in the section entry, and that there are no other invalid characters") + + # It's not a section, so the current state tells us what kind of + # definition it must be. The individual parsers will raise an + # error if we feed them something they can't digest. + + # [groupname] contains host definitions that must be added to + # the current group. + if state == 'hosts': + hosts, port, variables = self._parse_host_definition(line) + self._populate_host_vars(hosts, variables, groupname, port) + + # [groupname:vars] contains variable definitions that must be + # applied to the current group. + elif state == 'vars': + (k, v) = self._parse_variable_definition(line) + self.inventory.set_variable(groupname, k, v) + + # [groupname:children] contains subgroup names that must be + # added as children of the current group. The subgroup names + # must themselves be declared as groups, but as before, they + # may only be declared later. + elif state == 'children': + child = self._parse_group_name(line) + if child not in self.inventory.groups: + if child not in pending_declarations: + pending_declarations[child] = dict(line=self.lineno, state=state, name=child, parents=[groupname]) + else: + pending_declarations[child]['parents'].append(groupname) + else: + self.inventory.add_child(groupname, child) + else: + # This can happen only if the state checker accepts a state that isn't handled above. + self._raise_error("Entered unhandled state: %s" % (state)) + + # Any entries in pending_declarations not removed by a group declaration above mean that there was an unresolved reference. + # We report only the first such error here. + for g in pending_declarations: + decl = pending_declarations[g] + if decl['state'] == 'vars': + raise AnsibleError("%s:%d: Section [%s:vars] not valid for undefined group: %s" % (path, decl['line'], decl['name'], decl['name'])) + elif decl['state'] == 'children': + raise AnsibleError("%s:%d: Section [%s:children] includes undefined group: %s" % (path, decl['line'], decl['parents'].pop(), decl['name'])) + + def _add_pending_children(self, group, pending): + for parent in pending[group]['parents']: + self.inventory.add_child(parent, group) + if parent in pending and pending[parent]['state'] == 'children': + self._add_pending_children(parent, pending) + del pending[group] + + def _parse_group_name(self, line): + ''' + Takes a single line and tries to parse it as a group name. Returns the + group name if successful, or raises an error. + ''' + + m = self.patterns['groupname'].match(line) + if m: + return m.group(1) + + self._raise_error("Expected group name, got: %s" % (line)) + + def _parse_variable_definition(self, line): + ''' + Takes a string and tries to parse it as a variable definition. Returns + the key and value if successful, or raises an error. + ''' + + # TODO: We parse variable assignments as a key (anything to the left of + # an '='"), an '=', and a value (anything left) and leave the value to + # _parse_value to sort out. We should be more systematic here about + # defining what is acceptable, how quotes work, and so on. + + if '=' in line: + (k, v) = [e.strip() for e in line.split("=", 1)] + return (k, self._parse_value(v)) + + self._raise_error("Expected key=value, got: %s" % (line)) + + def _parse_host_definition(self, line): + ''' + Takes a single line and tries to parse it as a host definition. Returns + a list of Hosts if successful, or raises an error. + ''' + + # A host definition comprises (1) a non-whitespace hostname or range, + # optionally followed by (2) a series of key="some value" assignments. + # We ignore any trailing whitespace and/or comments. For example, here + # are a series of host definitions in a group: + # + # [groupname] + # alpha + # beta:2345 user=admin # we'll tell shlex + # gamma sudo=True user=root # to ignore comments + + try: + tokens = shlex_split(line, comments=True) + except ValueError as e: + self._raise_error("Error parsing host definition '%s': %s" % (line, e)) + + (hostnames, port) = self._expand_hostpattern(tokens[0]) + + # Try to process anything remaining as a series of key=value pairs. + variables = {} + for t in tokens[1:]: + if '=' not in t: + self._raise_error("Expected key=value host variable assignment, got: %s" % (t)) + (k, v) = t.split('=', 1) + variables[k] = self._parse_value(v) + + return hostnames, port, variables + + def _expand_hostpattern(self, hostpattern): + ''' + do some extra checks over normal processing + ''' + # specification? + + hostnames, port = super(InventoryModule, self)._expand_hostpattern(hostpattern) + + if hostpattern.strip().endswith(':') and port is None: + raise AnsibleParserError("Invalid host pattern '%s' supplied, ending in ':' is not allowed, this character is reserved to provide a port." % + hostpattern) + for pattern in hostnames: + # some YAML parsing prevention checks + if pattern.strip() == '---': + raise AnsibleParserError("Invalid host pattern '%s' supplied, '---' is normally a sign this is a YAML file." % hostpattern) + + return (hostnames, port) + + @staticmethod + def _parse_value(v): + ''' + Attempt to transform the string value from an ini file into a basic python object + (int, dict, list, unicode string, etc). + ''' + try: + v = ast.literal_eval(v) + # Using explicit exceptions. + # Likely a string that literal_eval does not like. We wil then just set it. + except ValueError: + # For some reason this was thought to be malformed. + pass + except SyntaxError: + # Is this a hash with an equals at the end? + pass + return to_text(v, nonstring='passthru', errors='surrogate_or_strict') + + def _compile_patterns(self): + ''' + Compiles the regular expressions required to parse the inventory and + stores them in self.patterns. + ''' + + # Section names are square-bracketed expressions at the beginning of a + # line, comprising (1) a group name optionally followed by (2) a tag + # that specifies the contents of the section. We ignore any trailing + # whitespace and/or comments. For example: + # + # [groupname] + # [somegroup:vars] + # [naughty:children] # only get coal in their stockings + + self.patterns['section'] = re.compile( + to_text(r'''^\[ + ([^:\]\s]+) # group name (see groupname below) + (?::(\w+))? # optional : and tag name + \] + \s* # ignore trailing whitespace + (?:\#.*)? # and/or a comment till the + $ # end of the line + ''', errors='surrogate_or_strict'), re.X + ) + + # FIXME: What are the real restrictions on group names, or rather, what + # should they be? At the moment, they must be non-empty sequences of non + # whitespace characters excluding ':' and ']', but we should define more + # precise rules in order to support better diagnostics. + + self.patterns['groupname'] = re.compile( + to_text(r'''^ + ([^:\]\s]+) + \s* # ignore trailing whitespace + (?:\#.*)? # and/or a comment till the + $ # end of the line + ''', errors='surrogate_or_strict'), re.X + ) diff --git a/lib/ansible/plugins/inventory/script.py b/lib/ansible/plugins/inventory/script.py new file mode 100644 index 0000000..4ffd8e1 --- /dev/null +++ b/lib/ansible/plugins/inventory/script.py @@ -0,0 +1,196 @@ +# Copyright (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# Copyright (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' + name: script + version_added: "2.4" + short_description: Executes an inventory script that returns JSON + options: + always_show_stderr: + description: Toggle display of stderr even when script was successful + version_added: "2.5.1" + default: True + type: boolean + ini: + - section: inventory_plugin_script + key: always_show_stderr + env: + - name: ANSIBLE_INVENTORY_PLUGIN_SCRIPT_STDERR + description: + - The source provided must be an executable that returns Ansible inventory JSON + - The source must accept C(--list) and C(--host <hostname>) as arguments. + C(--host) will only be used if no C(_meta) key is present. + This is a performance optimization as the script would be called per host otherwise. + notes: + - Enabled in configuration by default. + - The plugin does not cache results because external inventory scripts are responsible for their own caching. +''' + +import os +import subprocess + +from collections.abc import Mapping + +from ansible.errors import AnsibleError, AnsibleParserError +from ansible.module_utils.basic import json_dict_bytes_to_unicode +from ansible.module_utils._text import to_native, to_text +from ansible.plugins.inventory import BaseInventoryPlugin +from ansible.utils.display import Display + +display = Display() + + +class InventoryModule(BaseInventoryPlugin): + ''' Host inventory parser for ansible using external inventory scripts. ''' + + NAME = 'script' + + def __init__(self): + + super(InventoryModule, self).__init__() + + self._hosts = set() + + def verify_file(self, path): + ''' Verify if file is usable by this plugin, base does minimal accessibility check ''' + + valid = super(InventoryModule, self).verify_file(path) + + if valid: + # not only accessible, file must be executable and/or have shebang + shebang_present = False + try: + with open(path, 'rb') as inv_file: + initial_chars = inv_file.read(2) + if initial_chars.startswith(b'#!'): + shebang_present = True + except Exception: + pass + + if not os.access(path, os.X_OK) and not shebang_present: + valid = False + + return valid + + def parse(self, inventory, loader, path, cache=None): + + super(InventoryModule, self).parse(inventory, loader, path) + self.set_options() + + # Support inventory scripts that are not prefixed with some + # path information but happen to be in the current working + # directory when '.' is not in PATH. + cmd = [path, "--list"] + + try: + try: + sp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except OSError as e: + raise AnsibleParserError("problem running %s (%s)" % (' '.join(cmd), to_native(e))) + (stdout, stderr) = sp.communicate() + + path = to_native(path) + err = to_native(stderr or "") + + if err and not err.endswith('\n'): + err += '\n' + + if sp.returncode != 0: + raise AnsibleError("Inventory script (%s) had an execution error: %s " % (path, err)) + + # make sure script output is unicode so that json loader will output unicode strings itself + try: + data = to_text(stdout, errors="strict") + except Exception as e: + raise AnsibleError("Inventory {0} contained characters that cannot be interpreted as UTF-8: {1}".format(path, to_native(e))) + + try: + processed = self.loader.load(data, json_only=True) + except Exception as e: + raise AnsibleError("failed to parse executable inventory script results from {0}: {1}\n{2}".format(path, to_native(e), err)) + + # if no other errors happened and you want to force displaying stderr, do so now + if stderr and self.get_option('always_show_stderr'): + self.display.error(msg=to_text(err)) + + if not isinstance(processed, Mapping): + raise AnsibleError("failed to parse executable inventory script results from {0}: needs to be a json dict\n{1}".format(path, err)) + + group = None + data_from_meta = None + + # A "_meta" subelement may contain a variable "hostvars" which contains a hash for each host + # if this "hostvars" exists at all then do not call --host for each # host. + # This is for efficiency and scripts should still return data + # if called with --host for backwards compat with 1.2 and earlier. + for (group, gdata) in processed.items(): + if group == '_meta': + if 'hostvars' in gdata: + data_from_meta = gdata['hostvars'] + else: + self._parse_group(group, gdata) + + for host in self._hosts: + got = {} + if data_from_meta is None: + got = self.get_host_variables(path, host) + else: + try: + got = data_from_meta.get(host, {}) + except AttributeError as e: + raise AnsibleError("Improperly formatted host information for %s: %s" % (host, to_native(e)), orig_exc=e) + + self._populate_host_vars([host], got) + + except Exception as e: + raise AnsibleParserError(to_native(e)) + + def _parse_group(self, group, data): + + group = self.inventory.add_group(group) + + if not isinstance(data, dict): + data = {'hosts': data} + # is not those subkeys, then simplified syntax, host with vars + elif not any(k in data for k in ('hosts', 'vars', 'children')): + data = {'hosts': [group], 'vars': data} + + if 'hosts' in data: + if not isinstance(data['hosts'], list): + raise AnsibleError("You defined a group '%s' with bad data for the host list:\n %s" % (group, data)) + + for hostname in data['hosts']: + self._hosts.add(hostname) + self.inventory.add_host(hostname, group) + + if 'vars' in data: + if not isinstance(data['vars'], dict): + raise AnsibleError("You defined a group '%s' with bad data for variables:\n %s" % (group, data)) + + for k, v in data['vars'].items(): + self.inventory.set_variable(group, k, v) + + if group != '_meta' and isinstance(data, dict) and 'children' in data: + for child_name in data['children']: + child_name = self.inventory.add_group(child_name) + self.inventory.add_child(group, child_name) + + def get_host_variables(self, path, host): + """ Runs <script> --host <hostname>, to determine additional host variables """ + + cmd = [path, "--host", host] + try: + sp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except OSError as e: + raise AnsibleError("problem running %s (%s)" % (' '.join(cmd), e)) + (out, err) = sp.communicate() + if out.strip() == '': + return {} + try: + return json_dict_bytes_to_unicode(self.loader.load(out, file_name=path)) + except ValueError: + raise AnsibleError("could not parse post variable response: %s, %s" % (cmd, out)) diff --git a/lib/ansible/plugins/inventory/toml.py b/lib/ansible/plugins/inventory/toml.py new file mode 100644 index 0000000..f68b34a --- /dev/null +++ b/lib/ansible/plugins/inventory/toml.py @@ -0,0 +1,298 @@ +# Copyright (c) 2018 Matt Martz <matt@sivel.net> +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = r''' + name: toml + version_added: "2.8" + short_description: Uses a specific TOML file as an inventory source. + description: + - TOML based inventory format + - File MUST have a valid '.toml' file extension + notes: + - > + Requires one of the following python libraries: 'toml', 'tomli', or 'tomllib' +''' + +EXAMPLES = r'''# fmt: toml +# Example 1 +[all.vars] +has_java = false + +[web] +children = [ + "apache", + "nginx" +] +vars = { http_port = 8080, myvar = 23 } + +[web.hosts] +host1 = {} +host2 = { ansible_port = 222 } + +[apache.hosts] +tomcat1 = {} +tomcat2 = { myvar = 34 } +tomcat3 = { mysecret = "03#pa33w0rd" } + +[nginx.hosts] +jenkins1 = {} + +[nginx.vars] +has_java = true + +# Example 2 +[all.vars] +has_java = false + +[web] +children = [ + "apache", + "nginx" +] + +[web.vars] +http_port = 8080 +myvar = 23 + +[web.hosts.host1] +[web.hosts.host2] +ansible_port = 222 + +[apache.hosts.tomcat1] + +[apache.hosts.tomcat2] +myvar = 34 + +[apache.hosts.tomcat3] +mysecret = "03#pa33w0rd" + +[nginx.hosts.jenkins1] + +[nginx.vars] +has_java = true + +# Example 3 +[ungrouped.hosts] +host1 = {} +host2 = { ansible_host = "127.0.0.1", ansible_port = 44 } +host3 = { ansible_host = "127.0.0.1", ansible_port = 45 } + +[g1.hosts] +host4 = {} + +[g2.hosts] +host4 = {} +''' + +import os +import typing as t + +from collections.abc import MutableMapping, MutableSequence +from functools import partial + +from ansible.errors import AnsibleFileNotFound, AnsibleParserError, AnsibleRuntimeError +from ansible.module_utils._text import to_bytes, to_native, to_text +from ansible.module_utils.six import string_types, text_type +from ansible.parsing.yaml.objects import AnsibleSequence, AnsibleUnicode +from ansible.plugins.inventory import BaseFileInventoryPlugin +from ansible.utils.display import Display +from ansible.utils.unsafe_proxy import AnsibleUnsafeBytes, AnsibleUnsafeText + +HAS_TOML = False +try: + import toml + HAS_TOML = True +except ImportError: + pass + +HAS_TOMLIW = False +try: + import tomli_w # type: ignore[import] + HAS_TOMLIW = True +except ImportError: + pass + +HAS_TOMLLIB = False +try: + import tomllib # type: ignore[import] + HAS_TOMLLIB = True +except ImportError: + try: + import tomli as tomllib # type: ignore[no-redef] + HAS_TOMLLIB = True + except ImportError: + pass + +display = Display() + + +# dumps +if HAS_TOML and hasattr(toml, 'TomlEncoder'): + # toml>=0.10.0 + class AnsibleTomlEncoder(toml.TomlEncoder): + def __init__(self, *args, **kwargs): + super(AnsibleTomlEncoder, self).__init__(*args, **kwargs) + # Map our custom YAML object types to dump_funcs from ``toml`` + self.dump_funcs.update({ + AnsibleSequence: self.dump_funcs.get(list), + AnsibleUnicode: self.dump_funcs.get(str), + AnsibleUnsafeBytes: self.dump_funcs.get(str), + AnsibleUnsafeText: self.dump_funcs.get(str), + }) + toml_dumps = partial(toml.dumps, encoder=AnsibleTomlEncoder()) # type: t.Callable[[t.Any], str] +else: + # toml<0.10.0 + # tomli-w + def toml_dumps(data): # type: (t.Any) -> str + if HAS_TOML: + return toml.dumps(convert_yaml_objects_to_native(data)) + elif HAS_TOMLIW: + return tomli_w.dumps(convert_yaml_objects_to_native(data)) + raise AnsibleRuntimeError( + 'The python "toml" or "tomli-w" library is required when using the TOML output format' + ) + +# loads +if HAS_TOML: + # prefer toml if installed, since it supports both encoding and decoding + toml_loads = toml.loads # type: ignore[assignment] + TOMLDecodeError = toml.TomlDecodeError # type: t.Any +elif HAS_TOMLLIB: + toml_loads = tomllib.loads # type: ignore[assignment] + TOMLDecodeError = tomllib.TOMLDecodeError # type: t.Any # type: ignore[no-redef] + + +def convert_yaml_objects_to_native(obj): + """Older versions of the ``toml`` python library, and tomllib, don't have + a pluggable way to tell the encoder about custom types, so we need to + ensure objects that we pass are native types. + + Used with: + - ``toml<0.10.0`` where ``toml.TomlEncoder`` is missing + - ``tomli`` or ``tomllib`` + + This function recurses an object and ensures we cast any of the types from + ``ansible.parsing.yaml.objects`` into their native types, effectively cleansing + the data before we hand it over to the toml library. + + This function doesn't directly check for the types from ``ansible.parsing.yaml.objects`` + but instead checks for the types those objects inherit from, to offer more flexibility. + """ + if isinstance(obj, dict): + return dict((k, convert_yaml_objects_to_native(v)) for k, v in obj.items()) + elif isinstance(obj, list): + return [convert_yaml_objects_to_native(v) for v in obj] + elif isinstance(obj, text_type): + return text_type(obj) + else: + return obj + + +class InventoryModule(BaseFileInventoryPlugin): + NAME = 'toml' + + def _parse_group(self, group, group_data): + if group_data is not None and not isinstance(group_data, MutableMapping): + self.display.warning("Skipping '%s' as this is not a valid group definition" % group) + return + + group = self.inventory.add_group(group) + if group_data is None: + return + + for key, data in group_data.items(): + if key == 'vars': + if not isinstance(data, MutableMapping): + raise AnsibleParserError( + 'Invalid "vars" entry for "%s" group, requires a dict, found "%s" instead.' % + (group, type(data)) + ) + for var, value in data.items(): + self.inventory.set_variable(group, var, value) + + elif key == 'children': + if not isinstance(data, MutableSequence): + raise AnsibleParserError( + 'Invalid "children" entry for "%s" group, requires a list, found "%s" instead.' % + (group, type(data)) + ) + for subgroup in data: + self._parse_group(subgroup, {}) + self.inventory.add_child(group, subgroup) + + elif key == 'hosts': + if not isinstance(data, MutableMapping): + raise AnsibleParserError( + 'Invalid "hosts" entry for "%s" group, requires a dict, found "%s" instead.' % + (group, type(data)) + ) + for host_pattern, value in data.items(): + hosts, port = self._expand_hostpattern(host_pattern) + self._populate_host_vars(hosts, value, group, port) + else: + self.display.warning( + 'Skipping unexpected key "%s" in group "%s", only "vars", "children" and "hosts" are valid' % + (key, group) + ) + + def _load_file(self, file_name): + if not file_name or not isinstance(file_name, string_types): + raise AnsibleParserError("Invalid filename: '%s'" % to_native(file_name)) + + b_file_name = to_bytes(self.loader.path_dwim(file_name)) + if not self.loader.path_exists(b_file_name): + raise AnsibleFileNotFound("Unable to retrieve file contents", file_name=file_name) + + try: + (b_data, private) = self.loader._get_file_contents(file_name) + return toml_loads(to_text(b_data, errors='surrogate_or_strict')) + except TOMLDecodeError as e: + raise AnsibleParserError( + 'TOML file (%s) is invalid: %s' % (file_name, to_native(e)), + orig_exc=e + ) + except (IOError, OSError) as e: + raise AnsibleParserError( + "An error occurred while trying to read the file '%s': %s" % (file_name, to_native(e)), + orig_exc=e + ) + except Exception as e: + raise AnsibleParserError( + "An unexpected error occurred while parsing the file '%s': %s" % (file_name, to_native(e)), + orig_exc=e + ) + + def parse(self, inventory, loader, path, cache=True): + ''' parses the inventory file ''' + if not HAS_TOMLLIB and not HAS_TOML: + # tomllib works here too, but we don't call it out in the error, + # since you either have it or not as part of cpython stdlib >= 3.11 + raise AnsibleParserError( + 'The TOML inventory plugin requires the python "toml", or "tomli" library' + ) + + super(InventoryModule, self).parse(inventory, loader, path) + self.set_options() + + try: + data = self._load_file(path) + except Exception as e: + raise AnsibleParserError(e) + + if not data: + raise AnsibleParserError('Parsed empty TOML file') + elif data.get('plugin'): + raise AnsibleParserError('Plugin configuration TOML file, not TOML inventory') + + for group_name in data: + self._parse_group(group_name, data[group_name]) + + def verify_file(self, path): + if super(InventoryModule, self).verify_file(path): + file_name, ext = os.path.splitext(path) + if ext == '.toml': + return True + return False diff --git a/lib/ansible/plugins/inventory/yaml.py b/lib/ansible/plugins/inventory/yaml.py new file mode 100644 index 0000000..9d5812f --- /dev/null +++ b/lib/ansible/plugins/inventory/yaml.py @@ -0,0 +1,183 @@ +# Copyright (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' + name: yaml + version_added: "2.4" + short_description: Uses a specific YAML file as an inventory source. + description: + - "YAML-based inventory, should start with the C(all) group and contain hosts/vars/children entries." + - Host entries can have sub-entries defined, which will be treated as variables. + - Vars entries are normal group vars. + - "Children are 'child groups', which can also have their own vars/hosts/children and so on." + - File MUST have a valid extension, defined in configuration. + notes: + - If you want to set vars for the C(all) group inside the inventory file, the C(all) group must be the first entry in the file. + - Enabled in configuration by default. + options: + yaml_extensions: + description: list of 'valid' extensions for files containing YAML + type: list + elements: string + default: ['.yaml', '.yml', '.json'] + env: + - name: ANSIBLE_YAML_FILENAME_EXT + - name: ANSIBLE_INVENTORY_PLUGIN_EXTS + ini: + - key: yaml_valid_extensions + section: defaults + - section: inventory_plugin_yaml + key: yaml_valid_extensions + +''' +EXAMPLES = ''' +all: # keys must be unique, i.e. only one 'hosts' per group + hosts: + test1: + test2: + host_var: value + vars: + group_all_var: value + children: # key order does not matter, indentation does + other_group: + children: + group_x: + hosts: + test5 # Note that one machine will work without a colon + #group_x: + # hosts: + # test5 # But this won't + # test7 # + group_y: + hosts: + test6: # So always use a colon + vars: + g2_var2: value3 + hosts: + test4: + ansible_host: 127.0.0.1 + last_group: + hosts: + test1 # same host as above, additional group membership + vars: + group_last_var: value +''' + +import os + +from collections.abc import MutableMapping + +from ansible.errors import AnsibleError, AnsibleParserError +from ansible.module_utils.six import string_types +from ansible.module_utils._text import to_native, to_text +from ansible.plugins.inventory import BaseFileInventoryPlugin + +NoneType = type(None) + + +class InventoryModule(BaseFileInventoryPlugin): + + NAME = 'yaml' + + def __init__(self): + + super(InventoryModule, self).__init__() + + def verify_file(self, path): + + valid = False + if super(InventoryModule, self).verify_file(path): + file_name, ext = os.path.splitext(path) + if not ext or ext in self.get_option('yaml_extensions'): + valid = True + return valid + + def parse(self, inventory, loader, path, cache=True): + ''' parses the inventory file ''' + + super(InventoryModule, self).parse(inventory, loader, path) + self.set_options() + + try: + data = self.loader.load_from_file(path, cache=False) + except Exception as e: + raise AnsibleParserError(e) + + if not data: + raise AnsibleParserError('Parsed empty YAML file') + elif not isinstance(data, MutableMapping): + raise AnsibleParserError('YAML inventory has invalid structure, it should be a dictionary, got: %s' % type(data)) + elif data.get('plugin'): + raise AnsibleParserError('Plugin configuration YAML file, not YAML inventory') + + # We expect top level keys to correspond to groups, iterate over them + # to get host, vars and subgroups (which we iterate over recursivelly) + if isinstance(data, MutableMapping): + for group_name in data: + self._parse_group(group_name, data[group_name]) + else: + raise AnsibleParserError("Invalid data from file, expected dictionary and got:\n\n%s" % to_native(data)) + + def _parse_group(self, group, group_data): + + if isinstance(group_data, (MutableMapping, NoneType)): # type: ignore[misc] + + try: + group = self.inventory.add_group(group) + except AnsibleError as e: + raise AnsibleParserError("Unable to add group %s: %s" % (group, to_text(e))) + + if group_data is not None: + # make sure they are dicts + for section in ['vars', 'children', 'hosts']: + if section in group_data: + # convert strings to dicts as these are allowed + if isinstance(group_data[section], string_types): + group_data[section] = {group_data[section]: None} + + if not isinstance(group_data[section], (MutableMapping, NoneType)): # type: ignore[misc] + raise AnsibleParserError('Invalid "%s" entry for "%s" group, requires a dictionary, found "%s" instead.' % + (section, group, type(group_data[section]))) + + for key in group_data: + + if not isinstance(group_data[key], (MutableMapping, NoneType)): # type: ignore[misc] + self.display.warning('Skipping key (%s) in group (%s) as it is not a mapping, it is a %s' % (key, group, type(group_data[key]))) + continue + + if isinstance(group_data[key], NoneType): # type: ignore[misc] + self.display.vvv('Skipping empty key (%s) in group (%s)' % (key, group)) + elif key == 'vars': + for var in group_data[key]: + self.inventory.set_variable(group, var, group_data[key][var]) + elif key == 'children': + for subgroup in group_data[key]: + subgroup = self._parse_group(subgroup, group_data[key][subgroup]) + self.inventory.add_child(group, subgroup) + + elif key == 'hosts': + for host_pattern in group_data[key]: + hosts, port = self._parse_host(host_pattern) + self._populate_host_vars(hosts, group_data[key][host_pattern] or {}, group, port) + else: + self.display.warning('Skipping unexpected key (%s) in group (%s), only "vars", "children" and "hosts" are valid' % (key, group)) + + else: + self.display.warning("Skipping '%s' as this is not a valid group definition" % group) + + return group + + def _parse_host(self, host_pattern): + ''' + Each host key can be a pattern, try to process it and add variables as needed + ''' + try: + (hostnames, port) = self._expand_hostpattern(host_pattern) + except TypeError: + raise AnsibleParserError( + f"Host pattern {host_pattern} must be a string. Enclose integers/floats in quotation marks." + ) + return hostnames, port diff --git a/lib/ansible/plugins/list.py b/lib/ansible/plugins/list.py new file mode 100644 index 0000000..e09b293 --- /dev/null +++ b/lib/ansible/plugins/list.py @@ -0,0 +1,210 @@ +# (c) Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +import os + +from ansible import context +from ansible import constants as C +from ansible.collections.list import list_collections +from ansible.errors import AnsibleError +from ansible.module_utils._text import to_native, to_bytes +from ansible.plugins import loader +from ansible.utils.display import Display +from ansible.utils.collection_loader._collection_finder import _get_collection_path, AnsibleCollectionRef + +display = Display() + +# not real plugins +IGNORE = { + # ptype: names + 'module': ('async_wrapper', ), + 'cache': ('base', ), +} + + +def get_composite_name(collection, name, path, depth): + resolved_collection = collection + if '.' not in name: + resource_name = name + else: + if collection == 'ansible.legacy' and name.startswith('ansible.builtin.'): + resolved_collection = 'ansible.builtin' + resource_name = '.'.join(name.split(f"{resolved_collection}.")[1:]) + + # collectionize name + composite = [resolved_collection] + if depth: + composite.extend(path.split(os.path.sep)[depth * -1:]) + composite.append(to_native(resource_name)) + return '.'.join(composite) + + +def _list_plugins_from_paths(ptype, dirs, collection, depth=0): + + plugins = {} + + for path in dirs: + display.debug("Searching '{0}'s '{1}' for {2} plugins".format(collection, path, ptype)) + b_path = to_bytes(path) + + if os.path.basename(b_path).startswith((b'.', b'__')): + # skip hidden/special dirs + continue + + if os.path.exists(b_path): + if os.path.isdir(b_path): + bkey = ptype.lower() + for plugin_file in os.listdir(b_path): + + if plugin_file.startswith((b'.', b'__')): + # hidden or python internal file/dir + continue + + display.debug("Found possible plugin: '{0}'".format(plugin_file)) + b_plugin, b_ext = os.path.splitext(plugin_file) + plugin = to_native(b_plugin) + full_path = os.path.join(b_path, plugin_file) + + if os.path.isdir(full_path): + # its a dir, recurse + if collection in C.SYNTHETIC_COLLECTIONS: + if not os.path.exists(os.path.join(full_path, b'__init__.py')): + # dont recurse for synthetic unless init.py present + continue + + # actually recurse dirs + plugins.update(_list_plugins_from_paths(ptype, [to_native(full_path)], collection, depth=depth + 1)) + else: + if any([ + plugin in C.IGNORE_FILES, # general files to ignore + to_native(b_ext) in C.REJECT_EXTS, # general extensions to ignore + b_ext in (b'.yml', b'.yaml', b'.json'), # ignore docs files TODO: constant! + plugin in IGNORE.get(bkey, ()), # plugin in reject list + os.path.islink(full_path), # skip aliases, author should document in 'aliaes' field + ]): + continue + + if ptype in ('test', 'filter'): + try: + file_plugins = _list_j2_plugins_from_file(collection, full_path, ptype, plugin) + except KeyError as e: + display.warning('Skipping file %s: %s' % (full_path, to_native(e))) + continue + + for plugin in file_plugins: + plugin_name = get_composite_name(collection, plugin.ansible_name, os.path.dirname(to_native(full_path)), depth) + plugins[plugin_name] = full_path + else: + plugin_name = get_composite_name(collection, plugin, os.path.dirname(to_native(full_path)), depth) + plugins[plugin_name] = full_path + else: + display.debug("Skip listing plugins in '{0}' as it is not a directory".format(path)) + else: + display.debug("Skip listing plugins in '{0}' as it does not exist".format(path)) + + return plugins + + +def _list_j2_plugins_from_file(collection, plugin_path, ptype, plugin_name): + + ploader = getattr(loader, '{0}_loader'.format(ptype)) + file_plugins = ploader.get_contained_plugins(collection, plugin_path, plugin_name) + return file_plugins + + +def list_collection_plugins(ptype, collections, search_paths=None): + + # starts at {plugin_name: filepath, ...}, but changes at the end + plugins = {} + try: + ploader = getattr(loader, '{0}_loader'.format(ptype)) + except AttributeError: + raise AnsibleError('Cannot list plugins, incorrect plugin type supplied: {0}'.format(ptype)) + + # get plugins for each collection + for collection in collections.keys(): + if collection == 'ansible.builtin': + # dirs from ansible install, but not configured paths + dirs = [d.path for d in ploader._get_paths_with_context() if d.internal] + elif collection == 'ansible.legacy': + # configured paths + search paths (should include basedirs/-M) + dirs = [d.path for d in ploader._get_paths_with_context() if not d.internal] + if context.CLIARGS.get('module_path', None): + dirs.extend(context.CLIARGS['module_path']) + else: + # search path in this case is for locating collection itselfA + b_ptype = to_bytes(C.COLLECTION_PTYPE_COMPAT.get(ptype, ptype)) + dirs = [to_native(os.path.join(collections[collection], b'plugins', b_ptype))] + # acr = AnsibleCollectionRef.try_parse_fqcr(collection, ptype) + # if acr: + # dirs = acr.subdirs + # else: + + # raise Exception('bad acr for %s, %s' % (collection, ptype)) + + plugins.update(_list_plugins_from_paths(ptype, dirs, collection)) + + # return plugin and it's class object, None for those not verifiable or failing + if ptype in ('module',): + # no 'invalid' tests for modules + for plugin in plugins.keys(): + plugins[plugin] = (plugins[plugin], None) + else: + # detect invalid plugin candidates AND add loaded object to return data + for plugin in list(plugins.keys()): + pobj = None + try: + pobj = ploader.get(plugin, class_only=True) + except Exception as e: + display.vvv("The '{0}' {1} plugin could not be loaded from '{2}': {3}".format(plugin, ptype, plugins[plugin], to_native(e))) + + # sets final {plugin_name: (filepath, class|NONE if not loaded), ...} + plugins[plugin] = (plugins[plugin], pobj) + + # {plugin_name: (filepath, class), ...} + return plugins + + +def list_plugins(ptype, collection=None, search_paths=None): + + # {plugin_name: (filepath, class), ...} + plugins = {} + collections = {} + if collection is None: + # list all collections, add synthetic ones + collections['ansible.builtin'] = b'' + collections['ansible.legacy'] = b'' + collections.update(list_collections(search_paths=search_paths, dedupe=True)) + elif collection == 'ansible.legacy': + # add builtin, since legacy also resolves to these + collections[collection] = b'' + collections['ansible.builtin'] = b'' + else: + try: + collections[collection] = to_bytes(_get_collection_path(collection)) + except ValueError as e: + raise AnsibleError("Cannot use supplied collection {0}: {1}".format(collection, to_native(e)), orig_exc=e) + + if collections: + plugins.update(list_collection_plugins(ptype, collections)) + + return plugins + + +# wrappers +def list_plugin_names(ptype, collection=None): + return [plugin.ansible_name for plugin in list_plugins(ptype, collection)] + + +def list_plugin_files(ptype, collection=None): + plugins = list_plugins(ptype, collection) + return [plugins[k][0] for k in plugins.keys()] + + +def list_plugin_classes(ptype, collection=None): + plugins = list_plugins(ptype, collection) + return [plugins[k][1] for k in plugins.keys()] diff --git a/lib/ansible/plugins/loader.py b/lib/ansible/plugins/loader.py new file mode 100644 index 0000000..845fdcd --- /dev/null +++ b/lib/ansible/plugins/loader.py @@ -0,0 +1,1622 @@ +# (c) 2012, Daniel Hokka Zakrisson <daniel@hozac.com> +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> and others +# (c) 2017, Toshio Kuratomi <tkuratomi@ansible.com> +# (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import glob +import os +import os.path +import pkgutil +import sys +import warnings + +from collections import defaultdict, namedtuple +from traceback import format_exc + +from ansible import __version__ as ansible_version +from ansible import constants as C +from ansible.errors import AnsibleError, AnsiblePluginCircularRedirect, AnsiblePluginRemovedError, AnsibleCollectionUnsupportedVersionError +from ansible.module_utils._text import to_bytes, to_text, to_native +from ansible.module_utils.compat.importlib import import_module +from ansible.module_utils.six import string_types +from ansible.parsing.utils.yaml import from_yaml +from ansible.parsing.yaml.loader import AnsibleLoader +from ansible.plugins import get_plugin_class, MODULE_CACHE, PATH_CACHE, PLUGIN_PATH_CACHE +from ansible.utils.collection_loader import AnsibleCollectionConfig, AnsibleCollectionRef +from ansible.utils.collection_loader._collection_finder import _AnsibleCollectionFinder, _get_collection_metadata +from ansible.utils.display import Display +from ansible.utils.plugin_docs import add_fragments, find_plugin_docfile + +# TODO: take the packaging dep, or vendor SpecifierSet? + +try: + from packaging.specifiers import SpecifierSet + from packaging.version import Version +except ImportError: + SpecifierSet = None # type: ignore[misc] + Version = None # type: ignore[misc] + +import importlib.util + +display = Display() + +get_with_context_result = namedtuple('get_with_context_result', ['object', 'plugin_load_context']) + + +def get_all_plugin_loaders(): + return [(name, obj) for (name, obj) in globals().items() if isinstance(obj, PluginLoader)] + + +def add_all_plugin_dirs(path): + ''' add any existing plugin dirs in the path provided ''' + b_path = os.path.expanduser(to_bytes(path, errors='surrogate_or_strict')) + if os.path.isdir(b_path): + for name, obj in get_all_plugin_loaders(): + if obj.subdir: + plugin_path = os.path.join(b_path, to_bytes(obj.subdir)) + if os.path.isdir(plugin_path): + obj.add_directory(to_text(plugin_path)) + else: + display.warning("Ignoring invalid path provided to plugin path: '%s' is not a directory" % to_text(path)) + + +def get_shell_plugin(shell_type=None, executable=None): + + if not shell_type: + # default to sh + shell_type = 'sh' + + # mostly for backwards compat + if executable: + if isinstance(executable, string_types): + shell_filename = os.path.basename(executable) + try: + shell = shell_loader.get(shell_filename) + except Exception: + shell = None + + if shell is None: + for shell in shell_loader.all(): + if shell_filename in shell.COMPATIBLE_SHELLS: + shell_type = shell.SHELL_FAMILY + break + else: + raise AnsibleError("Either a shell type or a shell executable must be provided ") + + shell = shell_loader.get(shell_type) + if not shell: + raise AnsibleError("Could not find the shell plugin required (%s)." % shell_type) + + if executable: + setattr(shell, 'executable', executable) + + return shell + + +def add_dirs_to_loader(which_loader, paths): + + loader = getattr(sys.modules[__name__], '%s_loader' % which_loader) + for path in paths: + loader.add_directory(path, with_subdir=True) + + +class PluginPathContext(object): + def __init__(self, path, internal): + self.path = path + self.internal = internal + + +class PluginLoadContext(object): + def __init__(self): + self.original_name = None + self.redirect_list = [] + self.error_list = [] + self.import_error_list = [] + self.load_attempts = [] + self.pending_redirect = None + self.exit_reason = None + self.plugin_resolved_path = None + self.plugin_resolved_name = None + self.plugin_resolved_collection = None # empty string for resolved plugins from user-supplied paths + self.deprecated = False + self.removal_date = None + self.removal_version = None + self.deprecation_warnings = [] + self.resolved = False + self._resolved_fqcn = None + self.action_plugin = None + + @property + def resolved_fqcn(self): + if not self.resolved: + return + + if not self._resolved_fqcn: + final_plugin = self.redirect_list[-1] + if AnsibleCollectionRef.is_valid_fqcr(final_plugin) and final_plugin.startswith('ansible.legacy.'): + final_plugin = final_plugin.split('ansible.legacy.')[-1] + if self.plugin_resolved_collection and not AnsibleCollectionRef.is_valid_fqcr(final_plugin): + final_plugin = self.plugin_resolved_collection + '.' + final_plugin + self._resolved_fqcn = final_plugin + + return self._resolved_fqcn + + def record_deprecation(self, name, deprecation, collection_name): + if not deprecation: + return self + + # The `or ''` instead of using `.get(..., '')` makes sure that even if the user explicitly + # sets `warning_text` to `~` (None) or `false`, we still get an empty string. + warning_text = deprecation.get('warning_text', None) or '' + removal_date = deprecation.get('removal_date', None) + removal_version = deprecation.get('removal_version', None) + # If both removal_date and removal_version are specified, use removal_date + if removal_date is not None: + removal_version = None + warning_text = '{0} has been deprecated.{1}{2}'.format(name, ' ' if warning_text else '', warning_text) + + display.deprecated(warning_text, date=removal_date, version=removal_version, collection_name=collection_name) + + self.deprecated = True + if removal_date: + self.removal_date = removal_date + if removal_version: + self.removal_version = removal_version + self.deprecation_warnings.append(warning_text) + return self + + def resolve(self, resolved_name, resolved_path, resolved_collection, exit_reason, action_plugin): + self.pending_redirect = None + self.plugin_resolved_name = resolved_name + self.plugin_resolved_path = resolved_path + self.plugin_resolved_collection = resolved_collection + self.exit_reason = exit_reason + self.resolved = True + self.action_plugin = action_plugin + return self + + def redirect(self, redirect_name): + self.pending_redirect = redirect_name + self.exit_reason = 'pending redirect resolution from {0} to {1}'.format(self.original_name, redirect_name) + self.resolved = False + return self + + def nope(self, exit_reason): + self.pending_redirect = None + self.exit_reason = exit_reason + self.resolved = False + return self + + +class PluginLoader: + ''' + PluginLoader loads plugins from the configured plugin directories. + + It searches for plugins by iterating through the combined list of play basedirs, configured + paths, and the python path. The first match is used. + ''' + + def __init__(self, class_name, package, config, subdir, aliases=None, required_base_class=None): + aliases = {} if aliases is None else aliases + + self.class_name = class_name + self.base_class = required_base_class + self.package = package + self.subdir = subdir + + # FIXME: remove alias dict in favor of alias by symlink? + self.aliases = aliases + + if config and not isinstance(config, list): + config = [config] + elif not config: + config = [] + + self.config = config + + if class_name not in MODULE_CACHE: + MODULE_CACHE[class_name] = {} + if class_name not in PATH_CACHE: + PATH_CACHE[class_name] = None + if class_name not in PLUGIN_PATH_CACHE: + PLUGIN_PATH_CACHE[class_name] = defaultdict(dict) + + # hold dirs added at runtime outside of config + self._extra_dirs = [] + + # caches + self._module_cache = MODULE_CACHE[class_name] + self._paths = PATH_CACHE[class_name] + self._plugin_path_cache = PLUGIN_PATH_CACHE[class_name] + + self._searched_paths = set() + + @property + def type(self): + return AnsibleCollectionRef.legacy_plugin_dir_to_plugin_type(self.subdir) + + def __repr__(self): + return 'PluginLoader(type={0})'.format(self.type) + + def _clear_caches(self): + + if C.OLD_PLUGIN_CACHE_CLEARING: + self._paths = None + else: + # reset global caches + MODULE_CACHE[self.class_name] = {} + PATH_CACHE[self.class_name] = None + PLUGIN_PATH_CACHE[self.class_name] = defaultdict(dict) + + # reset internal caches + self._module_cache = MODULE_CACHE[self.class_name] + self._paths = PATH_CACHE[self.class_name] + self._plugin_path_cache = PLUGIN_PATH_CACHE[self.class_name] + self._searched_paths = set() + + def __setstate__(self, data): + ''' + Deserializer. + ''' + + class_name = data.get('class_name') + package = data.get('package') + config = data.get('config') + subdir = data.get('subdir') + aliases = data.get('aliases') + base_class = data.get('base_class') + + PATH_CACHE[class_name] = data.get('PATH_CACHE') + PLUGIN_PATH_CACHE[class_name] = data.get('PLUGIN_PATH_CACHE') + + self.__init__(class_name, package, config, subdir, aliases, base_class) + self._extra_dirs = data.get('_extra_dirs', []) + self._searched_paths = data.get('_searched_paths', set()) + + def __getstate__(self): + ''' + Serializer. + ''' + + return dict( + class_name=self.class_name, + base_class=self.base_class, + package=self.package, + config=self.config, + subdir=self.subdir, + aliases=self.aliases, + _extra_dirs=self._extra_dirs, + _searched_paths=self._searched_paths, + PATH_CACHE=PATH_CACHE[self.class_name], + PLUGIN_PATH_CACHE=PLUGIN_PATH_CACHE[self.class_name], + ) + + def format_paths(self, paths): + ''' Returns a string suitable for printing of the search path ''' + + # Uses a list to get the order right + ret = [] + for i in paths: + if i not in ret: + ret.append(i) + return os.pathsep.join(ret) + + def print_paths(self): + return self.format_paths(self._get_paths(subdirs=False)) + + def _all_directories(self, dir): + results = [] + results.append(dir) + for root, subdirs, files in os.walk(dir, followlinks=True): + if '__init__.py' in files: + for x in subdirs: + results.append(os.path.join(root, x)) + return results + + def _get_package_paths(self, subdirs=True): + ''' Gets the path of a Python package ''' + + if not self.package: + return [] + if not hasattr(self, 'package_path'): + m = __import__(self.package) + parts = self.package.split('.')[1:] + for parent_mod in parts: + m = getattr(m, parent_mod) + self.package_path = to_text(os.path.dirname(m.__file__), errors='surrogate_or_strict') + if subdirs: + return self._all_directories(self.package_path) + return [self.package_path] + + def _get_paths_with_context(self, subdirs=True): + ''' Return a list of PluginPathContext objects to search for plugins in ''' + + # FIXME: This is potentially buggy if subdirs is sometimes True and sometimes False. + # In current usage, everything calls this with subdirs=True except for module_utils_loader and ansible-doc + # which always calls it with subdirs=False. So there currently isn't a problem with this caching. + if self._paths is not None: + return self._paths + + ret = [PluginPathContext(p, False) for p in self._extra_dirs] + + # look in any configured plugin paths, allow one level deep for subcategories + if self.config is not None: + for path in self.config: + path = os.path.abspath(os.path.expanduser(path)) + if subdirs: + contents = glob.glob("%s/*" % path) + glob.glob("%s/*/*" % path) + for c in contents: + c = to_text(c, errors='surrogate_or_strict') + if os.path.isdir(c) and c not in ret: + ret.append(PluginPathContext(c, False)) + + path = to_text(path, errors='surrogate_or_strict') + if path not in ret: + ret.append(PluginPathContext(path, False)) + + # look for any plugins installed in the package subtree + # Note package path always gets added last so that every other type of + # path is searched before it. + ret.extend([PluginPathContext(p, True) for p in self._get_package_paths(subdirs=subdirs)]) + + # HACK: because powershell modules are in the same directory + # hierarchy as other modules we have to process them last. This is + # because powershell only works on windows but the other modules work + # anywhere (possibly including windows if the correct language + # interpreter is installed). the non-powershell modules can have any + # file extension and thus powershell modules are picked up in that. + # The non-hack way to fix this is to have powershell modules be + # a different PluginLoader/ModuleLoader. But that requires changing + # other things too (known thing to change would be PATHS_CACHE, + # PLUGIN_PATHS_CACHE, and MODULE_CACHE. Since those three dicts key + # on the class_name and neither regular modules nor powershell modules + # would have class_names, they would not work as written. + # + # The expected sort order is paths in the order in 'ret' with paths ending in '/windows' at the end, + # also in the original order they were found in 'ret'. + # The .sort() method is guaranteed to be stable, so original order is preserved. + ret.sort(key=lambda p: p.path.endswith('/windows')) + + # cache and return the result + self._paths = ret + return ret + + def _get_paths(self, subdirs=True): + ''' Return a list of paths to search for plugins in ''' + + paths_with_context = self._get_paths_with_context(subdirs=subdirs) + return [path_with_context.path for path_with_context in paths_with_context] + + def _load_config_defs(self, name, module, path): + ''' Reads plugin docs to find configuration setting definitions, to push to config manager for later use ''' + + # plugins w/o class name don't support config + if self.class_name: + type_name = get_plugin_class(self.class_name) + + # if type name != 'module_doc_fragment': + if type_name in C.CONFIGURABLE_PLUGINS and not C.config.has_configuration_definition(type_name, name): + dstring = AnsibleLoader(getattr(module, 'DOCUMENTATION', ''), file_name=path).get_single_data() + + # TODO: allow configurable plugins to use sidecar + # if not dstring: + # filename, cn = find_plugin_docfile( name, type_name, self, [os.path.dirname(path)], C.YAML_DOC_EXTENSIONS) + # # TODO: dstring = AnsibleLoader(, file_name=path).get_single_data() + + if dstring: + add_fragments(dstring, path, fragment_loader=fragment_loader, is_module=(type_name == 'module')) + + if 'options' in dstring and isinstance(dstring['options'], dict): + C.config.initialize_plugin_configuration_definitions(type_name, name, dstring['options']) + display.debug('Loaded config def from plugin (%s/%s)' % (type_name, name)) + + def add_directory(self, directory, with_subdir=False): + ''' Adds an additional directory to the search path ''' + + directory = os.path.realpath(directory) + + if directory is not None: + if with_subdir: + directory = os.path.join(directory, self.subdir) + if directory not in self._extra_dirs: + # append the directory and invalidate the path cache + self._extra_dirs.append(directory) + self._clear_caches() + display.debug('Added %s to loader search path' % (directory)) + + def _query_collection_routing_meta(self, acr, plugin_type, extension=None): + collection_pkg = import_module(acr.n_python_collection_package_name) + if not collection_pkg: + return None + + # FIXME: shouldn't need this... + try: + # force any type-specific metadata postprocessing to occur + import_module(acr.n_python_collection_package_name + '.plugins.{0}'.format(plugin_type)) + except ImportError: + pass + + # this will be created by the collection PEP302 loader + collection_meta = getattr(collection_pkg, '_collection_meta', None) + + if not collection_meta: + return None + + # TODO: add subdirs support + # check for extension-specific entry first (eg 'setup.ps1') + # TODO: str/bytes on extension/name munging + if acr.subdirs: + subdir_qualified_resource = '.'.join([acr.subdirs, acr.resource]) + else: + subdir_qualified_resource = acr.resource + entry = collection_meta.get('plugin_routing', {}).get(plugin_type, {}).get(subdir_qualified_resource + extension, None) + if not entry: + # try for extension-agnostic entry + entry = collection_meta.get('plugin_routing', {}).get(plugin_type, {}).get(subdir_qualified_resource, None) + return entry + + def _find_fq_plugin(self, fq_name, extension, plugin_load_context, ignore_deprecated=False): + """Search builtin paths to find a plugin. No external paths are searched, + meaning plugins inside roles inside collections will be ignored. + """ + + plugin_load_context.resolved = False + + plugin_type = AnsibleCollectionRef.legacy_plugin_dir_to_plugin_type(self.subdir) + + acr = AnsibleCollectionRef.from_fqcr(fq_name, plugin_type) + + # check collection metadata to see if any special handling is required for this plugin + routing_metadata = self._query_collection_routing_meta(acr, plugin_type, extension=extension) + + action_plugin = None + # TODO: factor this into a wrapper method + if routing_metadata: + deprecation = routing_metadata.get('deprecation', None) + + # this will no-op if there's no deprecation metadata for this plugin + if not ignore_deprecated: + plugin_load_context.record_deprecation(fq_name, deprecation, acr.collection) + + tombstone = routing_metadata.get('tombstone', None) + + # FIXME: clean up text gen + if tombstone: + removal_date = tombstone.get('removal_date') + removal_version = tombstone.get('removal_version') + warning_text = tombstone.get('warning_text') or '' + warning_text = '{0} has been removed.{1}{2}'.format(fq_name, ' ' if warning_text else '', warning_text) + removed_msg = display.get_deprecation_message(msg=warning_text, version=removal_version, + date=removal_date, removed=True, + collection_name=acr.collection) + plugin_load_context.removal_date = removal_date + plugin_load_context.removal_version = removal_version + plugin_load_context.resolved = True + plugin_load_context.exit_reason = removed_msg + raise AnsiblePluginRemovedError(removed_msg, plugin_load_context=plugin_load_context) + + redirect = routing_metadata.get('redirect', None) + + if redirect: + # Prevent mystery redirects that would be determined by the collections keyword + if not AnsibleCollectionRef.is_valid_fqcr(redirect): + raise AnsibleError( + f"Collection {acr.collection} contains invalid redirect for {fq_name}: {redirect}. " + "Redirects must use fully qualified collection names." + ) + + # FIXME: remove once this is covered in debug or whatever + display.vv("redirecting (type: {0}) {1} to {2}".format(plugin_type, fq_name, redirect)) + + # The name doing the redirection is added at the beginning of _resolve_plugin_step, + # but if the unqualified name is used in conjunction with the collections keyword, only + # the unqualified name is in the redirect list. + if fq_name not in plugin_load_context.redirect_list: + plugin_load_context.redirect_list.append(fq_name) + return plugin_load_context.redirect(redirect) + # TODO: non-FQCN case, do we support `.` prefix for current collection, assume it with no dots, require it for subdirs in current, or ? + + if self.type == 'modules': + action_plugin = routing_metadata.get('action_plugin') + + n_resource = to_native(acr.resource, errors='strict') + # we want this before the extension is added + full_name = '{0}.{1}'.format(acr.n_python_package_name, n_resource) + + if extension: + n_resource += extension + + pkg = sys.modules.get(acr.n_python_package_name) + if not pkg: + # FIXME: there must be cheaper/safer way to do this + try: + pkg = import_module(acr.n_python_package_name) + except ImportError: + return plugin_load_context.nope('Python package {0} not found'.format(acr.n_python_package_name)) + + pkg_path = os.path.dirname(pkg.__file__) + + n_resource_path = os.path.join(pkg_path, n_resource) + + # FIXME: and is file or file link or ... + if os.path.exists(n_resource_path): + return plugin_load_context.resolve( + full_name, to_text(n_resource_path), acr.collection, 'found exact match for {0} in {1}'.format(full_name, acr.collection), action_plugin) + + if extension: + # the request was extension-specific, don't try for an extensionless match + return plugin_load_context.nope('no match for {0} in {1}'.format(to_text(n_resource), acr.collection)) + + # look for any matching extension in the package location (sans filter) + found_files = [f + for f in glob.iglob(os.path.join(pkg_path, n_resource) + '.*') + if os.path.isfile(f) and not f.endswith(C.MODULE_IGNORE_EXTS)] + + if not found_files: + return plugin_load_context.nope('failed fuzzy extension match for {0} in {1}'.format(full_name, acr.collection)) + + found_files = sorted(found_files) # sort to ensure deterministic results, with the shortest match first + + if len(found_files) > 1: + display.debug('Found several possible candidates for the plugin but using first: %s' % ','.join(found_files)) + + return plugin_load_context.resolve( + full_name, to_text(found_files[0]), acr.collection, + 'found fuzzy extension match for {0} in {1}'.format(full_name, acr.collection), action_plugin) + + def find_plugin(self, name, mod_type='', ignore_deprecated=False, check_aliases=False, collection_list=None): + ''' Find a plugin named name ''' + result = self.find_plugin_with_context(name, mod_type, ignore_deprecated, check_aliases, collection_list) + if result.resolved and result.plugin_resolved_path: + return result.plugin_resolved_path + + return None + + def find_plugin_with_context(self, name, mod_type='', ignore_deprecated=False, check_aliases=False, collection_list=None): + ''' Find a plugin named name, returning contextual info about the load, recursively resolving redirection ''' + plugin_load_context = PluginLoadContext() + plugin_load_context.original_name = name + while True: + result = self._resolve_plugin_step(name, mod_type, ignore_deprecated, check_aliases, collection_list, plugin_load_context=plugin_load_context) + if result.pending_redirect: + if result.pending_redirect in result.redirect_list: + raise AnsiblePluginCircularRedirect('plugin redirect loop resolving {0} (path: {1})'.format(result.original_name, result.redirect_list)) + name = result.pending_redirect + result.pending_redirect = None + plugin_load_context = result + else: + break + + # TODO: smuggle these to the controller when we're in a worker, reduce noise from normal things like missing plugin packages during collection search + if plugin_load_context.error_list: + display.warning("errors were encountered during the plugin load for {0}:\n{1}".format(name, plugin_load_context.error_list)) + + # TODO: display/return import_error_list? Only useful for forensics... + + # FIXME: store structured deprecation data in PluginLoadContext and use display.deprecate + # if plugin_load_context.deprecated and C.config.get_config_value('DEPRECATION_WARNINGS'): + # for dw in plugin_load_context.deprecation_warnings: + # # TODO: need to smuggle these to the controller if we're in a worker context + # display.warning('[DEPRECATION WARNING] ' + dw) + + return plugin_load_context + + # FIXME: name bikeshed + def _resolve_plugin_step(self, name, mod_type='', ignore_deprecated=False, + check_aliases=False, collection_list=None, plugin_load_context=PluginLoadContext()): + if not plugin_load_context: + raise ValueError('A PluginLoadContext is required') + + plugin_load_context.redirect_list.append(name) + plugin_load_context.resolved = False + + if name in _PLUGIN_FILTERS[self.package]: + plugin_load_context.exit_reason = '{0} matched a defined plugin filter'.format(name) + return plugin_load_context + + if mod_type: + suffix = mod_type + elif self.class_name: + # Ansible plugins that run in the controller process (most plugins) + suffix = '.py' + else: + # Only Ansible Modules. Ansible modules can be any executable so + # they can have any suffix + suffix = '' + + # FIXME: need this right now so we can still load shipped PS module_utils- come up with a more robust solution + if (AnsibleCollectionRef.is_valid_fqcr(name) or collection_list) and not name.startswith('Ansible'): + if '.' in name or not collection_list: + candidates = [name] + else: + candidates = ['{0}.{1}'.format(c, name) for c in collection_list] + + for candidate_name in candidates: + try: + plugin_load_context.load_attempts.append(candidate_name) + # HACK: refactor this properly + if candidate_name.startswith('ansible.legacy'): + # 'ansible.legacy' refers to the plugin finding behavior used before collections existed. + # They need to search 'library' and the various '*_plugins' directories in order to find the file. + plugin_load_context = self._find_plugin_legacy(name.removeprefix('ansible.legacy.'), + plugin_load_context, ignore_deprecated, check_aliases, suffix) + else: + # 'ansible.builtin' should be handled here. This means only internal, or builtin, paths are searched. + plugin_load_context = self._find_fq_plugin(candidate_name, suffix, plugin_load_context=plugin_load_context, + ignore_deprecated=ignore_deprecated) + + # Pending redirects are added to the redirect_list at the beginning of _resolve_plugin_step. + # Once redirects are resolved, ensure the final FQCN is added here. + # e.g. 'ns.coll.module' is included rather than only 'module' if a collections list is provided: + # - module: + # collections: ['ns.coll'] + if plugin_load_context.resolved and candidate_name not in plugin_load_context.redirect_list: + plugin_load_context.redirect_list.append(candidate_name) + + if plugin_load_context.resolved or plugin_load_context.pending_redirect: # if we got an answer or need to chase down a redirect, return + return plugin_load_context + except (AnsiblePluginRemovedError, AnsiblePluginCircularRedirect, AnsibleCollectionUnsupportedVersionError): + # these are generally fatal, let them fly + raise + except ImportError as ie: + plugin_load_context.import_error_list.append(ie) + except Exception as ex: + # FIXME: keep actual errors, not just assembled messages + plugin_load_context.error_list.append(to_native(ex)) + + if plugin_load_context.error_list: + display.debug(msg='plugin lookup for {0} failed; errors: {1}'.format(name, '; '.join(plugin_load_context.error_list))) + + plugin_load_context.exit_reason = 'no matches found for {0}'.format(name) + + return plugin_load_context + + # if we got here, there's no collection list and it's not an FQ name, so do legacy lookup + + return self._find_plugin_legacy(name, plugin_load_context, ignore_deprecated, check_aliases, suffix) + + def _find_plugin_legacy(self, name, plugin_load_context, ignore_deprecated=False, check_aliases=False, suffix=None): + """Search library and various *_plugins paths in order to find the file. + This was behavior prior to the existence of collections. + """ + plugin_load_context.resolved = False + + if check_aliases: + name = self.aliases.get(name, name) + + # The particular cache to look for modules within. This matches the + # requested mod_type + pull_cache = self._plugin_path_cache[suffix] + try: + path_with_context = pull_cache[name] + plugin_load_context.plugin_resolved_path = path_with_context.path + plugin_load_context.plugin_resolved_name = name + plugin_load_context.plugin_resolved_collection = 'ansible.builtin' if path_with_context.internal else '' + plugin_load_context._resolved_fqcn = ('ansible.builtin.' + name if path_with_context.internal else name) + plugin_load_context.resolved = True + return plugin_load_context + except KeyError: + # Cache miss. Now let's find the plugin + pass + + # TODO: Instead of using the self._paths cache (PATH_CACHE) and + # self._searched_paths we could use an iterator. Before enabling that + # we need to make sure we don't want to add additional directories + # (add_directory()) once we start using the iterator. + # We can use _get_paths_with_context() since add_directory() forces a cache refresh. + for path_with_context in (p for p in self._get_paths_with_context() if p.path not in self._searched_paths and os.path.isdir(to_bytes(p.path))): + path = path_with_context.path + b_path = to_bytes(path) + display.debug('trying %s' % path) + plugin_load_context.load_attempts.append(path) + internal = path_with_context.internal + try: + full_paths = (os.path.join(b_path, f) for f in os.listdir(b_path)) + except OSError as e: + display.warning("Error accessing plugin paths: %s" % to_text(e)) + + for full_path in (to_native(f) for f in full_paths if os.path.isfile(f) and not f.endswith(b'__init__.py')): + full_name = os.path.basename(full_path) + + # HACK: We have no way of executing python byte compiled files as ansible modules so specifically exclude them + # FIXME: I believe this is only correct for modules and module_utils. + # For all other plugins we want .pyc and .pyo should be valid + if any(full_path.endswith(x) for x in C.MODULE_IGNORE_EXTS): + continue + splitname = os.path.splitext(full_name) + base_name = splitname[0] + try: + extension = splitname[1] + except IndexError: + extension = '' + + # everything downstream expects unicode + full_path = to_text(full_path, errors='surrogate_or_strict') + # Module found, now enter it into the caches that match this file + if base_name not in self._plugin_path_cache['']: + self._plugin_path_cache[''][base_name] = PluginPathContext(full_path, internal) + + if full_name not in self._plugin_path_cache['']: + self._plugin_path_cache[''][full_name] = PluginPathContext(full_path, internal) + + if base_name not in self._plugin_path_cache[extension]: + self._plugin_path_cache[extension][base_name] = PluginPathContext(full_path, internal) + + if full_name not in self._plugin_path_cache[extension]: + self._plugin_path_cache[extension][full_name] = PluginPathContext(full_path, internal) + + self._searched_paths.add(path) + try: + path_with_context = pull_cache[name] + plugin_load_context.plugin_resolved_path = path_with_context.path + plugin_load_context.plugin_resolved_name = name + plugin_load_context.plugin_resolved_collection = 'ansible.builtin' if path_with_context.internal else '' + plugin_load_context._resolved_fqcn = 'ansible.builtin.' + name if path_with_context.internal else name + plugin_load_context.resolved = True + return plugin_load_context + except KeyError: + # Didn't find the plugin in this directory. Load modules from the next one + pass + + # if nothing is found, try finding alias/deprecated + if not name.startswith('_'): + alias_name = '_' + name + # We've already cached all the paths at this point + if alias_name in pull_cache: + path_with_context = pull_cache[alias_name] + if not ignore_deprecated and not os.path.islink(path_with_context.path): + # FIXME: this is not always the case, some are just aliases + display.deprecated('%s is kept for backwards compatibility but usage is discouraged. ' # pylint: disable=ansible-deprecated-no-version + 'The module documentation details page may explain more about this rationale.' % name.lstrip('_')) + plugin_load_context.plugin_resolved_path = path_with_context.path + plugin_load_context.plugin_resolved_name = alias_name + plugin_load_context.plugin_resolved_collection = 'ansible.builtin' if path_with_context.internal else '' + plugin_load_context._resolved_fqcn = 'ansible.builtin.' + alias_name if path_with_context.internal else alias_name + plugin_load_context.resolved = True + return plugin_load_context + + # last ditch, if it's something that can be redirected, look for a builtin redirect before giving up + candidate_fqcr = 'ansible.builtin.{0}'.format(name) + if '.' not in name and AnsibleCollectionRef.is_valid_fqcr(candidate_fqcr): + return self._find_fq_plugin(fq_name=candidate_fqcr, extension=suffix, plugin_load_context=plugin_load_context, ignore_deprecated=ignore_deprecated) + + return plugin_load_context.nope('{0} is not eligible for last-chance resolution'.format(name)) + + def has_plugin(self, name, collection_list=None): + ''' Checks if a plugin named name exists ''' + + try: + return self.find_plugin(name, collection_list=collection_list) is not None + except Exception as ex: + if isinstance(ex, AnsibleError): + raise + # log and continue, likely an innocuous type/package loading failure in collections import + display.debug('has_plugin error: {0}'.format(to_text(ex))) + + __contains__ = has_plugin + + def _load_module_source(self, name, path): + + # avoid collisions across plugins + if name.startswith('ansible_collections.'): + full_name = name + else: + full_name = '.'.join([self.package, name]) + + if full_name in sys.modules: + # Avoids double loading, See https://github.com/ansible/ansible/issues/13110 + return sys.modules[full_name] + + with warnings.catch_warnings(): + # FIXME: this still has issues if the module was previously imported but not "cached", + # we should bypass this entire codepath for things that are directly importable + warnings.simplefilter("ignore", RuntimeWarning) + spec = importlib.util.spec_from_file_location(to_native(full_name), to_native(path)) + module = importlib.util.module_from_spec(spec) + + # mimic import machinery; make the module-being-loaded available in sys.modules during import + # and remove if there's a failure... + sys.modules[full_name] = module + + try: + spec.loader.exec_module(module) + except Exception: + del sys.modules[full_name] + raise + + return module + + def _update_object(self, obj, name, path, redirected_names=None, resolved=None): + + # set extra info on the module, in case we want it later + setattr(obj, '_original_path', path) + setattr(obj, '_load_name', name) + setattr(obj, '_redirected_names', redirected_names or []) + + names = [] + if resolved: + names.append(resolved) + if redirected_names: + # reverse list so best name comes first + names.extend(redirected_names[::-1]) + if not names: + raise AnsibleError(f"Missing FQCN for plugin source {name}") + + setattr(obj, 'ansible_aliases', names) + setattr(obj, 'ansible_name', names[0]) + + def get(self, name, *args, **kwargs): + return self.get_with_context(name, *args, **kwargs).object + + def get_with_context(self, name, *args, **kwargs): + ''' instantiates a plugin of the given name using arguments ''' + + found_in_cache = True + class_only = kwargs.pop('class_only', False) + collection_list = kwargs.pop('collection_list', None) + if name in self.aliases: + name = self.aliases[name] + plugin_load_context = self.find_plugin_with_context(name, collection_list=collection_list) + if not plugin_load_context.resolved or not plugin_load_context.plugin_resolved_path: + # FIXME: this is probably an error (eg removed plugin) + return get_with_context_result(None, plugin_load_context) + + fq_name = plugin_load_context.resolved_fqcn + if '.' not in fq_name: + fq_name = '.'.join((plugin_load_context.plugin_resolved_collection, fq_name)) + name = plugin_load_context.plugin_resolved_name + path = plugin_load_context.plugin_resolved_path + redirected_names = plugin_load_context.redirect_list or [] + + if path not in self._module_cache: + self._module_cache[path] = self._load_module_source(name, path) + found_in_cache = False + + self._load_config_defs(name, self._module_cache[path], path) + + obj = getattr(self._module_cache[path], self.class_name) + + if self.base_class: + # The import path is hardcoded and should be the right place, + # so we are not expecting an ImportError. + module = __import__(self.package, fromlist=[self.base_class]) + # Check whether this obj has the required base class. + try: + plugin_class = getattr(module, self.base_class) + except AttributeError: + return get_with_context_result(None, plugin_load_context) + if not issubclass(obj, plugin_class): + return get_with_context_result(None, plugin_load_context) + + # FIXME: update this to use the load context + self._display_plugin_load(self.class_name, name, self._searched_paths, path, found_in_cache=found_in_cache, class_only=class_only) + + if not class_only: + try: + # A plugin may need to use its _load_name in __init__ (for example, to set + # or get options from config), so update the object before using the constructor + instance = object.__new__(obj) + self._update_object(instance, name, path, redirected_names, fq_name) + obj.__init__(instance, *args, **kwargs) # pylint: disable=unnecessary-dunder-call + obj = instance + except TypeError as e: + if "abstract" in e.args[0]: + # Abstract Base Class or incomplete plugin, don't load + display.v('Returning not found on "%s" as it has unimplemented abstract methods; %s' % (name, to_native(e))) + return get_with_context_result(None, plugin_load_context) + raise + + self._update_object(obj, name, path, redirected_names, fq_name) + return get_with_context_result(obj, plugin_load_context) + + def _display_plugin_load(self, class_name, name, searched_paths, path, found_in_cache=None, class_only=None): + ''' formats data to display debug info for plugin loading, also avoids processing unless really needed ''' + if C.DEFAULT_DEBUG: + msg = 'Loading %s \'%s\' from %s' % (class_name, os.path.basename(name), path) + + if len(searched_paths) > 1: + msg = '%s (searched paths: %s)' % (msg, self.format_paths(searched_paths)) + + if found_in_cache or class_only: + msg = '%s (found_in_cache=%s, class_only=%s)' % (msg, found_in_cache, class_only) + + display.debug(msg) + + def all(self, *args, **kwargs): + ''' + Iterate through all plugins of this type, in configured paths (no collections) + + A plugin loader is initialized with a specific type. This function is an iterator returning + all of the plugins of that type to the caller. + + :kwarg path_only: If this is set to True, then we return the paths to where the plugins reside + instead of an instance of the plugin. This conflicts with class_only and both should + not be set. + :kwarg class_only: If this is set to True then we return the python class which implements + a plugin rather than an instance of the plugin. This conflicts with path_only and both + should not be set. + :kwarg _dedupe: By default, we only return one plugin per plugin name. Deduplication happens + in the same way as the :meth:`get` and :meth:`find_plugin` methods resolve which plugin + should take precedence. If this is set to False, then we return all of the plugins + found, including those with duplicate names. In the case of duplicates, the order in + which they are returned is the one that would take precedence first, followed by the + others in decreasing precedence order. This should only be used by subclasses which + want to manage their own deduplication of the plugins. + :*args: Any extra arguments are passed to each plugin when it is instantiated. + :**kwargs: Any extra keyword arguments are passed to each plugin when it is instantiated. + ''' + # TODO: Change the signature of this method to: + # def all(return_type='instance', args=None, kwargs=None): + # if args is None: args = [] + # if kwargs is None: kwargs = {} + # return_type can be instance, class, or path. + # These changes will mean that plugin parameters won't conflict with our params and + # will also make it impossible to request both a path and a class at the same time. + # + # Move _dedupe to be a class attribute, CUSTOM_DEDUPE, with subclasses for filters and + # tests setting it to True + + dedupe = kwargs.pop('_dedupe', True) + path_only = kwargs.pop('path_only', False) + class_only = kwargs.pop('class_only', False) + # Having both path_only and class_only is a coding bug + if path_only and class_only: + raise AnsibleError('Do not set both path_only and class_only when calling PluginLoader.all()') + + all_matches = [] + found_in_cache = True + + legacy_excluding_builtin = set() + for path_with_context in self._get_paths_with_context(): + matches = glob.glob(to_native(os.path.join(path_with_context.path, "*.py"))) + if not path_with_context.internal: + legacy_excluding_builtin.update(matches) + # we sort within each path, but keep path precedence from config + all_matches.extend(sorted(matches, key=os.path.basename)) + + loaded_modules = set() + for path in all_matches: + name = os.path.splitext(path)[0] + basename = os.path.basename(name) + + if basename in _PLUGIN_FILTERS[self.package]: + display.debug("'%s' skipped due to a defined plugin filter" % basename) + continue + + if basename == '__init__' or (basename == 'base' and self.package == 'ansible.plugins.cache'): + # cache has legacy 'base.py' file, which is wrapper for __init__.py + display.debug("'%s' skipped due to reserved name" % basename) + continue + + if dedupe and basename in loaded_modules: + display.debug("'%s' skipped as duplicate" % basename) + continue + + loaded_modules.add(basename) + + if path_only: + yield path + continue + + if path not in self._module_cache: + if self.type in ('filter', 'test'): + # filter and test plugin files can contain multiple plugins + # they must have a unique python module name to prevent them from shadowing each other + full_name = '{0}_{1}'.format(abs(hash(path)), basename) + else: + full_name = basename + + try: + module = self._load_module_source(full_name, path) + except Exception as e: + display.warning("Skipping plugin (%s), cannot load: %s" % (path, to_text(e))) + continue + + self._module_cache[path] = module + found_in_cache = False + else: + module = self._module_cache[path] + + self._load_config_defs(basename, module, path) + + try: + obj = getattr(module, self.class_name) + except AttributeError as e: + display.warning("Skipping plugin (%s) as it seems to be invalid: %s" % (path, to_text(e))) + continue + + if self.base_class: + # The import path is hardcoded and should be the right place, + # so we are not expecting an ImportError. + module = __import__(self.package, fromlist=[self.base_class]) + # Check whether this obj has the required base class. + try: + plugin_class = getattr(module, self.base_class) + except AttributeError: + continue + if not issubclass(obj, plugin_class): + continue + + self._display_plugin_load(self.class_name, basename, self._searched_paths, path, found_in_cache=found_in_cache, class_only=class_only) + + if not class_only: + try: + obj = obj(*args, **kwargs) + except TypeError as e: + display.warning("Skipping plugin (%s) as it seems to be incomplete: %s" % (path, to_text(e))) + + if path in legacy_excluding_builtin: + fqcn = basename + else: + fqcn = f"ansible.builtin.{basename}" + self._update_object(obj, basename, path, resolved=fqcn) + yield obj + + +class Jinja2Loader(PluginLoader): + """ + PluginLoader optimized for Jinja2 plugins + + The filter and test plugins are Jinja2 plugins encapsulated inside of our plugin format. + We need to do a few things differently in the base class because of file == plugin + assumptions and dedupe logic. + """ + def __init__(self, class_name, package, config, subdir, aliases=None, required_base_class=None): + + super(Jinja2Loader, self).__init__(class_name, package, config, subdir, aliases=aliases, required_base_class=required_base_class) + self._loaded_j2_file_maps = [] + + def _clear_caches(self): + super(Jinja2Loader, self)._clear_caches() + self._loaded_j2_file_maps = [] + + def find_plugin(self, name, mod_type='', ignore_deprecated=False, check_aliases=False, collection_list=None): + + # TODO: handle collection plugin find, see 'get_with_context' + # this can really 'find plugin file' + plugin = super(Jinja2Loader, self).find_plugin(name, mod_type=mod_type, ignore_deprecated=ignore_deprecated, check_aliases=check_aliases, + collection_list=collection_list) + + # if not found, try loading all non collection plugins and see if this in there + if not plugin: + all_plugins = self.all() + plugin = all_plugins.get(name, None) + + return plugin + + @property + def method_map_name(self): + return get_plugin_class(self.class_name) + 's' + + def get_contained_plugins(self, collection, plugin_path, name): + + plugins = [] + + full_name = '.'.join(['ansible_collections', collection, 'plugins', self.type, name]) + try: + # use 'parent' loader class to find files, but cannot return this as it can contain multiple plugins per file + if plugin_path not in self._module_cache: + self._module_cache[plugin_path] = self._load_module_source(full_name, plugin_path) + module = self._module_cache[plugin_path] + obj = getattr(module, self.class_name) + except Exception as e: + raise KeyError('Failed to load %s for %s: %s' % (plugin_path, collection, to_native(e))) + + plugin_impl = obj() + if plugin_impl is None: + raise KeyError('Could not find %s.%s' % (collection, name)) + + try: + method_map = getattr(plugin_impl, self.method_map_name) + plugin_map = method_map().items() + except Exception as e: + display.warning("Ignoring %s plugins in '%s' as it seems to be invalid: %r" % (self.type, to_text(plugin_path), e)) + return plugins + + for func_name, func in plugin_map: + fq_name = '.'.join((collection, func_name)) + full = '.'.join((full_name, func_name)) + pclass = self._load_jinja2_class() + plugin = pclass(func) + if plugin in plugins: + continue + self._update_object(plugin, full, plugin_path, resolved=fq_name) + plugins.append(plugin) + + return plugins + + def get_with_context(self, name, *args, **kwargs): + + # found_in_cache = True + class_only = kwargs.pop('class_only', False) # just pop it, dont want to pass through + collection_list = kwargs.pop('collection_list', None) + + context = PluginLoadContext() + + # avoid collection path for legacy + name = name.removeprefix('ansible.legacy.') + + if '.' not in name: + # Filter/tests must always be FQCN except builtin and legacy + for known_plugin in self.all(*args, **kwargs): + if known_plugin.matches_name([name]): + context.resolved = True + context.plugin_resolved_name = name + context.plugin_resolved_path = known_plugin._original_path + context.plugin_resolved_collection = 'ansible.builtin' if known_plugin.ansible_name.startswith('ansible.builtin.') else '' + context._resolved_fqcn = known_plugin.ansible_name + return get_with_context_result(known_plugin, context) + + plugin = None + key, leaf_key = get_fqcr_and_name(name) + seen = set() + + # follow the meta! + while True: + + if key in seen: + raise AnsibleError('recursive collection redirect found for %r' % name, 0) + seen.add(key) + + acr = AnsibleCollectionRef.try_parse_fqcr(key, self.type) + if not acr: + raise KeyError('invalid plugin name: {0}'.format(key)) + + try: + ts = _get_collection_metadata(acr.collection) + except ValueError as e: + # no collection + raise KeyError('Invalid plugin FQCN ({0}): {1}'.format(key, to_native(e))) + + # TODO: implement cycle detection (unified across collection redir as well) + routing_entry = ts.get('plugin_routing', {}).get(self.type, {}).get(leaf_key, {}) + + # check deprecations + deprecation_entry = routing_entry.get('deprecation') + if deprecation_entry: + warning_text = deprecation_entry.get('warning_text') + removal_date = deprecation_entry.get('removal_date') + removal_version = deprecation_entry.get('removal_version') + + if not warning_text: + warning_text = '{0} "{1}" is deprecated'.format(self.type, key) + + display.deprecated(warning_text, version=removal_version, date=removal_date, collection_name=acr.collection) + + # check removal + tombstone_entry = routing_entry.get('tombstone') + if tombstone_entry: + warning_text = tombstone_entry.get('warning_text') + removal_date = tombstone_entry.get('removal_date') + removal_version = tombstone_entry.get('removal_version') + + if not warning_text: + warning_text = '{0} "{1}" has been removed'.format(self.type, key) + + exc_msg = display.get_deprecation_message(warning_text, version=removal_version, date=removal_date, + collection_name=acr.collection, removed=True) + + raise AnsiblePluginRemovedError(exc_msg) + + # check redirects + redirect = routing_entry.get('redirect', None) + if redirect: + if not AnsibleCollectionRef.is_valid_fqcr(redirect): + raise AnsibleError( + f"Collection {acr.collection} contains invalid redirect for {acr.collection}.{acr.resource}: {redirect}. " + "Redirects must use fully qualified collection names." + ) + + next_key, leaf_key = get_fqcr_and_name(redirect, collection=acr.collection) + display.vvv('redirecting (type: {0}) {1}.{2} to {3}'.format(self.type, acr.collection, acr.resource, next_key)) + key = next_key + else: + break + + try: + pkg = import_module(acr.n_python_package_name) + except ImportError as e: + raise KeyError(to_native(e)) + + parent_prefix = acr.collection + if acr.subdirs: + parent_prefix = '{0}.{1}'.format(parent_prefix, acr.subdirs) + + try: + for dummy, module_name, ispkg in pkgutil.iter_modules(pkg.__path__, prefix=parent_prefix + '.'): + if ispkg: + continue + + try: + # use 'parent' loader class to find files, but cannot return this as it can contain + # multiple plugins per file + plugin_impl = super(Jinja2Loader, self).get_with_context(module_name, *args, **kwargs) + except Exception as e: + raise KeyError(to_native(e)) + + try: + method_map = getattr(plugin_impl.object, self.method_map_name) + plugin_map = method_map().items() + except Exception as e: + display.warning("Skipping %s plugins in '%s' as it seems to be invalid: %r" % (self.type, to_text(plugin_impl.object._original_path), e)) + continue + + for func_name, func in plugin_map: + fq_name = '.'.join((parent_prefix, func_name)) + src_name = f"ansible_collections.{acr.collection}.plugins.{self.type}.{acr.subdirs}.{func_name}" + # TODO: load anyways into CACHE so we only match each at end of loop + # the files themseves should already be cached by base class caching of modules(python) + if key in (func_name, fq_name): + pclass = self._load_jinja2_class() + plugin = pclass(func) + if plugin: + context = plugin_impl.plugin_load_context + self._update_object(plugin, src_name, plugin_impl.object._original_path, resolved=fq_name) + break # go to next file as it can override if dupe (dont break both loops) + + except AnsiblePluginRemovedError as apre: + raise AnsibleError(to_native(apre), 0, orig_exc=apre) + except (AnsibleError, KeyError): + raise + except Exception as ex: + display.warning('An unexpected error occurred during Jinja2 plugin loading: {0}'.format(to_native(ex))) + display.vvv('Unexpected error during Jinja2 plugin loading: {0}'.format(format_exc())) + raise AnsibleError(to_native(ex), 0, orig_exc=ex) + + return get_with_context_result(plugin, context) + + def all(self, *args, **kwargs): + + # inputs, we ignore 'dedupe' we always do, used in base class to find files for this one + path_only = kwargs.pop('path_only', False) + class_only = kwargs.pop('class_only', False) # basically ignored for test/filters since they are functions + + # Having both path_only and class_only is a coding bug + if path_only and class_only: + raise AnsibleError('Do not set both path_only and class_only when calling PluginLoader.all()') + + found = set() + # get plugins from files in configured paths (multiple in each) + for p_map in self._j2_all_file_maps(*args, **kwargs): + + # p_map is really object from file with class that holds multiple plugins + plugins_list = getattr(p_map, self.method_map_name) + try: + plugins = plugins_list() + except Exception as e: + display.vvvv("Skipping %s plugins in '%s' as it seems to be invalid: %r" % (self.type, to_text(p_map._original_path), e)) + continue + + for plugin_name in plugins.keys(): + if plugin_name in _PLUGIN_FILTERS[self.package]: + display.debug("%s skipped due to a defined plugin filter" % plugin_name) + continue + + if plugin_name in found: + display.debug("%s skipped as duplicate" % plugin_name) + continue + + if path_only: + result = p_map._original_path + else: + # loader class is for the file with multiple plugins, but each plugin now has it's own class + pclass = self._load_jinja2_class() + result = pclass(plugins[plugin_name]) # if bad plugin, let exception rise + found.add(plugin_name) + fqcn = plugin_name + collection = '.'.join(p_map.ansible_name.split('.')[:2]) if p_map.ansible_name.count('.') >= 2 else '' + if not plugin_name.startswith(collection): + fqcn = f"{collection}.{plugin_name}" + + self._update_object(result, plugin_name, p_map._original_path, resolved=fqcn) + yield result + + def _load_jinja2_class(self): + """ override the normal method of plugin classname as these are used in the generic funciton + to access the 'multimap' of filter/tests to function, this is a 'singular' plugin for + each entry. + """ + class_name = 'AnsibleJinja2%s' % get_plugin_class(self.class_name).capitalize() + module = __import__(self.package, fromlist=[class_name]) + + return getattr(module, class_name) + + def _j2_all_file_maps(self, *args, **kwargs): + """ + * Unlike other plugin types, file != plugin, a file can contain multiple plugins (of same type). + This is why we do not deduplicate ansible file names at this point, we mostly care about + the names of the actual jinja2 plugins which are inside of our files. + * This method will NOT fetch collection plugin files, only those that would be expected under 'ansible.builtin/legacy'. + """ + # populate cache if needed + if not self._loaded_j2_file_maps: + + # We don't deduplicate ansible file names. + # Instead, calling code deduplicates jinja2 plugin names when loading each file. + kwargs['_dedupe'] = False + + # To match correct precedence, call base class' all() to get a list of files, + self._loaded_j2_file_maps = list(super(Jinja2Loader, self).all(*args, **kwargs)) + + return self._loaded_j2_file_maps + + +def get_fqcr_and_name(resource, collection='ansible.builtin'): + if '.' not in resource: + name = resource + fqcr = collection + '.' + resource + else: + name = resource.split('.')[-1] + fqcr = resource + + return fqcr, name + + +def _load_plugin_filter(): + filters = defaultdict(frozenset) + user_set = False + if C.PLUGIN_FILTERS_CFG is None: + filter_cfg = '/etc/ansible/plugin_filters.yml' + else: + filter_cfg = C.PLUGIN_FILTERS_CFG + user_set = True + + if os.path.exists(filter_cfg): + with open(filter_cfg, 'rb') as f: + try: + filter_data = from_yaml(f.read()) + except Exception as e: + display.warning(u'The plugin filter file, {0} was not parsable.' + u' Skipping: {1}'.format(filter_cfg, to_text(e))) + return filters + + try: + version = filter_data['filter_version'] + except KeyError: + display.warning(u'The plugin filter file, {0} was invalid.' + u' Skipping.'.format(filter_cfg)) + return filters + + # Try to convert for people specifying version as a float instead of string + version = to_text(version) + version = version.strip() + + if version == u'1.0': + # Modules and action plugins share the same blacklist since the difference between the + # two isn't visible to the users + try: + # reject list was documented but we never changed the code from blacklist, will be deprected in 2.15 + filters['ansible.modules'] = frozenset(filter_data.get('module_rejectlist)', filter_data['module_blacklist'])) + except TypeError: + display.warning(u'Unable to parse the plugin filter file {0} as' + u' module_blacklist is not a list.' + u' Skipping.'.format(filter_cfg)) + return filters + filters['ansible.plugins.action'] = filters['ansible.modules'] + else: + display.warning(u'The plugin filter file, {0} was a version not recognized by this' + u' version of Ansible. Skipping.'.format(filter_cfg)) + else: + if user_set: + display.warning(u'The plugin filter file, {0} does not exist.' + u' Skipping.'.format(filter_cfg)) + + # Specialcase the stat module as Ansible can run very few things if stat is blacklisted. + if 'stat' in filters['ansible.modules']: + raise AnsibleError('The stat module was specified in the module blacklist file, {0}, but' + ' Ansible will not function without the stat module. Please remove stat' + ' from the blacklist.'.format(to_native(filter_cfg))) + return filters + + +# since we don't want the actual collection loader understanding metadata, we'll do it in an event handler +def _on_collection_load_handler(collection_name, collection_path): + display.vvvv(to_text('Loading collection {0} from {1}'.format(collection_name, collection_path))) + + collection_meta = _get_collection_metadata(collection_name) + + try: + if not _does_collection_support_ansible_version(collection_meta.get('requires_ansible', ''), ansible_version): + mismatch_behavior = C.config.get_config_value('COLLECTIONS_ON_ANSIBLE_VERSION_MISMATCH') + message = 'Collection {0} does not support Ansible version {1}'.format(collection_name, ansible_version) + if mismatch_behavior == 'warning': + display.warning(message) + elif mismatch_behavior == 'error': + raise AnsibleCollectionUnsupportedVersionError(message) + except AnsibleError: + raise + except Exception as ex: + display.warning('Error parsing collection metadata requires_ansible value from collection {0}: {1}'.format(collection_name, ex)) + + +def _does_collection_support_ansible_version(requirement_string, ansible_version): + if not requirement_string: + return True + + if not SpecifierSet: + display.warning('packaging Python module unavailable; unable to validate collection Ansible version requirements') + return True + + ss = SpecifierSet(requirement_string) + + # ignore prerelease/postrelease/beta/dev flags for simplicity + base_ansible_version = Version(ansible_version).base_version + + return ss.contains(base_ansible_version) + + +def _configure_collection_loader(): + if AnsibleCollectionConfig.collection_finder: + # this must be a Python warning so that it can be filtered out by the import sanity test + warnings.warn('AnsibleCollectionFinder has already been configured') + return + + finder = _AnsibleCollectionFinder(C.COLLECTIONS_PATHS, C.COLLECTIONS_SCAN_SYS_PATH) + finder._install() + + # this should succeed now + AnsibleCollectionConfig.on_collection_load += _on_collection_load_handler + + +# TODO: All of the following is initialization code It should be moved inside of an initialization +# function which is called at some point early in the ansible and ansible-playbook CLI startup. + +_PLUGIN_FILTERS = _load_plugin_filter() + +_configure_collection_loader() + +# doc fragments first +fragment_loader = PluginLoader( + 'ModuleDocFragment', + 'ansible.plugins.doc_fragments', + C.DOC_FRAGMENT_PLUGIN_PATH, + 'doc_fragments', +) + +action_loader = PluginLoader( + 'ActionModule', + 'ansible.plugins.action', + C.DEFAULT_ACTION_PLUGIN_PATH, + 'action_plugins', + required_base_class='ActionBase', +) + +cache_loader = PluginLoader( + 'CacheModule', + 'ansible.plugins.cache', + C.DEFAULT_CACHE_PLUGIN_PATH, + 'cache_plugins', +) + +callback_loader = PluginLoader( + 'CallbackModule', + 'ansible.plugins.callback', + C.DEFAULT_CALLBACK_PLUGIN_PATH, + 'callback_plugins', +) + +connection_loader = PluginLoader( + 'Connection', + 'ansible.plugins.connection', + C.DEFAULT_CONNECTION_PLUGIN_PATH, + 'connection_plugins', + aliases={'paramiko': 'paramiko_ssh'}, + required_base_class='ConnectionBase', +) + +shell_loader = PluginLoader( + 'ShellModule', + 'ansible.plugins.shell', + 'shell_plugins', + 'shell_plugins', +) + +module_loader = PluginLoader( + '', + 'ansible.modules', + C.DEFAULT_MODULE_PATH, + 'library', +) + +module_utils_loader = PluginLoader( + '', + 'ansible.module_utils', + C.DEFAULT_MODULE_UTILS_PATH, + 'module_utils', +) + +# NB: dedicated loader is currently necessary because PS module_utils expects "with subdir" lookup where +# regular module_utils doesn't. This can be revisited once we have more granular loaders. +ps_module_utils_loader = PluginLoader( + '', + 'ansible.module_utils', + C.DEFAULT_MODULE_UTILS_PATH, + 'module_utils', +) + +lookup_loader = PluginLoader( + 'LookupModule', + 'ansible.plugins.lookup', + C.DEFAULT_LOOKUP_PLUGIN_PATH, + 'lookup_plugins', + required_base_class='LookupBase', +) + +filter_loader = Jinja2Loader( + 'FilterModule', + 'ansible.plugins.filter', + C.DEFAULT_FILTER_PLUGIN_PATH, + 'filter_plugins', +) + +test_loader = Jinja2Loader( + 'TestModule', + 'ansible.plugins.test', + C.DEFAULT_TEST_PLUGIN_PATH, + 'test_plugins' +) + +strategy_loader = PluginLoader( + 'StrategyModule', + 'ansible.plugins.strategy', + C.DEFAULT_STRATEGY_PLUGIN_PATH, + 'strategy_plugins', + required_base_class='StrategyBase', +) + +terminal_loader = PluginLoader( + 'TerminalModule', + 'ansible.plugins.terminal', + C.DEFAULT_TERMINAL_PLUGIN_PATH, + 'terminal_plugins', + required_base_class='TerminalBase' +) + +vars_loader = PluginLoader( + 'VarsModule', + 'ansible.plugins.vars', + C.DEFAULT_VARS_PLUGIN_PATH, + 'vars_plugins', +) + +cliconf_loader = PluginLoader( + 'Cliconf', + 'ansible.plugins.cliconf', + C.DEFAULT_CLICONF_PLUGIN_PATH, + 'cliconf_plugins', + required_base_class='CliconfBase' +) + +netconf_loader = PluginLoader( + 'Netconf', + 'ansible.plugins.netconf', + C.DEFAULT_NETCONF_PLUGIN_PATH, + 'netconf_plugins', + required_base_class='NetconfBase' +) + +inventory_loader = PluginLoader( + 'InventoryModule', + 'ansible.plugins.inventory', + C.DEFAULT_INVENTORY_PLUGIN_PATH, + 'inventory_plugins' +) + +httpapi_loader = PluginLoader( + 'HttpApi', + 'ansible.plugins.httpapi', + C.DEFAULT_HTTPAPI_PLUGIN_PATH, + 'httpapi_plugins', + required_base_class='HttpApiBase', +) + +become_loader = PluginLoader( + 'BecomeModule', + 'ansible.plugins.become', + C.BECOME_PLUGIN_PATH, + 'become_plugins' +) diff --git a/lib/ansible/plugins/lookup/__init__.py b/lib/ansible/plugins/lookup/__init__.py new file mode 100644 index 0000000..470f060 --- /dev/null +++ b/lib/ansible/plugins/lookup/__init__.py @@ -0,0 +1,130 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from abc import abstractmethod + +from ansible.errors import AnsibleFileNotFound +from ansible.plugins import AnsiblePlugin +from ansible.utils.display import Display + +display = Display() + +__all__ = ['LookupBase'] + + +class LookupBase(AnsiblePlugin): + + def __init__(self, loader=None, templar=None, **kwargs): + + super(LookupBase, self).__init__() + + self._loader = loader + self._templar = templar + + # Backwards compat: self._display isn't really needed, just import the global display and use that. + self._display = display + + def get_basedir(self, variables): + if 'role_path' in variables: + return variables['role_path'] + else: + return self._loader.get_basedir() + + @staticmethod + def _flatten(terms): + ret = [] + for term in terms: + if isinstance(term, (list, tuple)): + ret.extend(term) + else: + ret.append(term) + return ret + + @staticmethod + def _combine(a, b): + results = [] + for x in a: + for y in b: + results.append(LookupBase._flatten([x, y])) + return results + + @staticmethod + def _flatten_hash_to_list(terms): + ret = [] + for key in terms: + ret.append({'key': key, 'value': terms[key]}) + return ret + + @abstractmethod + def run(self, terms, variables=None, **kwargs): + """ + When the playbook specifies a lookup, this method is run. The + arguments to the lookup become the arguments to this method. One + additional keyword argument named ``variables`` is added to the method + call. It contains the variables available to ansible at the time the + lookup is templated. For instance:: + + "{{ lookup('url', 'https://toshio.fedorapeople.org/one.txt', validate_certs=True) }}" + + would end up calling the lookup plugin named url's run method like this:: + run(['https://toshio.fedorapeople.org/one.txt'], variables=available_variables, validate_certs=True) + + Lookup plugins can be used within playbooks for looping. When this + happens, the first argument is a list containing the terms. Lookup + plugins can also be called from within playbooks to return their + values into a variable or parameter. If the user passes a string in + this case, it is converted into a list. + + Errors encountered during execution should be returned by raising + AnsibleError() with a message describing the error. + + Any strings returned by this method that could ever contain non-ascii + must be converted into python's unicode type as the strings will be run + through jinja2 which has this requirement. You can use:: + + from ansible.module_utils._text import to_text + result_string = to_text(result_string) + """ + pass + + def find_file_in_search_path(self, myvars, subdir, needle, ignore_missing=False): + ''' + Return a file (needle) in the task's expected search path. + ''' + + if 'ansible_search_path' in myvars: + paths = myvars['ansible_search_path'] + else: + paths = [self.get_basedir(myvars)] + + result = None + try: + result = self._loader.path_dwim_relative_stack(paths, subdir, needle) + except AnsibleFileNotFound: + if not ignore_missing: + self._display.warning("Unable to find '%s' in expected paths (use -vvvvv to see paths)" % needle) + + return result + + def _deprecate_inline_kv(self): + # TODO: place holder to deprecate in future version allowing for long transition period + # self._display.deprecated('Passing inline k=v values embedded in a string to this lookup. Use direct ,k=v, k2=v2 syntax instead.', version='2.18') + pass diff --git a/lib/ansible/plugins/lookup/config.py b/lib/ansible/plugins/lookup/config.py new file mode 100644 index 0000000..3e5529b --- /dev/null +++ b/lib/ansible/plugins/lookup/config.py @@ -0,0 +1,156 @@ +# (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = """ + name: config + author: Ansible Core Team + version_added: "2.5" + short_description: Lookup current Ansible configuration values + description: + - Retrieves the value of an Ansible configuration setting. + - You can use C(ansible-config list) to see all available settings. + options: + _terms: + description: The key(s) to look up + required: True + on_missing: + description: + - action to take if term is missing from config + - Error will raise a fatal error + - Skip will just ignore the term + - Warn will skip over it but issue a warning + default: error + type: string + choices: ['error', 'skip', 'warn'] + plugin_type: + description: the type of the plugin referenced by 'plugin_name' option. + choices: ['become', 'cache', 'callback', 'cliconf', 'connection', 'httpapi', 'inventory', 'lookup', 'netconf', 'shell', 'vars'] + type: string + version_added: '2.12' + plugin_name: + description: name of the plugin for which you want to retrieve configuration settings. + type: string + version_added: '2.12' +""" + +EXAMPLES = """ + - name: Show configured default become user + ansible.builtin.debug: msg="{{ lookup('ansible.builtin.config', 'DEFAULT_BECOME_USER')}}" + + - name: print out role paths + ansible.builtin.debug: + msg: "These are the configured role paths: {{lookup('ansible.builtin.config', 'DEFAULT_ROLES_PATH')}}" + + - name: find retry files, skip if missing that key + ansible.builtin.find: + paths: "{{lookup('ansible.builtin.config', 'RETRY_FILES_SAVE_PATH')|default(playbook_dir, True)}}" + patterns: "*.retry" + + - name: see the colors + ansible.builtin.debug: msg="{{item}}" + loop: "{{lookup('ansible.builtin.config', 'COLOR_OK', 'COLOR_CHANGED', 'COLOR_SKIP', wantlist=True)}}" + + - name: skip if bad value in var + ansible.builtin.debug: msg="{{ lookup('ansible.builtin.config', config_in_var, on_missing='skip')}}" + var: + config_in_var: UNKNOWN + + - name: show remote user and port for ssh connection + ansible.builtin.debug: msg={{q("ansible.builtin.config", "remote_user", "port", plugin_type="connection", plugin_name="ssh", on_missing='skip')}} + + - name: show remote_tmp setting for shell (sh) plugin + ansible.builtin.debug: msg={{q("ansible.builtin.config", "remote_tmp", plugin_type="shell", plugin_name="sh")}} +""" + +RETURN = """ +_raw: + description: + - value(s) of the key(s) in the config + type: raw +""" + +import ansible.plugins.loader as plugin_loader + +from ansible import constants as C +from ansible.errors import AnsibleError, AnsibleLookupError, AnsibleOptionsError +from ansible.module_utils._text import to_native +from ansible.module_utils.six import string_types +from ansible.plugins.lookup import LookupBase +from ansible.utils.sentinel import Sentinel + + +class MissingSetting(AnsibleOptionsError): + pass + + +def _get_plugin_config(pname, ptype, config, variables): + try: + # plugin creates settings on load, this is cached so not too expensive to redo + loader = getattr(plugin_loader, '%s_loader' % ptype) + p = loader.get(pname, class_only=True) + if p is None: + raise AnsibleLookupError('Unable to load %s plugin "%s"' % (ptype, pname)) + result = C.config.get_config_value(config, plugin_type=ptype, plugin_name=p._load_name, variables=variables) + except AnsibleLookupError: + raise + except AnsibleError as e: + msg = to_native(e) + if 'was not defined' in msg: + raise MissingSetting(msg, orig_exc=e) + raise e + + return result + + +def _get_global_config(config): + try: + result = getattr(C, config) + if callable(result): + raise AnsibleLookupError('Invalid setting "%s" attempted' % config) + except AttributeError as e: + raise MissingSetting(to_native(e), orig_exc=e) + + return result + + +class LookupModule(LookupBase): + + def run(self, terms, variables=None, **kwargs): + + self.set_options(var_options=variables, direct=kwargs) + + missing = self.get_option('on_missing') + ptype = self.get_option('plugin_type') + pname = self.get_option('plugin_name') + + if (ptype or pname) and not (ptype and pname): + raise AnsibleOptionsError('Both plugin_type and plugin_name are required, cannot use one without the other') + + if not isinstance(missing, string_types) or missing not in ['error', 'warn', 'skip']: + raise AnsibleOptionsError('"on_missing" must be a string and one of "error", "warn" or "skip", not %s' % missing) + + ret = [] + + for term in terms: + if not isinstance(term, string_types): + raise AnsibleOptionsError('Invalid setting identifier, "%s" is not a string, its a %s' % (term, type(term))) + + result = Sentinel + try: + if pname: + result = _get_plugin_config(pname, ptype, term, variables) + else: + result = _get_global_config(term) + except MissingSetting as e: + if missing == 'error': + raise AnsibleLookupError('Unable to find setting %s' % term, orig_exc=e) + elif missing == 'warn': + self._display.warning('Skipping, did not find setting %s' % term) + elif missing == 'skip': + pass # this is not needed, but added to have all 3 options stated + + if result is not Sentinel: + ret.append(result) + return ret diff --git a/lib/ansible/plugins/lookup/csvfile.py b/lib/ansible/plugins/lookup/csvfile.py new file mode 100644 index 0000000..5932d77 --- /dev/null +++ b/lib/ansible/plugins/lookup/csvfile.py @@ -0,0 +1,181 @@ +# (c) 2013, Jan-Piet Mens <jpmens(at)gmail.com> +# (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = r""" + name: csvfile + author: Jan-Piet Mens (@jpmens) <jpmens(at)gmail.com> + version_added: "1.5" + short_description: read data from a TSV or CSV file + description: + - The csvfile lookup reads the contents of a file in CSV (comma-separated value) format. + The lookup looks for the row where the first column matches keyname (which can be multiple words) + and returns the value in the C(col) column (default 1, which indexed from 0 means the second column in the file). + options: + col: + description: column to return (0 indexed). + default: "1" + default: + description: what to return if the value is not found in the file. + delimiter: + description: field separator in the file, for a tab you can specify C(TAB) or C(\t). + default: TAB + file: + description: name of the CSV/TSV file to open. + default: ansible.csv + encoding: + description: Encoding (character set) of the used CSV file. + default: utf-8 + version_added: "2.1" + notes: + - The default is for TSV files (tab delimited) not CSV (comma delimited) ... yes the name is misleading. + - As of version 2.11, the search parameter (text that must match the first column of the file) and filename parameter can be multi-word. + - For historical reasons, in the search keyname, quotes are treated + literally and cannot be used around the string unless they appear + (escaped as required) in the first column of the file you are parsing. +""" + +EXAMPLES = """ +- name: Match 'Li' on the first column, return the second column (0 based index) + ansible.builtin.debug: msg="The atomic number of Lithium is {{ lookup('ansible.builtin.csvfile', 'Li file=elements.csv delimiter=,') }}" + +- name: msg="Match 'Li' on the first column, but return the 3rd column (columns start counting after the match)" + ansible.builtin.debug: msg="The atomic mass of Lithium is {{ lookup('ansible.builtin.csvfile', 'Li file=elements.csv delimiter=, col=2') }}" + +- name: Define Values From CSV File, this reads file in one go, but you could also use col= to read each in it's own lookup. + ansible.builtin.set_fact: + loop_ip: "{{ csvline[0] }}" + int_ip: "{{ csvline[1] }}" + int_mask: "{{ csvline[2] }}" + int_name: "{{ csvline[3] }}" + local_as: "{{ csvline[4] }}" + neighbor_as: "{{ csvline[5] }}" + neigh_int_ip: "{{ csvline[6] }}" + vars: + csvline = "{{ lookup('ansible.builtin.csvfile', bgp_neighbor_ip, file='bgp_neighbors.csv', delimiter=',') }}" + delegate_to: localhost +""" + +RETURN = """ + _raw: + description: + - value(s) stored in file column + type: list + elements: str +""" + +import codecs +import csv + +from collections.abc import MutableSequence + +from ansible.errors import AnsibleError, AnsibleAssertionError +from ansible.parsing.splitter import parse_kv +from ansible.plugins.lookup import LookupBase +from ansible.module_utils.six import PY2 +from ansible.module_utils._text import to_bytes, to_native, to_text + + +class CSVRecoder: + """ + Iterator that reads an encoded stream and reencodes the input to UTF-8 + """ + def __init__(self, f, encoding='utf-8'): + self.reader = codecs.getreader(encoding)(f) + + def __iter__(self): + return self + + def __next__(self): + return next(self.reader).encode("utf-8") + + next = __next__ # For Python 2 + + +class CSVReader: + """ + A CSV reader which will iterate over lines in the CSV file "f", + which is encoded in the given encoding. + """ + + def __init__(self, f, dialect=csv.excel, encoding='utf-8', **kwds): + if PY2: + f = CSVRecoder(f, encoding) + else: + f = codecs.getreader(encoding)(f) + + self.reader = csv.reader(f, dialect=dialect, **kwds) + + def __next__(self): + row = next(self.reader) + return [to_text(s) for s in row] + + next = __next__ # For Python 2 + + def __iter__(self): + return self + + +class LookupModule(LookupBase): + + def read_csv(self, filename, key, delimiter, encoding='utf-8', dflt=None, col=1): + + try: + f = open(to_bytes(filename), 'rb') + creader = CSVReader(f, delimiter=to_native(delimiter), encoding=encoding) + + for row in creader: + if len(row) and row[0] == key: + return row[int(col)] + except Exception as e: + raise AnsibleError("csvfile: %s" % to_native(e)) + + return dflt + + def run(self, terms, variables=None, **kwargs): + + ret = [] + + self.set_options(var_options=variables, direct=kwargs) + + # populate options + paramvals = self.get_options() + + for term in terms: + kv = parse_kv(term) + + if '_raw_params' not in kv: + raise AnsibleError('Search key is required but was not found') + + key = kv['_raw_params'] + + # parameters override per term using k/v + try: + for name, value in kv.items(): + if name == '_raw_params': + continue + if name not in paramvals: + raise AnsibleAssertionError('%s is not a valid option' % name) + + self._deprecate_inline_kv() + paramvals[name] = value + + except (ValueError, AssertionError) as e: + raise AnsibleError(e) + + # default is just placeholder for real tab + if paramvals['delimiter'] == 'TAB': + paramvals['delimiter'] = "\t" + + lookupfile = self.find_file_in_search_path(variables, 'files', paramvals['file']) + var = self.read_csv(lookupfile, key, paramvals['delimiter'], paramvals['encoding'], paramvals['default'], paramvals['col']) + if var is not None: + if isinstance(var, MutableSequence): + for v in var: + ret.append(v) + else: + ret.append(var) + + return ret diff --git a/lib/ansible/plugins/lookup/dict.py b/lib/ansible/plugins/lookup/dict.py new file mode 100644 index 0000000..af9a081 --- /dev/null +++ b/lib/ansible/plugins/lookup/dict.py @@ -0,0 +1,77 @@ +# (c) 2014, Kent R. Spillner <kspillner@acm.org> +# (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = """ + name: dict + version_added: "1.5" + short_description: returns key/value pair items from dictionaries + description: + - Takes dictionaries as input and returns a list with each item in the list being a dictionary with 'key' and 'value' as + keys to the previous dictionary's structure. + options: + _terms: + description: + - A list of dictionaries + required: True +""" + +EXAMPLES = """ +vars: + users: + alice: + name: Alice Appleworth + telephone: 123-456-7890 + bob: + name: Bob Bananarama + telephone: 987-654-3210 +tasks: + # with predefined vars + - name: Print phone records + ansible.builtin.debug: + msg: "User {{ item.key }} is {{ item.value.name }} ({{ item.value.telephone }})" + loop: "{{ lookup('ansible.builtin.dict', users) }}" + # with inline dictionary + - name: show dictionary + ansible.builtin.debug: + msg: "{{item.key}}: {{item.value}}" + with_dict: {a: 1, b: 2, c: 3} + # Items from loop can be used in when: statements + - name: set_fact when alice in key + ansible.builtin.set_fact: + alice_exists: true + loop: "{{ lookup('ansible.builtin.dict', users) }}" + when: "'alice' in item.key" +""" + +RETURN = """ + _list: + description: + - list of composed dictonaries with key and value + type: list +""" + +from collections.abc import Mapping + +from ansible.errors import AnsibleError +from ansible.plugins.lookup import LookupBase + + +class LookupModule(LookupBase): + + def run(self, terms, variables=None, **kwargs): + + # NOTE: can remove if with_ is removed + if not isinstance(terms, list): + terms = [terms] + + results = [] + for term in terms: + # Expect any type of Mapping, notably hostvars + if not isinstance(term, Mapping): + raise AnsibleError("with_dict expects a dict") + + results.extend(self._flatten_hash_to_list(term)) + return results diff --git a/lib/ansible/plugins/lookup/env.py b/lib/ansible/plugins/lookup/env.py new file mode 100644 index 0000000..3c37b90 --- /dev/null +++ b/lib/ansible/plugins/lookup/env.py @@ -0,0 +1,79 @@ +# (c) 2012, Jan-Piet Mens <jpmens(at)gmail.com> +# (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = """ + name: env + author: Jan-Piet Mens (@jpmens) <jpmens(at)gmail.com> + version_added: "0.9" + short_description: Read the value of environment variables + description: + - Allows you to query the environment variables available on the + controller when you invoked Ansible. + options: + _terms: + description: + - Environment variable or list of them to lookup the values for. + required: True + default: + description: What return when the variable is undefined + type: raw + default: '' + version_added: '2.13' + notes: + - You can pass the C(Undefined) object as C(default) to force an undefined error +""" + +EXAMPLES = """ +- name: Basic usage + ansible.builtin.debug: + msg: "'{{ lookup('ansible.builtin.env', 'HOME') }}' is the HOME environment variable." + +- name: Before 2.13, how to set default value if the variable is not defined. + This cannot distinguish between USR undefined and USR=''. + ansible.builtin.debug: + msg: "{{ lookup('ansible.builtin.env', 'USR')|default('nobody', True) }} is the user." + +- name: Example how to set default value if the variable is not defined, ignores USR='' + ansible.builtin.debug: + msg: "{{ lookup('ansible.builtin.env', 'USR', default='nobody') }} is the user." + +- name: Set default value to Undefined, if the variable is not defined + ansible.builtin.debug: + msg: "{{ lookup('ansible.builtin.env', 'USR', default=Undefined) }} is the user." + +- name: Set default value to undef(), if the variable is not defined + ansible.builtin.debug: + msg: "{{ lookup('ansible.builtin.env', 'USR', default=undef()) }} is the user." +""" + +RETURN = """ + _list: + description: + - Values from the environment variables. + type: list +""" + +from jinja2.runtime import Undefined + +from ansible.errors import AnsibleUndefinedVariable +from ansible.plugins.lookup import LookupBase +from ansible.utils import py3compat + + +class LookupModule(LookupBase): + def run(self, terms, variables, **kwargs): + + self.set_options(var_options=variables, direct=kwargs) + + ret = [] + d = self.get_option('default') + for term in terms: + var = term.split()[0] + val = py3compat.environ.get(var, d) + if isinstance(val, Undefined): + raise AnsibleUndefinedVariable('The "env" lookup, found an undefined variable: %s' % var) + ret.append(val) + return ret diff --git a/lib/ansible/plugins/lookup/file.py b/lib/ansible/plugins/lookup/file.py new file mode 100644 index 0000000..fa9191e --- /dev/null +++ b/lib/ansible/plugins/lookup/file.py @@ -0,0 +1,88 @@ +# (c) 2012, Daniel Hokka Zakrisson <daniel@hozac.com> +# (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = """ + name: file + author: Daniel Hokka Zakrisson (!UNKNOWN) <daniel@hozac.com> + version_added: "0.9" + short_description: read file contents + description: + - This lookup returns the contents from a file on the Ansible controller's file system. + options: + _terms: + description: path(s) of files to read + required: True + rstrip: + description: whether or not to remove whitespace from the ending of the looked-up file + type: bool + required: False + default: True + lstrip: + description: whether or not to remove whitespace from the beginning of the looked-up file + type: bool + required: False + default: False + notes: + - if read in variable context, the file can be interpreted as YAML if the content is valid to the parser. + - this lookup does not understand 'globbing', use the fileglob lookup instead. +""" + +EXAMPLES = """ +- ansible.builtin.debug: + msg: "the value of foo.txt is {{lookup('ansible.builtin.file', '/etc/foo.txt') }}" + +- name: display multiple file contents + ansible.builtin.debug: var=item + with_file: + - "/path/to/foo.txt" + - "bar.txt" # will be looked in files/ dir relative to play or in role + - "/path/to/biz.txt" +""" + +RETURN = """ + _raw: + description: + - content of file(s) + type: list + elements: str +""" + +from ansible.errors import AnsibleError, AnsibleParserError +from ansible.plugins.lookup import LookupBase +from ansible.module_utils._text import to_text +from ansible.utils.display import Display + +display = Display() + + +class LookupModule(LookupBase): + + def run(self, terms, variables=None, **kwargs): + + ret = [] + self.set_options(var_options=variables, direct=kwargs) + + for term in terms: + display.debug("File lookup term: %s" % term) + + # Find the file in the expected search path + lookupfile = self.find_file_in_search_path(variables, 'files', term) + display.vvvv(u"File lookup using %s as file" % lookupfile) + try: + if lookupfile: + b_contents, show_data = self._loader._get_file_contents(lookupfile) + contents = to_text(b_contents, errors='surrogate_or_strict') + if self.get_option('lstrip'): + contents = contents.lstrip() + if self.get_option('rstrip'): + contents = contents.rstrip() + ret.append(contents) + else: + raise AnsibleParserError() + except AnsibleParserError: + raise AnsibleError("could not locate file in lookup: %s" % term) + + return ret diff --git a/lib/ansible/plugins/lookup/fileglob.py b/lib/ansible/plugins/lookup/fileglob.py new file mode 100644 index 0000000..abf8202 --- /dev/null +++ b/lib/ansible/plugins/lookup/fileglob.py @@ -0,0 +1,84 @@ +# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com> +# (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = """ + name: fileglob + author: Michael DeHaan + version_added: "1.4" + short_description: list files matching a pattern + description: + - Matches all files in a single directory, non-recursively, that match a pattern. + It calls Python's "glob" library. + options: + _terms: + description: path(s) of files to read + required: True + notes: + - Patterns are only supported on files, not directory/paths. + - See R(Ansible task paths,playbook_task_paths) to understand how file lookup occurs with paths. + - Matching is against local system files on the Ansible controller. + To iterate a list of files on a remote node, use the M(ansible.builtin.find) module. + - Returns a string list of paths joined by commas, or an empty list if no files match. For a 'true list' pass C(wantlist=True) to the lookup. +""" + +EXAMPLES = """ +- name: Display paths of all .txt files in dir + ansible.builtin.debug: msg={{ lookup('ansible.builtin.fileglob', '/my/path/*.txt') }} + +- name: Copy each file over that matches the given pattern + ansible.builtin.copy: + src: "{{ item }}" + dest: "/etc/fooapp/" + owner: "root" + mode: 0600 + with_fileglob: + - "/playbooks/files/fooapp/*" +""" + +RETURN = """ + _list: + description: + - list of files + type: list + elements: path +""" + +import os +import glob + +from ansible.plugins.lookup import LookupBase +from ansible.errors import AnsibleFileNotFound +from ansible.module_utils._text import to_bytes, to_text + + +class LookupModule(LookupBase): + + def run(self, terms, variables=None, **kwargs): + + ret = [] + for term in terms: + term_file = os.path.basename(term) + found_paths = [] + if term_file != term: + found_paths.append(self.find_file_in_search_path(variables, 'files', os.path.dirname(term))) + else: + # no dir, just file, so use paths and 'files' paths instead + if 'ansible_search_path' in variables: + paths = variables['ansible_search_path'] + else: + paths = [self.get_basedir(variables)] + for p in paths: + found_paths.append(os.path.join(p, 'files')) + found_paths.append(p) + + for dwimmed_path in found_paths: + if dwimmed_path: + globbed = glob.glob(to_bytes(os.path.join(dwimmed_path, term_file), errors='surrogate_or_strict')) + term_results = [to_text(g, errors='surrogate_or_strict') for g in globbed if os.path.isfile(g)] + if term_results: + ret.extend(term_results) + break + return ret diff --git a/lib/ansible/plugins/lookup/first_found.py b/lib/ansible/plugins/lookup/first_found.py new file mode 100644 index 0000000..5b94b10 --- /dev/null +++ b/lib/ansible/plugins/lookup/first_found.py @@ -0,0 +1,235 @@ +# (c) 2013, seth vidal <skvidal@fedoraproject.org> red hat, inc +# (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = """ + name: first_found + author: Seth Vidal (!UNKNOWN) <skvidal@fedoraproject.org> + version_added: historical + short_description: return first file found from list + description: + - This lookup checks a list of files and paths and returns the full path to the first combination found. + - As all lookups, when fed relative paths it will try use the current task's location first and go up the chain + to the containing locations of role / play / include and so on. + - The list of files has precedence over the paths searched. + For example, A task in a role has a 'file1' in the play's relative path, this will be used, 'file2' in role's relative path will not. + - Either a list of files C(_terms) or a key C(files) with a list of files is required for this plugin to operate. + notes: + - This lookup can be used in 'dual mode', either passing a list of file names or a dictionary that has C(files) and C(paths). + options: + _terms: + description: A list of file names. + files: + description: A list of file names. + type: list + elements: string + default: [] + paths: + description: A list of paths in which to look for the files. + type: list + elements: string + default: [] + skip: + type: boolean + default: False + description: + - When C(True), return an empty list when no files are matched. + - This is useful when used with C(with_first_found), as an empty list return to C(with_) calls + causes the calling task to be skipped. + - When used as a template via C(lookup) or C(query), setting I(skip=True) will *not* cause the task to skip. + Tasks must handle the empty list return from the template. + - When C(False) and C(lookup) or C(query) specifies I(errors='ignore') all errors (including no file found, + but potentially others) return an empty string or an empty list respectively. + - When C(True) and C(lookup) or C(query) specifies I(errors='ignore'), no file found will return an empty + list and other potential errors return an empty string or empty list depending on the template call + (in other words return values of C(lookup) v C(query)). +""" + +EXAMPLES = """ +- name: Set _found_file to the first existing file, raising an error if a file is not found + ansible.builtin.set_fact: + _found_file: "{{ lookup('ansible.builtin.first_found', findme) }}" + vars: + findme: + - /path/to/foo.txt + - bar.txt # will be looked in files/ dir relative to role and/or play + - /path/to/biz.txt + +- name: Set _found_file to the first existing file, or an empty list if no files found + ansible.builtin.set_fact: + _found_file: "{{ lookup('ansible.builtin.first_found', files, paths=['/extra/path'], skip=True) }}" + vars: + files: + - /path/to/foo.txt + - /path/to/bar.txt + +- name: Include tasks only if one of the files exist, otherwise skip the task + ansible.builtin.include_tasks: + file: "{{ item }}" + with_first_found: + - files: + - path/tasks.yaml + - path/other_tasks.yaml + skip: True + +- name: Include tasks only if one of the files exists, otherwise skip + ansible.builtin.include_tasks: '{{ tasks_file }}' + when: tasks_file != "" + vars: + tasks_file: "{{ lookup('ansible.builtin.first_found', files=['tasks.yaml', 'other_tasks.yaml'], errors='ignore') }}" + +- name: | + copy first existing file found to /some/file, + looking in relative directories from where the task is defined and + including any play objects that contain it + ansible.builtin.copy: + src: "{{ lookup('ansible.builtin.first_found', findme) }}" + dest: /some/file + vars: + findme: + - foo + - "{{ inventory_hostname }}" + - bar + +- name: same copy but specific paths + ansible.builtin.copy: + src: "{{ lookup('ansible.builtin.first_found', params) }}" + dest: /some/file + vars: + params: + files: + - foo + - "{{ inventory_hostname }}" + - bar + paths: + - /tmp/production + - /tmp/staging + +- name: INTERFACES | Create Ansible header for /etc/network/interfaces + ansible.builtin.template: + src: "{{ lookup('ansible.builtin.first_found', findme)}}" + dest: "/etc/foo.conf" + vars: + findme: + - "{{ ansible_virtualization_type }}_foo.conf" + - "default_foo.conf" + +- name: read vars from first file found, use 'vars/' relative subdir + ansible.builtin.include_vars: "{{lookup('ansible.builtin.first_found', params)}}" + vars: + params: + files: + - '{{ ansible_distribution }}.yml' + - '{{ ansible_os_family }}.yml' + - default.yml + paths: + - 'vars' +""" + +RETURN = """ + _raw: + description: + - path to file found + type: list + elements: path +""" +import os +import re + +from collections.abc import Mapping, Sequence + +from jinja2.exceptions import UndefinedError + +from ansible.errors import AnsibleLookupError, AnsibleUndefinedVariable +from ansible.module_utils.six import string_types +from ansible.plugins.lookup import LookupBase + + +def _split_on(terms, spliters=','): + termlist = [] + if isinstance(terms, string_types): + termlist = re.split(r'[%s]' % ''.join(map(re.escape, spliters)), terms) + else: + # added since options will already listify + for t in terms: + termlist.extend(_split_on(t, spliters)) + return termlist + + +class LookupModule(LookupBase): + + def _process_terms(self, terms, variables, kwargs): + + total_search = [] + skip = False + + # can use a dict instead of list item to pass inline config + for term in terms: + if isinstance(term, Mapping): + self.set_options(var_options=variables, direct=term) + elif isinstance(term, string_types): + self.set_options(var_options=variables, direct=kwargs) + elif isinstance(term, Sequence): + partial, skip = self._process_terms(term, variables, kwargs) + total_search.extend(partial) + continue + else: + raise AnsibleLookupError("Invalid term supplied, can handle string, mapping or list of strings but got: %s for %s" % (type(term), term)) + + files = self.get_option('files') + paths = self.get_option('paths') + + # NOTE: this is used as 'global' but can be set many times?!?!? + skip = self.get_option('skip') + + # magic extra spliting to create lists + filelist = _split_on(files, ',;') + pathlist = _split_on(paths, ',:;') + + # create search structure + if pathlist: + for path in pathlist: + for fn in filelist: + f = os.path.join(path, fn) + total_search.append(f) + elif filelist: + # NOTE: this seems wrong, should be 'extend' as any option/entry can clobber all + total_search = filelist + else: + total_search.append(term) + + return total_search, skip + + def run(self, terms, variables, **kwargs): + + total_search, skip = self._process_terms(terms, variables, kwargs) + + # NOTE: during refactor noticed that the 'using a dict' as term + # is designed to only work with 'one' otherwise inconsistencies will appear. + # see other notes below. + + # actually search + subdir = getattr(self, '_subdir', 'files') + + path = None + for fn in total_search: + + try: + fn = self._templar.template(fn) + except (AnsibleUndefinedVariable, UndefinedError): + continue + + # get subdir if set by task executor, default to files otherwise + path = self.find_file_in_search_path(variables, subdir, fn, ignore_missing=True) + + # exit if we find one! + if path is not None: + return [path] + + # if we get here, no file was found + if skip: + # NOTE: global skip wont matter, only last 'skip' value in dict term + return [] + raise AnsibleLookupError("No file was found when using first_found.") diff --git a/lib/ansible/plugins/lookup/indexed_items.py b/lib/ansible/plugins/lookup/indexed_items.py new file mode 100644 index 0000000..f63a895 --- /dev/null +++ b/lib/ansible/plugins/lookup/indexed_items.py @@ -0,0 +1,52 @@ +# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com> +# (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = """ + name: indexed_items + author: Michael DeHaan + version_added: "1.3" + short_description: rewrites lists to return 'indexed items' + description: + - use this lookup if you want to loop over an array and also get the numeric index of where you are in the array as you go + - any list given will be transformed with each resulting element having the it's previous position in item.0 and its value in item.1 + options: + _terms: + description: list of items + required: True +""" + +EXAMPLES = """ +- name: indexed loop demo + ansible.builtin.debug: + msg: "at array position {{ item.0 }} there is a value {{ item.1 }}" + with_indexed_items: + - "{{ some_list }}" +""" + +RETURN = """ + _raw: + description: + - list with each item.0 giving you the position and item.1 the value + type: list + elements: list +""" + +from ansible.errors import AnsibleError +from ansible.plugins.lookup import LookupBase + + +class LookupModule(LookupBase): + + def __init__(self, basedir=None, **kwargs): + self.basedir = basedir + + def run(self, terms, variables, **kwargs): + + if not isinstance(terms, list): + raise AnsibleError("with_indexed_items expects a list") + + items = self._flatten(terms) + return list(zip(range(len(items)), items)) diff --git a/lib/ansible/plugins/lookup/ini.py b/lib/ansible/plugins/lookup/ini.py new file mode 100644 index 0000000..eea8634 --- /dev/null +++ b/lib/ansible/plugins/lookup/ini.py @@ -0,0 +1,204 @@ +# (c) 2015, Yannig Perre <yannig.perre(at)gmail.com> +# (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = """ + name: ini + author: Yannig Perre (!UNKNOWN) <yannig.perre(at)gmail.com> + version_added: "2.0" + short_description: read data from an ini file + description: + - "The ini lookup reads the contents of a file in INI format C(key1=value1). + This plugin retrieves the value on the right side after the equal sign C('=') of a given section C([section])." + - "You can also read a property file which - in this case - does not contain section." + options: + _terms: + description: The key(s) to look up. + required: True + type: + description: Type of the file. 'properties' refers to the Java properties files. + default: 'ini' + choices: ['ini', 'properties'] + file: + description: Name of the file to load. + default: 'ansible.ini' + section: + default: global + description: Section where to lookup the key. + re: + default: False + type: boolean + description: Flag to indicate if the key supplied is a regexp. + encoding: + default: utf-8 + description: Text encoding to use. + default: + description: Return value if the key is not in the ini file. + default: '' + case_sensitive: + description: + Whether key names read from C(file) should be case sensitive. This prevents + duplicate key errors if keys only differ in case. + default: False + version_added: '2.12' + allow_no_value: + description: + - Read an ini file which contains key without value and without '=' symbol. + type: bool + default: False + aliases: ['allow_none'] + version_added: '2.12' +""" + +EXAMPLES = """ +- ansible.builtin.debug: msg="User in integration is {{ lookup('ansible.builtin.ini', 'user', section='integration', file='users.ini') }}" + +- ansible.builtin.debug: msg="User in production is {{ lookup('ansible.builtin.ini', 'user', section='production', file='users.ini') }}" + +- ansible.builtin.debug: msg="user.name is {{ lookup('ansible.builtin.ini', 'user.name', type='properties', file='user.properties') }}" + +- ansible.builtin.debug: + msg: "{{ item }}" + loop: "{{ q('ansible.builtin.ini', '.*', section='section1', file='test.ini', re=True) }}" + +- name: Read an ini file with allow_no_value + ansible.builtin.debug: + msg: "{{ lookup('ansible.builtin.ini', 'user', file='mysql.ini', section='mysqld', allow_no_value=True) }}" +""" + +RETURN = """ +_raw: + description: + - value(s) of the key(s) in the ini file + type: list + elements: str +""" + +import configparser +import os +import re + +from io import StringIO +from collections import defaultdict +from collections.abc import MutableSequence + +from ansible.errors import AnsibleLookupError, AnsibleOptionsError +from ansible.module_utils._text import to_text, to_native +from ansible.plugins.lookup import LookupBase + + +def _parse_params(term, paramvals): + '''Safely split parameter term to preserve spaces''' + + # TODO: deprecate this method + valid_keys = paramvals.keys() + params = defaultdict(lambda: '') + + # TODO: check kv_parser to see if it can handle spaces this same way + keys = [] + thiskey = 'key' # initialize for 'lookup item' + for idp, phrase in enumerate(term.split()): + + # update current key if used + if '=' in phrase: + for k in valid_keys: + if ('%s=' % k) in phrase: + thiskey = k + + # if first term or key does not exist + if idp == 0 or not params[thiskey]: + params[thiskey] = phrase + keys.append(thiskey) + else: + # append to existing key + params[thiskey] += ' ' + phrase + + # return list of values + return [params[x] for x in keys] + + +class LookupModule(LookupBase): + + def get_value(self, key, section, dflt, is_regexp): + # Retrieve all values from a section using a regexp + if is_regexp: + return [v for k, v in self.cp.items(section) if re.match(key, k)] + value = None + # Retrieve a single value + try: + value = self.cp.get(section, key) + except configparser.NoOptionError: + return dflt + return value + + def run(self, terms, variables=None, **kwargs): + + self.set_options(var_options=variables, direct=kwargs) + paramvals = self.get_options() + + self.cp = configparser.ConfigParser(allow_no_value=paramvals.get('allow_no_value', paramvals.get('allow_none'))) + if paramvals['case_sensitive']: + self.cp.optionxform = to_native + + ret = [] + for term in terms: + + key = term + # parameters specified? + if '=' in term or ' ' in term.strip(): + self._deprecate_inline_kv() + params = _parse_params(term, paramvals) + try: + updated_key = False + for param in params: + if '=' in param: + name, value = param.split('=') + if name not in paramvals: + raise AnsibleLookupError('%s is not a valid option.' % name) + paramvals[name] = value + elif key == term: + # only take first, this format never supported multiple keys inline + key = param + updated_key = True + except ValueError as e: + # bad params passed + raise AnsibleLookupError("Could not use '%s' from '%s': %s" % (param, params, to_native(e)), orig_exc=e) + if not updated_key: + raise AnsibleOptionsError("No key to lookup was provided as first term with in string inline options: %s" % term) + # only passed options in inline string + + # TODO: look to use cache to avoid redoing this for every term if they use same file + # Retrieve file path + path = self.find_file_in_search_path(variables, 'files', paramvals['file']) + + # Create StringIO later used to parse ini + config = StringIO() + # Special case for java properties + if paramvals['type'] == "properties": + config.write(u'[java_properties]\n') + paramvals['section'] = 'java_properties' + + # Open file using encoding + contents, show_data = self._loader._get_file_contents(path) + contents = to_text(contents, errors='surrogate_or_strict', encoding=paramvals['encoding']) + config.write(contents) + config.seek(0, os.SEEK_SET) + + try: + self.cp.readfp(config) + except configparser.DuplicateOptionError as doe: + raise AnsibleLookupError("Duplicate option in '{file}': {error}".format(file=paramvals['file'], error=to_native(doe))) + + try: + var = self.get_value(key, paramvals['section'], paramvals['default'], paramvals['re']) + except configparser.NoSectionError: + raise AnsibleLookupError("No section '{section}' in {file}".format(section=paramvals['section'], file=paramvals['file'])) + if var is not None: + if isinstance(var, MutableSequence): + for v in var: + ret.append(v) + else: + ret.append(var) + return ret diff --git a/lib/ansible/plugins/lookup/inventory_hostnames.py b/lib/ansible/plugins/lookup/inventory_hostnames.py new file mode 100644 index 0000000..4fa1d68 --- /dev/null +++ b/lib/ansible/plugins/lookup/inventory_hostnames.py @@ -0,0 +1,53 @@ +# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com> +# (c) 2013, Steven Dossett <sdossett@panath.com> +# (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = """ + name: inventory_hostnames + author: + - Michael DeHaan + - Steven Dossett (!UNKNOWN) <sdossett@panath.com> + version_added: "1.3" + short_description: list of inventory hosts matching a host pattern + description: + - "This lookup understands 'host patterns' as used by the C(hosts:) keyword in plays + and can return a list of matching hosts from inventory" + notes: + - this is only worth for 'hostname patterns' it is easier to loop over the group/group_names variables otherwise. +""" + +EXAMPLES = """ +- name: show all the hosts matching the pattern, i.e. all but the group www + ansible.builtin.debug: + msg: "{{ item }}" + with_inventory_hostnames: + - all:!www +""" + +RETURN = """ + _hostnames: + description: list of hostnames that matched the host pattern in inventory + type: list +""" + +from ansible.errors import AnsibleError +from ansible.inventory.manager import InventoryManager +from ansible.plugins.lookup import LookupBase + + +class LookupModule(LookupBase): + def run(self, terms, variables=None, **kwargs): + manager = InventoryManager(self._loader, parse=False) + for group, hosts in variables['groups'].items(): + manager.add_group(group) + for host in hosts: + manager.add_host(host, group=group) + + try: + return [h.name for h in manager.get_hosts(pattern=terms)] + except AnsibleError: + return [] diff --git a/lib/ansible/plugins/lookup/items.py b/lib/ansible/plugins/lookup/items.py new file mode 100644 index 0000000..162c1e7 --- /dev/null +++ b/lib/ansible/plugins/lookup/items.py @@ -0,0 +1,73 @@ +# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com> +# (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = """ + name: items + author: Michael DeHaan + version_added: historical + short_description: list of items + description: + - this lookup returns a list of items given to it, if any of the top level items is also a list it will flatten it, but it will not recurse + notes: + - this is the standard lookup used for loops in most examples + - check out the 'flattened' lookup for recursive flattening + - if you do not want flattening nor any other transformation look at the 'list' lookup. + options: + _terms: + description: list of items + required: True +""" + +EXAMPLES = """ +- name: "loop through list" + ansible.builtin.debug: + msg: "An item: {{ item }}" + with_items: + - 1 + - 2 + - 3 + +- name: add several users + ansible.builtin.user: + name: "{{ item }}" + groups: "wheel" + state: present + with_items: + - testuser1 + - testuser2 + +- name: "loop through list from a variable" + ansible.builtin.debug: + msg: "An item: {{ item }}" + with_items: "{{ somelist }}" + +- name: more complex items to add several users + ansible.builtin.user: + name: "{{ item.name }}" + uid: "{{ item.uid }}" + groups: "{{ item.groups }}" + state: present + with_items: + - { name: testuser1, uid: 1002, groups: "wheel, staff" } + - { name: testuser2, uid: 1003, groups: staff } + +""" + +RETURN = """ + _raw: + description: + - once flattened list + type: list +""" + +from ansible.plugins.lookup import LookupBase + + +class LookupModule(LookupBase): + + def run(self, terms, **kwargs): + + return self._flatten(terms) diff --git a/lib/ansible/plugins/lookup/lines.py b/lib/ansible/plugins/lookup/lines.py new file mode 100644 index 0000000..7676d01 --- /dev/null +++ b/lib/ansible/plugins/lookup/lines.py @@ -0,0 +1,62 @@ +# (c) 2012, Daniel Hokka Zakrisson <daniel@hozac.com> +# (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = """ + name: lines + author: Daniel Hokka Zakrisson (!UNKNOWN) <daniel@hozac.com> + version_added: "0.9" + short_description: read lines from command + description: + - Run one or more commands and split the output into lines, returning them as a list + options: + _terms: + description: command(s) to run + required: True + notes: + - Like all lookups, this runs on the Ansible controller and is unaffected by other keywords such as 'become'. + If you need to use different permissions, you must change the command or run Ansible as another user. + - Alternatively, you can use a shell/command task that runs against localhost and registers the result. +""" + +EXAMPLES = """ +- name: We could read the file directly, but this shows output from command + ansible.builtin.debug: msg="{{ item }} is an output line from running cat on /etc/motd" + with_lines: cat /etc/motd + +- name: More useful example of looping over a command result + ansible.builtin.shell: "/usr/bin/frobnicate {{ item }}" + with_lines: + - "/usr/bin/frobnications_per_host --param {{ inventory_hostname }}" +""" + +RETURN = """ + _list: + description: + - lines of stdout from command + type: list + elements: str +""" + +import subprocess +from ansible.errors import AnsibleError +from ansible.plugins.lookup import LookupBase +from ansible.module_utils._text import to_text + + +class LookupModule(LookupBase): + + def run(self, terms, variables, **kwargs): + + ret = [] + for term in terms: + p = subprocess.Popen(term, cwd=self._loader.get_basedir(), shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE) + (stdout, stderr) = p.communicate() + if p.returncode == 0: + ret.extend([to_text(l) for l in stdout.splitlines()]) + else: + raise AnsibleError("lookup_plugin.lines(%s) returned %d" % (term, p.returncode)) + return ret diff --git a/lib/ansible/plugins/lookup/list.py b/lib/ansible/plugins/lookup/list.py new file mode 100644 index 0000000..6c553ae --- /dev/null +++ b/lib/ansible/plugins/lookup/list.py @@ -0,0 +1,45 @@ +# (c) 2012-17 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) + +__metaclass__ = type + +DOCUMENTATION = """ + name: list + author: Ansible Core Team + version_added: "2.0" + short_description: simply returns what it is given. + description: + - this is mostly a noop, to be used as a with_list loop when you dont want the content transformed in any way. +""" + +EXAMPLES = """ +- name: unlike with_items you will get 3 items from this loop, the 2nd one being a list + ansible.builtin.debug: var=item + with_list: + - 1 + - [2,3] + - 4 +""" + +RETURN = """ + _list: + description: basically the same as you fed in + type: list + elements: raw +""" + +from collections.abc import Sequence + +from ansible.plugins.lookup import LookupBase +from ansible.errors import AnsibleError + + +class LookupModule(LookupBase): + + def run(self, terms, **kwargs): + if not isinstance(terms, Sequence): + raise AnsibleError("with_list expects a list") + return terms diff --git a/lib/ansible/plugins/lookup/nested.py b/lib/ansible/plugins/lookup/nested.py new file mode 100644 index 0000000..e768dba --- /dev/null +++ b/lib/ansible/plugins/lookup/nested.py @@ -0,0 +1,85 @@ +# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com> +# (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = """ + name: nested + version_added: "1.1" + short_description: composes a list with nested elements of other lists + description: + - Takes the input lists and returns a list with elements that are lists composed of the elements of the input lists + options: + _raw: + description: + - a set of lists + required: True +""" + +EXAMPLES = """ +- name: give users access to multiple databases + community.mysql.mysql_user: + name: "{{ item[0] }}" + priv: "{{ item[1] }}.*:ALL" + append_privs: yes + password: "foo" + with_nested: + - [ 'alice', 'bob' ] + - [ 'clientdb', 'employeedb', 'providerdb' ] +# As with the case of 'with_items' above, you can use previously defined variables.: + +- name: here, 'users' contains the above list of employees + community.mysql.mysql_user: + name: "{{ item[0] }}" + priv: "{{ item[1] }}.*:ALL" + append_privs: yes + password: "foo" + with_nested: + - "{{ users }}" + - [ 'clientdb', 'employeedb', 'providerdb' ] +""" + +RETURN = """ + _list: + description: + - A list composed of lists paring the elements of the input lists + type: list +""" + +from jinja2.exceptions import UndefinedError + +from ansible.errors import AnsibleError, AnsibleUndefinedVariable +from ansible.plugins.lookup import LookupBase +from ansible.utils.listify import listify_lookup_plugin_terms + + +class LookupModule(LookupBase): + + def _lookup_variables(self, terms, variables): + results = [] + for x in terms: + try: + intermediate = listify_lookup_plugin_terms(x, templar=self._templar, fail_on_undefined=True) + except UndefinedError as e: + raise AnsibleUndefinedVariable("One of the nested variables was undefined. The error was: %s" % e) + results.append(intermediate) + return results + + def run(self, terms, variables=None, **kwargs): + + terms = self._lookup_variables(terms, variables) + + my_list = terms[:] + my_list.reverse() + result = [] + if len(my_list) == 0: + raise AnsibleError("with_nested requires at least one element in the nested list") + result = my_list.pop() + while len(my_list) > 0: + result2 = self._combine(result, my_list.pop()) + result = result2 + new_result = [] + for x in result: + new_result.append(self._flatten(x)) + return new_result diff --git a/lib/ansible/plugins/lookup/password.py b/lib/ansible/plugins/lookup/password.py new file mode 100644 index 0000000..06ea8b3 --- /dev/null +++ b/lib/ansible/plugins/lookup/password.py @@ -0,0 +1,389 @@ +# (c) 2012, Daniel Hokka Zakrisson <daniel@hozac.com> +# (c) 2013, Javier Candeira <javier@candeira.com> +# (c) 2013, Maykel Moya <mmoya@speedyrails.com> +# (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = """ + name: password + version_added: "1.1" + author: + - Daniel Hokka Zakrisson (!UNKNOWN) <daniel@hozac.com> + - Javier Candeira (!UNKNOWN) <javier@candeira.com> + - Maykel Moya (!UNKNOWN) <mmoya@speedyrails.com> + short_description: retrieve or generate a random password, stored in a file + description: + - Generates a random plaintext password and stores it in a file at a given filepath. + - If the file exists previously, it will retrieve its contents, behaving just like with_file. + - 'Usage of variables like C("{{ inventory_hostname }}") in the filepath can be used to set up random passwords per host, + which simplifies password management in C("host_vars") variables.' + - A special case is using /dev/null as a path. The password lookup will generate a new random password each time, + but will not write it to /dev/null. This can be used when you need a password without storing it on the controller. + options: + _terms: + description: + - path to the file that stores/will store the passwords + required: True + encrypt: + description: + - Which hash scheme to encrypt the returning password, should be one hash scheme from C(passlib.hash; md5_crypt, bcrypt, sha256_crypt, sha512_crypt). + - If not provided, the password will be returned in plain text. + - Note that the password is always stored as plain text, only the returning password is encrypted. + - Encrypt also forces saving the salt value for idempotence. + - Note that before 2.6 this option was incorrectly labeled as a boolean for a long time. + ident: + description: + - Specify version of Bcrypt algorithm to be used while using C(encrypt) as C(bcrypt). + - The parameter is only available for C(bcrypt) - U(https://passlib.readthedocs.io/en/stable/lib/passlib.hash.bcrypt.html#passlib.hash.bcrypt). + - Other hash types will simply ignore this parameter. + - 'Valid values for this parameter are: C(2), C(2a), C(2y), C(2b).' + type: string + version_added: "2.12" + chars: + version_added: "1.4" + description: + - A list of names that compose a custom character set in the generated passwords. + - 'By default generated passwords contain a random mix of upper and lowercase ASCII letters, the numbers 0-9, and punctuation (". , : - _").' + - "They can be either parts of Python's string module attributes or represented literally ( :, -)." + - "Though string modules can vary by Python version, valid values for both major releases include: + 'ascii_lowercase', 'ascii_uppercase', 'digits', 'hexdigits', 'octdigits', 'printable', 'punctuation' and 'whitespace'." + - Be aware that Python's 'hexdigits' includes lower and upper case versions of a-f, so it is not a good choice as it doubles + the chances of those values for systems that won't distinguish case, distorting the expected entropy. + - "when using a comma separated string, to enter comma use two commas ',,' somewhere - preferably at the end. + Quotes and double quotes are not supported." + type: list + elements: str + default: ['ascii_letters', 'digits', ".,:-_"] + length: + description: The length of the generated password. + default: 20 + type: integer + seed: + version_added: "2.12" + description: + - A seed to initialize the random number generator. + - Identical seeds will yield identical passwords. + - Use this for random-but-idempotent password generation. + type: str + notes: + - A great alternative to the password lookup plugin, + if you don't need to generate random passwords on a per-host basis, + would be to use Vault in playbooks. + Read the documentation there and consider using it first, + it will be more desirable for most applications. + - If the file already exists, no data will be written to it. + If the file has contents, those contents will be read in as the password. + Empty files cause the password to return as an empty string. + - 'As all lookups, this runs on the Ansible host as the user running the playbook, and "become" does not apply, + the target file must be readable by the playbook user, or, if it does not exist, + the playbook user must have sufficient privileges to create it. + (So, for example, attempts to write into areas such as /etc will fail unless the entire playbook is being run as root).' +""" + +EXAMPLES = """ +- name: create a mysql user with a random password + community.mysql.mysql_user: + name: "{{ client }}" + password: "{{ lookup('ansible.builtin.password', 'credentials/' + client + '/' + tier + '/' + role + '/mysqlpassword length=15') }}" + priv: "{{ client }}_{{ tier }}_{{ role }}.*:ALL" + +- name: create a mysql user with a random password using only ascii letters + community.mysql.mysql_user: + name: "{{ client }}" + password: "{{ lookup('ansible.builtin.password', '/tmp/passwordfile chars=ascii_letters') }}" + priv: '{{ client }}_{{ tier }}_{{ role }}.*:ALL' + +- name: create a mysql user with an 8 character random password using only digits + community.mysql.mysql_user: + name: "{{ client }}" + password: "{{ lookup('ansible.builtin.password', '/tmp/passwordfile length=8 chars=digits') }}" + priv: "{{ client }}_{{ tier }}_{{ role }}.*:ALL" + +- name: create a mysql user with a random password using many different char sets + community.mysql.mysql_user: + name: "{{ client }}" + password: "{{ lookup('ansible.builtin.password', '/tmp/passwordfile chars=ascii_letters,digits,punctuation') }}" + priv: "{{ client }}_{{ tier }}_{{ role }}.*:ALL" + +- name: create lowercase 8 character name for Kubernetes pod name + ansible.builtin.set_fact: + random_pod_name: "web-{{ lookup('ansible.builtin.password', '/dev/null chars=ascii_lowercase,digits length=8') }}" + +- name: create random but idempotent password + ansible.builtin.set_fact: + password: "{{ lookup('ansible.builtin.password', '/dev/null', seed=inventory_hostname) }}" +""" + +RETURN = """ +_raw: + description: + - a password + type: list + elements: str +""" + +import os +import string +import time +import hashlib + +from ansible.errors import AnsibleError, AnsibleAssertionError +from ansible.module_utils._text import to_bytes, to_native, to_text +from ansible.module_utils.six import string_types +from ansible.parsing.splitter import parse_kv +from ansible.plugins.lookup import LookupBase +from ansible.utils.encrypt import BaseHash, do_encrypt, random_password, random_salt +from ansible.utils.path import makedirs_safe + + +VALID_PARAMS = frozenset(('length', 'encrypt', 'chars', 'ident', 'seed')) + + +def _read_password_file(b_path): + """Read the contents of a password file and return it + :arg b_path: A byte string containing the path to the password file + :returns: a text string containing the contents of the password file or + None if no password file was present. + """ + content = None + + if os.path.exists(b_path): + with open(b_path, 'rb') as f: + b_content = f.read().rstrip() + content = to_text(b_content, errors='surrogate_or_strict') + + return content + + +def _gen_candidate_chars(characters): + '''Generate a string containing all valid chars as defined by ``characters`` + + :arg characters: A list of character specs. The character specs are + shorthand names for sets of characters like 'digits', 'ascii_letters', + or 'punctuation' or a string to be included verbatim. + + The values of each char spec can be: + + * a name of an attribute in the 'strings' module ('digits' for example). + The value of the attribute will be added to the candidate chars. + * a string of characters. If the string isn't an attribute in 'string' + module, the string will be directly added to the candidate chars. + + For example:: + + characters=['digits', '?|']`` + + will match ``string.digits`` and add all ascii digits. ``'?|'`` will add + the question mark and pipe characters directly. Return will be the string:: + + u'0123456789?|' + ''' + chars = [] + for chars_spec in characters: + # getattr from string expands things like "ascii_letters" and "digits" + # into a set of characters. + chars.append(to_text(getattr(string, to_native(chars_spec), chars_spec), errors='strict')) + chars = u''.join(chars).replace(u'"', u'').replace(u"'", u'') + return chars + + +def _parse_content(content): + '''parse our password data format into password and salt + + :arg content: The data read from the file + :returns: password and salt + ''' + password = content + salt = None + + salt_slug = u' salt=' + try: + sep = content.rindex(salt_slug) + except ValueError: + # No salt + pass + else: + salt = password[sep + len(salt_slug):] + password = content[:sep] + + return password, salt + + +def _format_content(password, salt, encrypt=None, ident=None): + """Format the password and salt for saving + :arg password: the plaintext password to save + :arg salt: the salt to use when encrypting a password + :arg encrypt: Which method the user requests that this password is encrypted. + Note that the password is saved in clear. Encrypt just tells us if we + must save the salt value for idempotence. Defaults to None. + :arg ident: Which version of BCrypt algorithm to be used. + Valid only if value of encrypt is bcrypt. + Defaults to None. + :returns: a text string containing the formatted information + + .. warning:: Passwords are saved in clear. This is because the playbooks + expect to get cleartext passwords from this lookup. + """ + if not encrypt and not salt: + return password + + # At this point, the calling code should have assured us that there is a salt value. + if not salt: + raise AnsibleAssertionError('_format_content was called with encryption requested but no salt value') + + if ident: + return u'%s salt=%s ident=%s' % (password, salt, ident) + return u'%s salt=%s' % (password, salt) + + +def _write_password_file(b_path, content): + b_pathdir = os.path.dirname(b_path) + makedirs_safe(b_pathdir, mode=0o700) + + with open(b_path, 'wb') as f: + os.chmod(b_path, 0o600) + b_content = to_bytes(content, errors='surrogate_or_strict') + b'\n' + f.write(b_content) + + +def _get_lock(b_path): + """Get the lock for writing password file.""" + first_process = False + b_pathdir = os.path.dirname(b_path) + lockfile_name = to_bytes("%s.ansible_lockfile" % hashlib.sha1(b_path).hexdigest()) + lockfile = os.path.join(b_pathdir, lockfile_name) + if not os.path.exists(lockfile) and b_path != to_bytes('/dev/null'): + try: + makedirs_safe(b_pathdir, mode=0o700) + fd = os.open(lockfile, os.O_CREAT | os.O_EXCL) + os.close(fd) + first_process = True + except OSError as e: + if e.strerror != 'File exists': + raise + + counter = 0 + # if the lock is got by other process, wait until it's released + while os.path.exists(lockfile) and not first_process: + time.sleep(2 ** counter) + if counter >= 2: + raise AnsibleError("Password lookup cannot get the lock in 7 seconds, abort..." + "This may caused by un-removed lockfile" + "you can manually remove it from controller machine at %s and try again" % lockfile) + counter += 1 + return first_process, lockfile + + +def _release_lock(lockfile): + """Release the lock so other processes can read the password file.""" + if os.path.exists(lockfile): + os.remove(lockfile) + + +class LookupModule(LookupBase): + + def _parse_parameters(self, term): + """Hacky parsing of params + + See https://github.com/ansible/ansible-modules-core/issues/1968#issuecomment-136842156 + and the first_found lookup For how we want to fix this later + """ + first_split = term.split(' ', 1) + if len(first_split) <= 1: + # Only a single argument given, therefore it's a path + relpath = term + params = dict() + else: + relpath = first_split[0] + params = parse_kv(first_split[1]) + if '_raw_params' in params: + # Spaces in the path? + relpath = u' '.join((relpath, params['_raw_params'])) + del params['_raw_params'] + + # Check that we parsed the params correctly + if not term.startswith(relpath): + # Likely, the user had a non parameter following a parameter. + # Reject this as a user typo + raise AnsibleError('Unrecognized value after key=value parameters given to password lookup') + # No _raw_params means we already found the complete path when + # we split it initially + + # Check for invalid parameters. Probably a user typo + invalid_params = frozenset(params.keys()).difference(VALID_PARAMS) + if invalid_params: + raise AnsibleError('Unrecognized parameter(s) given to password lookup: %s' % ', '.join(invalid_params)) + + # Set defaults + params['length'] = int(params.get('length', self.get_option('length'))) + params['encrypt'] = params.get('encrypt', self.get_option('encrypt')) + params['ident'] = params.get('ident', self.get_option('ident')) + params['seed'] = params.get('seed', self.get_option('seed')) + + params['chars'] = params.get('chars', self.get_option('chars')) + if params['chars'] and isinstance(params['chars'], string_types): + tmp_chars = [] + if u',,' in params['chars']: + tmp_chars.append(u',') + tmp_chars.extend(c for c in params['chars'].replace(u',,', u',').split(u',') if c) + params['chars'] = tmp_chars + + return relpath, params + + def run(self, terms, variables, **kwargs): + ret = [] + + self.set_options(var_options=variables, direct=kwargs) + + for term in terms: + relpath, params = self._parse_parameters(term) + path = self._loader.path_dwim(relpath) + b_path = to_bytes(path, errors='surrogate_or_strict') + chars = _gen_candidate_chars(params['chars']) + + changed = None + # make sure only one process finishes all the job first + first_process, lockfile = _get_lock(b_path) + + content = _read_password_file(b_path) + + if content is None or b_path == to_bytes('/dev/null'): + plaintext_password = random_password(params['length'], chars, params['seed']) + salt = None + changed = True + else: + plaintext_password, salt = _parse_content(content) + + encrypt = params['encrypt'] + if encrypt and not salt: + changed = True + try: + salt = random_salt(BaseHash.algorithms[encrypt].salt_size) + except KeyError: + salt = random_salt() + + ident = params['ident'] + if encrypt and not ident: + changed = True + try: + ident = BaseHash.algorithms[encrypt].implicit_ident + except KeyError: + ident = None + + if changed and b_path != to_bytes('/dev/null'): + content = _format_content(plaintext_password, salt, encrypt=encrypt, ident=ident) + _write_password_file(b_path, content) + + if first_process: + # let other processes continue + _release_lock(lockfile) + + if encrypt: + password = do_encrypt(plaintext_password, encrypt, salt=salt, ident=ident) + ret.append(password) + else: + ret.append(plaintext_password) + + return ret diff --git a/lib/ansible/plugins/lookup/pipe.py b/lib/ansible/plugins/lookup/pipe.py new file mode 100644 index 0000000..54df3fc --- /dev/null +++ b/lib/ansible/plugins/lookup/pipe.py @@ -0,0 +1,76 @@ +# (c) 2012, Daniel Hokka Zakrisson <daniel@hozac.com> +# (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = r""" + name: pipe + author: Daniel Hokka Zakrisson (!UNKNOWN) <daniel@hozac.com> + version_added: "0.9" + short_description: read output from a command + description: + - Run a command and return the output. + options: + _terms: + description: command(s) to run. + required: True + notes: + - Like all lookups this runs on the Ansible controller and is unaffected by other keywords, such as become, + so if you need to different permissions you must change the command or run Ansible as another user. + - Alternatively you can use a shell/command task that runs against localhost and registers the result. + - Pipe lookup internally invokes Popen with shell=True (this is required and intentional). + This type of invocation is considered a security issue if appropriate care is not taken to sanitize any user provided or variable input. + It is strongly recommended to pass user input or variable input via quote filter before using with pipe lookup. + See example section for this. + Read more about this L(Bandit B602 docs,https://bandit.readthedocs.io/en/latest/plugins/b602_subprocess_popen_with_shell_equals_true.html) +""" + +EXAMPLES = r""" +- name: raw result of running date command + ansible.builtin.debug: + msg: "{{ lookup('ansible.builtin.pipe', 'date') }}" + +- name: Always use quote filter to make sure your variables are safe to use with shell + ansible.builtin.debug: + msg: "{{ lookup('ansible.builtin.pipe', 'getent passwd ' + myuser | quote ) }}" +""" + +RETURN = r""" + _string: + description: + - stdout from command + type: list + elements: str +""" + +import subprocess + +from ansible.errors import AnsibleError +from ansible.plugins.lookup import LookupBase + + +class LookupModule(LookupBase): + + def run(self, terms, variables, **kwargs): + + ret = [] + for term in terms: + ''' + https://docs.python.org/3/library/subprocess.html#popen-constructor + + The shell argument (which defaults to False) specifies whether to use the + shell as the program to execute. If shell is True, it is recommended to pass + args as a string rather than as a sequence + + https://github.com/ansible/ansible/issues/6550 + ''' + term = str(term) + + p = subprocess.Popen(term, cwd=self._loader.get_basedir(), shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE) + (stdout, stderr) = p.communicate() + if p.returncode == 0: + ret.append(stdout.decode("utf-8").rstrip()) + else: + raise AnsibleError("lookup_plugin.pipe(%s) returned %d" % (term, p.returncode)) + return ret diff --git a/lib/ansible/plugins/lookup/random_choice.py b/lib/ansible/plugins/lookup/random_choice.py new file mode 100644 index 0000000..9f8a6ae --- /dev/null +++ b/lib/ansible/plugins/lookup/random_choice.py @@ -0,0 +1,53 @@ +# (c) 2013, Michael DeHaan <michael.dehaan@gmail.com> +# (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = """ + name: random_choice + author: Michael DeHaan + version_added: "1.1" + short_description: return random element from list + description: + - The 'random_choice' feature can be used to pick something at random. While it's not a load balancer (there are modules for those), + it can somewhat be used as a poor man's load balancer in a MacGyver like situation. + - At a more basic level, they can be used to add chaos and excitement to otherwise predictable automation environments. +""" + +EXAMPLES = """ +- name: Magic 8 ball for MUDs + ansible.builtin.debug: + msg: "{{ item }}" + with_random_choice: + - "go through the door" + - "drink from the goblet" + - "press the red button" + - "do nothing" +""" + +RETURN = """ + _raw: + description: + - random item + type: raw +""" +import random + +from ansible.errors import AnsibleError +from ansible.module_utils._text import to_native +from ansible.plugins.lookup import LookupBase + + +class LookupModule(LookupBase): + + def run(self, terms, inject=None, **kwargs): + + ret = terms + if terms: + try: + ret = [random.choice(terms)] + except Exception as e: + raise AnsibleError("Unable to choose random term: %s" % to_native(e)) + + return ret diff --git a/lib/ansible/plugins/lookup/sequence.py b/lib/ansible/plugins/lookup/sequence.py new file mode 100644 index 0000000..8a000c5 --- /dev/null +++ b/lib/ansible/plugins/lookup/sequence.py @@ -0,0 +1,268 @@ +# (c) 2013, Jayson Vantuyl <jayson@aggressive.ly> +# (c) 2012-17 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = """ + name: sequence + author: Jayson Vantuyl (!UNKNOWN) <jayson@aggressive.ly> + version_added: "1.0" + short_description: generate a list based on a number sequence + description: + - generates a sequence of items. You can specify a start value, an end value, an optional "stride" value that specifies the number of steps + to increment the sequence, and an optional printf-style format string. + - 'Arguments can be specified as key=value pair strings or as a shortcut form of the arguments string is also accepted: [start-]end[/stride][:format].' + - 'Numerical values can be specified in decimal, hexadecimal (0x3f8) or octal (0600).' + - Starting at version 1.9.2, negative strides are allowed. + - Generated items are strings. Use Jinja2 filters to convert items to preferred type, e.g. C({{ 1 + item|int }}). + - See also Jinja2 C(range) filter as an alternative. + options: + start: + description: number at which to start the sequence + default: 0 + type: integer + end: + description: number at which to end the sequence, dont use this with count + type: integer + default: 0 + count: + description: number of elements in the sequence, this is not to be used with end + type: integer + default: 0 + stride: + description: increments between sequence numbers, the default is 1 unless the end is less than the start, then it is -1. + type: integer + format: + description: return a string with the generated number formatted in +""" + +EXAMPLES = """ +- name: create some test users + ansible.builtin.user: + name: "{{ item }}" + state: present + groups: "evens" + with_sequence: start=0 end=32 format=testuser%02x + +- name: create a series of directories with even numbers for some reason + ansible.builtin.file: + dest: "/var/stuff/{{ item }}" + state: directory + with_sequence: start=4 end=16 stride=2 + +- name: a simpler way to use the sequence plugin create 4 groups + ansible.builtin.group: + name: "group{{ item }}" + state: present + with_sequence: count=4 + +- name: the final countdown + ansible.builtin.debug: + msg: "{{item}} seconds to detonation" + with_sequence: start=10 end=0 stride=-1 + +- name: Use of variable + ansible.builtin.debug: + msg: "{{ item }}" + with_sequence: start=1 end="{{ end_at }}" + vars: + - end_at: 10 +""" + +RETURN = """ + _list: + description: + - A list containing generated sequence of items + type: list + elements: str +""" + +from re import compile as re_compile, IGNORECASE + +from ansible.errors import AnsibleError +from ansible.parsing.splitter import parse_kv +from ansible.plugins.lookup import LookupBase + + +# shortcut format +NUM = "(0?x?[0-9a-f]+)" +SHORTCUT = re_compile( + "^(" + # Group 0 + NUM + # Group 1: Start + "-)?" + + NUM + # Group 2: End + "(/" + # Group 3 + NUM + # Group 4: Stride + ")?" + + "(:(.+))?$", # Group 5, Group 6: Format String + IGNORECASE +) + + +class LookupModule(LookupBase): + """ + sequence lookup module + + Used to generate some sequence of items. Takes arguments in two forms. + + The simple / shortcut form is: + + [start-]end[/stride][:format] + + As indicated by the brackets: start, stride, and format string are all + optional. The format string is in the style of printf. This can be used + to pad with zeros, format in hexadecimal, etc. All of the numerical values + can be specified in octal (i.e. 0664) or hexadecimal (i.e. 0x3f8). + Negative numbers are not supported. + + Some examples: + + 5 -> ["1","2","3","4","5"] + 5-8 -> ["5", "6", "7", "8"] + 2-10/2 -> ["2", "4", "6", "8", "10"] + 4:host%02d -> ["host01","host02","host03","host04"] + + The standard Ansible key-value form is accepted as well. For example: + + start=5 end=11 stride=2 format=0x%02x -> ["0x05","0x07","0x09","0x0a"] + + This format takes an alternate form of "end" called "count", which counts + some number from the starting value. For example: + + count=5 -> ["1", "2", "3", "4", "5"] + start=0x0f00 count=4 format=%04x -> ["0f00", "0f01", "0f02", "0f03"] + start=0 count=5 stride=2 -> ["0", "2", "4", "6", "8"] + start=1 count=5 stride=2 -> ["1", "3", "5", "7", "9"] + + The count option is mostly useful for avoiding off-by-one errors and errors + calculating the number of entries in a sequence when a stride is specified. + """ + + def reset(self): + """set sensible defaults""" + self.start = 1 + self.count = None + self.end = None + self.stride = 1 + self.format = "%d" + + def parse_kv_args(self, args): + """parse key-value style arguments""" + for arg in ["start", "end", "count", "stride"]: + try: + arg_raw = args.pop(arg, None) + if arg_raw is None: + continue + arg_cooked = int(arg_raw, 0) + setattr(self, arg, arg_cooked) + except ValueError: + raise AnsibleError( + "can't parse %s=%s as integer" + % (arg, arg_raw) + ) + if 'format' in args: + self.format = args.pop("format") + if args: + raise AnsibleError( + "unrecognized arguments to with_sequence: %s" + % list(args.keys()) + ) + + def parse_simple_args(self, term): + """parse the shortcut forms, return True/False""" + match = SHORTCUT.match(term) + if not match: + return False + + _, start, end, _, stride, _, format = match.groups() + + if start is not None: + try: + start = int(start, 0) + except ValueError: + raise AnsibleError("can't parse start=%s as integer" % start) + if end is not None: + try: + end = int(end, 0) + except ValueError: + raise AnsibleError("can't parse end=%s as integer" % end) + if stride is not None: + try: + stride = int(stride, 0) + except ValueError: + raise AnsibleError("can't parse stride=%s as integer" % stride) + + if start is not None: + self.start = start + if end is not None: + self.end = end + if stride is not None: + self.stride = stride + if format is not None: + self.format = format + + return True + + def sanity_check(self): + if self.count is None and self.end is None: + raise AnsibleError("must specify count or end in with_sequence") + elif self.count is not None and self.end is not None: + raise AnsibleError("can't specify both count and end in with_sequence") + elif self.count is not None: + # convert count to end + if self.count != 0: + self.end = self.start + self.count * self.stride - 1 + else: + self.start = 0 + self.end = 0 + self.stride = 0 + del self.count + if self.stride > 0 and self.end < self.start: + raise AnsibleError("to count backwards make stride negative") + if self.stride < 0 and self.end > self.start: + raise AnsibleError("to count forward don't make stride negative") + if self.format.count('%') != 1: + raise AnsibleError("bad formatting string: %s" % self.format) + + def generate_sequence(self): + if self.stride >= 0: + adjust = 1 + else: + adjust = -1 + numbers = range(self.start, self.end + adjust, self.stride) + + for i in numbers: + try: + formatted = self.format % i + yield formatted + except (ValueError, TypeError): + raise AnsibleError( + "problem formatting %r with %r" % (i, self.format) + ) + + def run(self, terms, variables, **kwargs): + results = [] + + for term in terms: + try: + self.reset() # clear out things for this iteration + try: + if not self.parse_simple_args(term): + self.parse_kv_args(parse_kv(term)) + except AnsibleError: + raise + except Exception as e: + raise AnsibleError("unknown error parsing with_sequence arguments: %r. Error was: %s" % (term, e)) + + self.sanity_check() + if self.stride != 0: + results.extend(self.generate_sequence()) + except AnsibleError: + raise + except Exception as e: + raise AnsibleError( + "unknown error generating sequence: %s" % e + ) + + return results diff --git a/lib/ansible/plugins/lookup/subelements.py b/lib/ansible/plugins/lookup/subelements.py new file mode 100644 index 0000000..9b1af8b --- /dev/null +++ b/lib/ansible/plugins/lookup/subelements.py @@ -0,0 +1,169 @@ +# (c) 2013, Serge van Ginderachter <serge@vanginderachter.be> +# (c) 2012-17 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = """ + name: subelements + author: Serge van Ginderachter (!UNKNOWN) <serge@vanginderachter.be> + version_added: "1.4" + short_description: traverse nested key from a list of dictionaries + description: + - Subelements walks a list of hashes (aka dictionaries) and then traverses a list with a given (nested sub-)key inside of those records. + options: + _terms: + description: tuple of list of dictionaries and dictionary key to extract + required: True + skip_missing: + default: False + description: + - Lookup accepts this flag from a dictionary as optional. See Example section for more information. + - If set to C(True), the lookup plugin will skip the lists items that do not contain the given subkey. + - If set to C(False), the plugin will yield an error and complain about the missing subkey. +""" + +EXAMPLES = """ +- name: show var structure as it is needed for example to make sense + hosts: all + vars: + users: + - name: alice + authorized: + - /tmp/alice/onekey.pub + - /tmp/alice/twokey.pub + mysql: + password: mysql-password + hosts: + - "%" + - "127.0.0.1" + - "::1" + - "localhost" + privs: + - "*.*:SELECT" + - "DB1.*:ALL" + groups: + - wheel + - name: bob + authorized: + - /tmp/bob/id_rsa.pub + mysql: + password: other-mysql-password + hosts: + - "db1" + privs: + - "*.*:SELECT" + - "DB2.*:ALL" + tasks: + - name: Set authorized ssh key, extracting just that data from 'users' + ansible.posix.authorized_key: + user: "{{ item.0.name }}" + key: "{{ lookup('file', item.1) }}" + with_subelements: + - "{{ users }}" + - authorized + + - name: Setup MySQL users, given the mysql hosts and privs subkey lists + community.mysql.mysql_user: + name: "{{ item.0.name }}" + password: "{{ item.0.mysql.password }}" + host: "{{ item.1 }}" + priv: "{{ item.0.mysql.privs | join('/') }}" + with_subelements: + - "{{ users }}" + - mysql.hosts + + - name: list groups for users that have them, don't error if groups key is missing + ansible.builtin.debug: var=item + loop: "{{ q('ansible.builtin.subelements', users, 'groups', {'skip_missing': True}) }}" +""" + +RETURN = """ +_list: + description: list of subelements extracted +""" + +from ansible.errors import AnsibleError +from ansible.module_utils.six import string_types +from ansible.module_utils.parsing.convert_bool import boolean +from ansible.plugins.lookup import LookupBase +from ansible.utils.listify import listify_lookup_plugin_terms + + +FLAGS = ('skip_missing',) + + +class LookupModule(LookupBase): + + def run(self, terms, variables, **kwargs): + + def _raise_terms_error(msg=""): + raise AnsibleError( + "subelements lookup expects a list of two or three items, " + msg) + + terms[0] = listify_lookup_plugin_terms(terms[0], templar=self._templar) + + # check lookup terms - check number of terms + if not isinstance(terms, list) or not 2 <= len(terms) <= 3: + _raise_terms_error() + + # first term should be a list (or dict), second a string holding the subkey + if not isinstance(terms[0], (list, dict)) or not isinstance(terms[1], string_types): + _raise_terms_error("first a dict or a list, second a string pointing to the subkey") + subelements = terms[1].split(".") + + if isinstance(terms[0], dict): # convert to list: + if terms[0].get('skipped', False) is not False: + # the registered result was completely skipped + return [] + elementlist = [] + for key in terms[0]: + elementlist.append(terms[0][key]) + else: + elementlist = terms[0] + + # check for optional flags in third term + flags = {} + if len(terms) == 3: + flags = terms[2] + if not isinstance(flags, dict) and not all(isinstance(key, string_types) and key in FLAGS for key in flags): + _raise_terms_error("the optional third item must be a dict with flags %s" % FLAGS) + + # build_items + ret = [] + for item0 in elementlist: + if not isinstance(item0, dict): + raise AnsibleError("subelements lookup expects a dictionary, got '%s'" % item0) + if item0.get('skipped', False) is not False: + # this particular item is to be skipped + continue + + skip_missing = boolean(flags.get('skip_missing', False), strict=False) + subvalue = item0 + lastsubkey = False + sublist = [] + for subkey in subelements: + if subkey == subelements[-1]: + lastsubkey = True + if subkey not in subvalue: + if skip_missing: + continue + else: + raise AnsibleError("could not find '%s' key in iterated item '%s'" % (subkey, subvalue)) + if not lastsubkey: + if not isinstance(subvalue[subkey], dict): + if skip_missing: + continue + else: + raise AnsibleError("the key %s should point to a dictionary, got '%s'" % (subkey, subvalue[subkey])) + else: + subvalue = subvalue[subkey] + else: # lastsubkey + if not isinstance(subvalue[subkey], list): + raise AnsibleError("the key %s should point to a list, got '%s'" % (subkey, subvalue[subkey])) + else: + sublist = subvalue.pop(subkey, []) + for item1 in sublist: + ret.append((item0, item1)) + + return ret diff --git a/lib/ansible/plugins/lookup/template.py b/lib/ansible/plugins/lookup/template.py new file mode 100644 index 0000000..9c575b5 --- /dev/null +++ b/lib/ansible/plugins/lookup/template.py @@ -0,0 +1,165 @@ +# Copyright: (c) 2012, Michael DeHaan <michael.dehaan@gmail.com> +# Copyright: (c) 2012-17, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = """ + name: template + author: Michael DeHaan + version_added: "0.9" + short_description: retrieve contents of file after templating with Jinja2 + description: + - Returns a list of strings; for each template in the list of templates you pass in, returns a string containing the results of processing that template. + options: + _terms: + description: list of files to template + convert_data: + type: bool + description: + - Whether to convert YAML into data. If False, strings that are YAML will be left untouched. + - Mutually exclusive with the jinja2_native option. + default: true + variable_start_string: + description: The string marking the beginning of a print statement. + default: '{{' + version_added: '2.8' + type: str + variable_end_string: + description: The string marking the end of a print statement. + default: '}}' + version_added: '2.8' + type: str + jinja2_native: + description: + - Controls whether to use Jinja2 native types. + - It is off by default even if global jinja2_native is True. + - Has no effect if global jinja2_native is False. + - This offers more flexibility than the template module which does not use Jinja2 native types at all. + - Mutually exclusive with the convert_data option. + default: False + version_added: '2.11' + type: bool + template_vars: + description: A dictionary, the keys become additional variables available for templating. + default: {} + version_added: '2.3' + type: dict + comment_start_string: + description: The string marking the beginning of a comment statement. + version_added: '2.12' + type: str + comment_end_string: + description: The string marking the end of a comment statement. + version_added: '2.12' + type: str +""" + +EXAMPLES = """ +- name: show templating results + ansible.builtin.debug: + msg: "{{ lookup('ansible.builtin.template', './some_template.j2') }}" + +- name: show templating results with different variable start and end string + ansible.builtin.debug: + msg: "{{ lookup('ansible.builtin.template', './some_template.j2', variable_start_string='[%', variable_end_string='%]') }}" + +- name: show templating results with different comment start and end string + ansible.builtin.debug: + msg: "{{ lookup('ansible.builtin.template', './some_template.j2', comment_start_string='[#', comment_end_string='#]') }}" +""" + +RETURN = """ +_raw: + description: file(s) content after templating + type: list + elements: raw +""" + +from copy import deepcopy +import os + +import ansible.constants as C + +from ansible.errors import AnsibleError +from ansible.plugins.lookup import LookupBase +from ansible.module_utils._text import to_bytes, to_text +from ansible.template import generate_ansible_template_vars, AnsibleEnvironment +from ansible.utils.display import Display +from ansible.utils.native_jinja import NativeJinjaText + + +display = Display() + + +class LookupModule(LookupBase): + + def run(self, terms, variables, **kwargs): + + ret = [] + + self.set_options(var_options=variables, direct=kwargs) + + # capture options + convert_data_p = self.get_option('convert_data') + lookup_template_vars = self.get_option('template_vars') + jinja2_native = self.get_option('jinja2_native') and C.DEFAULT_JINJA2_NATIVE + variable_start_string = self.get_option('variable_start_string') + variable_end_string = self.get_option('variable_end_string') + comment_start_string = self.get_option('comment_start_string') + comment_end_string = self.get_option('comment_end_string') + + if jinja2_native: + templar = self._templar + else: + templar = self._templar.copy_with_new_env(environment_class=AnsibleEnvironment) + + for term in terms: + display.debug("File lookup term: %s" % term) + + lookupfile = self.find_file_in_search_path(variables, 'templates', term) + display.vvvv("File lookup using %s as file" % lookupfile) + if lookupfile: + b_template_data, show_data = self._loader._get_file_contents(lookupfile) + template_data = to_text(b_template_data, errors='surrogate_or_strict') + + # set jinja2 internal search path for includes + searchpath = variables.get('ansible_search_path', []) + if searchpath: + # our search paths aren't actually the proper ones for jinja includes. + # We want to search into the 'templates' subdir of each search path in + # addition to our original search paths. + newsearchpath = [] + for p in searchpath: + newsearchpath.append(os.path.join(p, 'templates')) + newsearchpath.append(p) + searchpath = newsearchpath + searchpath.insert(0, os.path.dirname(lookupfile)) + + # The template will have access to all existing variables, + # plus some added by ansible (e.g., template_{path,mtime}), + # plus anything passed to the lookup with the template_vars= + # argument. + vars = deepcopy(variables) + vars.update(generate_ansible_template_vars(term, lookupfile)) + vars.update(lookup_template_vars) + + with templar.set_temporary_context(variable_start_string=variable_start_string, + variable_end_string=variable_end_string, + comment_start_string=comment_start_string, + comment_end_string=comment_end_string, + available_variables=vars, searchpath=searchpath): + res = templar.template(template_data, preserve_trailing_newlines=True, + convert_data=convert_data_p, escape_backslashes=False) + + if (C.DEFAULT_JINJA2_NATIVE and not jinja2_native) or not convert_data_p: + # jinja2_native is true globally but off for the lookup, we need this text + # not to be processed by literal_eval anywhere in Ansible + res = NativeJinjaText(res) + + ret.append(res) + else: + raise AnsibleError("the template file %s could not be found for the lookup" % term) + + return ret diff --git a/lib/ansible/plugins/lookup/together.py b/lib/ansible/plugins/lookup/together.py new file mode 100644 index 0000000..c990e06 --- /dev/null +++ b/lib/ansible/plugins/lookup/together.py @@ -0,0 +1,68 @@ +# (c) 2013, Bradley Young <young.bradley@gmail.com> +# (c) 2012-17 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = """ + name: together + author: Bradley Young (!UNKNOWN) <young.bradley@gmail.com> + version_added: '1.3' + short_description: merges lists into synchronized list + description: + - Creates a list with the iterated elements of the supplied lists + - "To clarify with an example, [ 'a', 'b' ] and [ 1, 2 ] turn into [ ('a',1), ('b', 2) ]" + - This is basically the same as the 'zip_longest' filter and Python function + - Any 'unbalanced' elements will be substituted with 'None' + options: + _terms: + description: list of lists to merge + required: True +""" + +EXAMPLES = """ +- name: item.0 returns from the 'a' list, item.1 returns from the '1' list + ansible.builtin.debug: + msg: "{{ item.0 }} and {{ item.1 }}" + with_together: + - ['a', 'b', 'c', 'd'] + - [1, 2, 3, 4] +""" + +RETURN = """ + _list: + description: synchronized list + type: list + elements: list +""" +import itertools + +from ansible.errors import AnsibleError +from ansible.plugins.lookup import LookupBase +from ansible.utils.listify import listify_lookup_plugin_terms + + +class LookupModule(LookupBase): + """ + Transpose a list of arrays: + [1, 2, 3], [4, 5, 6] -> [1, 4], [2, 5], [3, 6] + Replace any empty spots in 2nd array with None: + [1, 2], [3] -> [1, 3], [2, None] + """ + + def _lookup_variables(self, terms): + results = [] + for x in terms: + intermediate = listify_lookup_plugin_terms(x, templar=self._templar) + results.append(intermediate) + return results + + def run(self, terms, variables=None, **kwargs): + + terms = self._lookup_variables(terms) + + my_list = terms[:] + if len(my_list) == 0: + raise AnsibleError("with_together requires at least one element in each list") + + return [self._flatten(x) for x in itertools.zip_longest(*my_list, fillvalue=None)] diff --git a/lib/ansible/plugins/lookup/unvault.py b/lib/ansible/plugins/lookup/unvault.py new file mode 100644 index 0000000..a9b7168 --- /dev/null +++ b/lib/ansible/plugins/lookup/unvault.py @@ -0,0 +1,63 @@ +# (c) 2020 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = """ + name: unvault + author: Ansible Core Team + version_added: "2.10" + short_description: read vaulted file(s) contents + description: + - This lookup returns the contents from vaulted (or not) file(s) on the Ansible controller's file system. + options: + _terms: + description: path(s) of files to read + required: True + notes: + - This lookup does not understand 'globbing' nor shell environment variables. +""" + +EXAMPLES = """ +- ansible.builtin.debug: msg="the value of foo.txt is {{ lookup('ansible.builtin.unvault', '/etc/foo.txt') | string | trim }}" +""" + +RETURN = """ + _raw: + description: + - content of file(s) as bytes + type: list + elements: raw +""" + +from ansible.errors import AnsibleParserError +from ansible.plugins.lookup import LookupBase +from ansible.module_utils._text import to_text +from ansible.utils.display import Display + +display = Display() + + +class LookupModule(LookupBase): + + def run(self, terms, variables=None, **kwargs): + + ret = [] + + self.set_options(var_options=variables, direct=kwargs) + + for term in terms: + display.debug("Unvault lookup term: %s" % term) + + # Find the file in the expected search path + lookupfile = self.find_file_in_search_path(variables, 'files', term) + display.vvvv(u"Unvault lookup found %s" % lookupfile) + if lookupfile: + actual_file = self._loader.get_real_file(lookupfile, decrypt=True) + with open(actual_file, 'rb') as f: + b_contents = f.read() + ret.append(to_text(b_contents)) + else: + raise AnsibleParserError('Unable to find file matching "%s" ' % term) + + return ret diff --git a/lib/ansible/plugins/lookup/url.py b/lib/ansible/plugins/lookup/url.py new file mode 100644 index 0000000..6790e1c --- /dev/null +++ b/lib/ansible/plugins/lookup/url.py @@ -0,0 +1,264 @@ +# (c) 2015, Brian Coca <bcoca@ansible.com> +# (c) 2012-17 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = """ +name: url +author: Brian Coca (@bcoca) +version_added: "1.9" +short_description: return contents from URL +description: + - Returns the content of the URL requested to be used as data in play. +options: + _terms: + description: urls to query + validate_certs: + description: Flag to control SSL certificate validation + type: boolean + default: True + split_lines: + description: Flag to control if content is returned as a list of lines or as a single text blob + type: boolean + default: True + use_proxy: + description: Flag to control if the lookup will observe HTTP proxy environment variables when present. + type: boolean + default: True + username: + description: Username to use for HTTP authentication. + type: string + version_added: "2.8" + password: + description: Password to use for HTTP authentication. + type: string + version_added: "2.8" + headers: + description: HTTP request headers + type: dictionary + default: {} + version_added: "2.9" + force: + description: Whether or not to set "cache-control" header with value "no-cache" + type: boolean + version_added: "2.10" + default: False + vars: + - name: ansible_lookup_url_force + env: + - name: ANSIBLE_LOOKUP_URL_FORCE + ini: + - section: url_lookup + key: force + timeout: + description: How long to wait for the server to send data before giving up + type: float + version_added: "2.10" + default: 10 + vars: + - name: ansible_lookup_url_timeout + env: + - name: ANSIBLE_LOOKUP_URL_TIMEOUT + ini: + - section: url_lookup + key: timeout + http_agent: + description: User-Agent to use in the request. The default was changed in 2.11 to C(ansible-httpget). + type: string + version_added: "2.10" + default: ansible-httpget + vars: + - name: ansible_lookup_url_agent + env: + - name: ANSIBLE_LOOKUP_URL_AGENT + ini: + - section: url_lookup + key: agent + force_basic_auth: + description: Force basic authentication + type: boolean + version_added: "2.10" + default: False + vars: + - name: ansible_lookup_url_agent + env: + - name: ANSIBLE_LOOKUP_URL_AGENT + ini: + - section: url_lookup + key: agent + follow_redirects: + description: String of urllib2, all/yes, safe, none to determine how redirects are followed, see RedirectHandlerFactory for more information + type: string + version_added: "2.10" + default: 'urllib2' + vars: + - name: ansible_lookup_url_follow_redirects + env: + - name: ANSIBLE_LOOKUP_URL_FOLLOW_REDIRECTS + ini: + - section: url_lookup + key: follow_redirects + use_gssapi: + description: + - Use GSSAPI handler of requests + - As of Ansible 2.11, GSSAPI credentials can be specified with I(username) and I(password). + type: boolean + version_added: "2.10" + default: False + vars: + - name: ansible_lookup_url_use_gssapi + env: + - name: ANSIBLE_LOOKUP_URL_USE_GSSAPI + ini: + - section: url_lookup + key: use_gssapi + use_netrc: + description: + - Determining whether to use credentials from ``~/.netrc`` file + - By default .netrc is used with Basic authentication headers + - When set to False, .netrc credentials are ignored + type: boolean + version_added: "2.14" + default: True + vars: + - name: ansible_lookup_url_use_netrc + env: + - name: ANSIBLE_LOOKUP_URL_USE_NETRC + ini: + - section: url_lookup + key: use_netrc + unix_socket: + description: String of file system path to unix socket file to use when establishing connection to the provided url + type: string + version_added: "2.10" + vars: + - name: ansible_lookup_url_unix_socket + env: + - name: ANSIBLE_LOOKUP_URL_UNIX_SOCKET + ini: + - section: url_lookup + key: unix_socket + ca_path: + description: String of file system path to CA cert bundle to use + type: string + version_added: "2.10" + vars: + - name: ansible_lookup_url_ca_path + env: + - name: ANSIBLE_LOOKUP_URL_CA_PATH + ini: + - section: url_lookup + key: ca_path + unredirected_headers: + description: A list of headers to not attach on a redirected request + type: list + elements: string + version_added: "2.10" + vars: + - name: ansible_lookup_url_unredir_headers + env: + - name: ANSIBLE_LOOKUP_URL_UNREDIR_HEADERS + ini: + - section: url_lookup + key: unredirected_headers + ciphers: + description: + - SSL/TLS Ciphers to use for the request + - 'When a list is provided, all ciphers are joined in order with C(:)' + - See the L(OpenSSL Cipher List Format,https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html#CIPHER-LIST-FORMAT) + for more details. + - The available ciphers is dependent on the Python and OpenSSL/LibreSSL versions + type: list + elements: string + version_added: '2.14' + vars: + - name: ansible_lookup_url_ciphers + env: + - name: ANSIBLE_LOOKUP_URL_CIPHERS + ini: + - section: url_lookup + key: ciphers +""" + +EXAMPLES = """ +- name: url lookup splits lines by default + ansible.builtin.debug: msg="{{item}}" + loop: "{{ lookup('ansible.builtin.url', 'https://github.com/gremlin.keys', wantlist=True) }}" + +- name: display ip ranges + ansible.builtin.debug: msg="{{ lookup('ansible.builtin.url', 'https://ip-ranges.amazonaws.com/ip-ranges.json', split_lines=False) }}" + +- name: url lookup using authentication + ansible.builtin.debug: msg="{{ lookup('ansible.builtin.url', 'https://some.private.site.com/file.txt', username='bob', password='hunter2') }}" + +- name: url lookup using basic authentication + ansible.builtin.debug: + msg: "{{ lookup('ansible.builtin.url', 'https://some.private.site.com/file.txt', username='bob', password='hunter2', force_basic_auth='True') }}" + +- name: url lookup using headers + ansible.builtin.debug: + msg: "{{ lookup('ansible.builtin.url', 'https://some.private.site.com/api/service', headers={'header1':'value1', 'header2':'value2'} ) }}" +""" + +RETURN = """ + _list: + description: list of list of lines or content of url(s) + type: list + elements: str +""" + +from urllib.error import HTTPError, URLError + +from ansible.errors import AnsibleError +from ansible.module_utils._text import to_text, to_native +from ansible.module_utils.urls import open_url, ConnectionError, SSLValidationError +from ansible.plugins.lookup import LookupBase +from ansible.utils.display import Display + +display = Display() + + +class LookupModule(LookupBase): + + def run(self, terms, variables=None, **kwargs): + + self.set_options(var_options=variables, direct=kwargs) + + ret = [] + for term in terms: + display.vvvv("url lookup connecting to %s" % term) + try: + response = open_url( + term, validate_certs=self.get_option('validate_certs'), + use_proxy=self.get_option('use_proxy'), + url_username=self.get_option('username'), + url_password=self.get_option('password'), + headers=self.get_option('headers'), + force=self.get_option('force'), + timeout=self.get_option('timeout'), + http_agent=self.get_option('http_agent'), + force_basic_auth=self.get_option('force_basic_auth'), + follow_redirects=self.get_option('follow_redirects'), + use_gssapi=self.get_option('use_gssapi'), + unix_socket=self.get_option('unix_socket'), + ca_path=self.get_option('ca_path'), + unredirected_headers=self.get_option('unredirected_headers'), + ciphers=self.get_option('ciphers'), + use_netrc=self.get_option('use_netrc') + ) + except HTTPError as e: + raise AnsibleError("Received HTTP error for %s : %s" % (term, to_native(e))) + except URLError as e: + raise AnsibleError("Failed lookup url for %s : %s" % (term, to_native(e))) + except SSLValidationError as e: + raise AnsibleError("Error validating the server's certificate for %s: %s" % (term, to_native(e))) + except ConnectionError as e: + raise AnsibleError("Error connecting to %s: %s" % (term, to_native(e))) + + if self.get_option('split_lines'): + for line in response.read().splitlines(): + ret.append(to_text(line)) + else: + ret.append(to_text(response.read())) + return ret diff --git a/lib/ansible/plugins/lookup/varnames.py b/lib/ansible/plugins/lookup/varnames.py new file mode 100644 index 0000000..442b81b --- /dev/null +++ b/lib/ansible/plugins/lookup/varnames.py @@ -0,0 +1,79 @@ +# (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = """ + name: varnames + author: Ansible Core Team + version_added: "2.8" + short_description: Lookup matching variable names + description: + - Retrieves a list of matching Ansible variable names. + options: + _terms: + description: List of Python regex patterns to search for in variable names. + required: True +""" + +EXAMPLES = """ +- name: List variables that start with qz_ + ansible.builtin.debug: msg="{{ lookup('ansible.builtin.varnames', '^qz_.+')}}" + vars: + qz_1: hello + qz_2: world + qa_1: "I won't show" + qz_: "I won't show either" + +- name: Show all variables + ansible.builtin.debug: msg="{{ lookup('ansible.builtin.varnames', '.+')}}" + +- name: Show variables with 'hosts' in their names + ansible.builtin.debug: msg="{{ lookup('ansible.builtin.varnames', 'hosts')}}" + +- name: Find several related variables that end specific way + ansible.builtin.debug: msg="{{ lookup('ansible.builtin.varnames', '.+_zone$', '.+_location$') }}" + +""" + +RETURN = """ +_value: + description: + - List of the variable names requested. + type: list +""" + +import re + +from ansible.errors import AnsibleError +from ansible.module_utils._text import to_native +from ansible.module_utils.six import string_types +from ansible.plugins.lookup import LookupBase + + +class LookupModule(LookupBase): + + def run(self, terms, variables=None, **kwargs): + + if variables is None: + raise AnsibleError('No variables available to search') + + self.set_options(var_options=variables, direct=kwargs) + + ret = [] + variable_names = list(variables.keys()) + for term in terms: + + if not isinstance(term, string_types): + raise AnsibleError('Invalid setting identifier, "%s" is not a string, it is a %s' % (term, type(term))) + + try: + name = re.compile(term) + except Exception as e: + raise AnsibleError('Unable to use "%s" as a search parameter: %s' % (term, to_native(e))) + + for varname in variable_names: + if name.search(varname): + ret.append(varname) + + return ret diff --git a/lib/ansible/plugins/lookup/vars.py b/lib/ansible/plugins/lookup/vars.py new file mode 100644 index 0000000..dd5f763 --- /dev/null +++ b/lib/ansible/plugins/lookup/vars.py @@ -0,0 +1,106 @@ +# (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = """ + name: vars + author: Ansible Core Team + version_added: "2.5" + short_description: Lookup templated value of variables + description: + - 'Retrieves the value of an Ansible variable. Note: Only returns top level variable names.' + options: + _terms: + description: The variable names to look up. + required: True + default: + description: + - What to return if a variable is undefined. + - If no default is set, it will result in an error if any of the variables is undefined. +""" + +EXAMPLES = """ +- name: Show value of 'variablename' + ansible.builtin.debug: msg="{{ lookup('ansible.builtin.vars', 'variabl' + myvar) }}" + vars: + variablename: hello + myvar: ename + +- name: Show default empty since i dont have 'variablnotename' + ansible.builtin.debug: msg="{{ lookup('ansible.builtin.vars', 'variabl' + myvar, default='')}}" + vars: + variablename: hello + myvar: notename + +- name: Produce an error since i dont have 'variablnotename' + ansible.builtin.debug: msg="{{ lookup('ansible.builtin.vars', 'variabl' + myvar)}}" + ignore_errors: True + vars: + variablename: hello + myvar: notename + +- name: find several related variables + ansible.builtin.debug: msg="{{ lookup('ansible.builtin.vars', 'ansible_play_hosts', 'ansible_play_batch', 'ansible_play_hosts_all') }}" + +- name: Access nested variables + ansible.builtin.debug: msg="{{ lookup('ansible.builtin.vars', 'variabl' + myvar).sub_var }}" + ignore_errors: True + vars: + variablename: + sub_var: 12 + myvar: ename + +- name: alternate way to find some 'prefixed vars' in loop + ansible.builtin.debug: msg="{{ lookup('ansible.builtin.vars', 'ansible_play_' + item) }}" + loop: + - hosts + - batch + - hosts_all +""" + +RETURN = """ +_value: + description: + - value of the variables requested. + type: list + elements: raw +""" + +from ansible.errors import AnsibleError, AnsibleUndefinedVariable +from ansible.module_utils.six import string_types +from ansible.plugins.lookup import LookupBase + + +class LookupModule(LookupBase): + + def run(self, terms, variables=None, **kwargs): + if variables is not None: + self._templar.available_variables = variables + myvars = getattr(self._templar, '_available_variables', {}) + + self.set_options(var_options=variables, direct=kwargs) + default = self.get_option('default') + + ret = [] + for term in terms: + if not isinstance(term, string_types): + raise AnsibleError('Invalid setting identifier, "%s" is not a string, its a %s' % (term, type(term))) + + try: + try: + value = myvars[term] + except KeyError: + try: + value = myvars['hostvars'][myvars['inventory_hostname']][term] + except KeyError: + raise AnsibleUndefinedVariable('No variable found with this name: %s' % term) + + ret.append(self._templar.template(value, fail_on_undefined=True)) + except AnsibleUndefinedVariable: + if default is not None: + ret.append(default) + else: + raise + + return ret diff --git a/lib/ansible/plugins/netconf/__init__.py b/lib/ansible/plugins/netconf/__init__.py new file mode 100644 index 0000000..e99efbd --- /dev/null +++ b/lib/ansible/plugins/netconf/__init__.py @@ -0,0 +1,375 @@ +# +# (c) 2017 Red Hat Inc. +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. +# +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from abc import abstractmethod +from functools import wraps + +from ansible.errors import AnsibleError +from ansible.plugins import AnsiblePlugin +from ansible.module_utils._text import to_native +from ansible.module_utils.basic import missing_required_lib + +try: + from ncclient.operations import RPCError + from ncclient.xml_ import to_xml, to_ele, NCElement + HAS_NCCLIENT = True + NCCLIENT_IMP_ERR = None +# paramiko and gssapi are incompatible and raise AttributeError not ImportError +# When running in FIPS mode, cryptography raises InternalError +# https://bugzilla.redhat.com/show_bug.cgi?id=1778939 +except Exception as err: + HAS_NCCLIENT = False + NCCLIENT_IMP_ERR = err + +try: + from lxml.etree import Element, SubElement, tostring, fromstring +except ImportError: + from xml.etree.ElementTree import Element, SubElement, tostring, fromstring + + +def ensure_ncclient(func): + @wraps(func) + def wrapped(self, *args, **kwargs): + if not HAS_NCCLIENT: + raise AnsibleError("%s: %s" % (missing_required_lib('ncclient'), to_native(NCCLIENT_IMP_ERR))) + return func(self, *args, **kwargs) + return wrapped + + +class NetconfBase(AnsiblePlugin): + """ + A base class for implementing Netconf connections + + .. note:: Unlike most of Ansible, nearly all strings in + :class:`TerminalBase` plugins are byte strings. This is because of + how close to the underlying platform these plugins operate. Remember + to mark literal strings as byte string (``b"string"``) and to use + :func:`~ansible.module_utils._text.to_bytes` and + :func:`~ansible.module_utils._text.to_text` to avoid unexpected + problems. + + List of supported rpc's: + :get: Retrieves running configuration and device state information + :get_config: Retrieves the specified configuration from the device + :edit_config: Loads the specified commands into the remote device + :commit: Load configuration from candidate to running + :discard_changes: Discard changes to candidate datastore + :validate: Validate the contents of the specified configuration. + :lock: Allows the client to lock the configuration system of a device. + :unlock: Release a configuration lock, previously obtained with the lock operation. + :copy_config: create or replace an entire configuration datastore with the contents of another complete + configuration datastore. + :get-schema: Retrieves the required schema from the device + :get_capabilities: Retrieves device information and supported rpc methods + + For JUNOS: + :execute_rpc: RPC to be execute on remote device + :load_configuration: Loads given configuration on device + + Note: rpc support depends on the capabilities of remote device. + + :returns: Returns output received from remote device as byte string + Note: the 'result' or 'error' from response should to be converted to object + of ElementTree using 'fromstring' to parse output as xml doc + + 'get_capabilities()' returns 'result' as a json string. + + Usage: + from ansible.module_utils.connection import Connection + + conn = Connection() + data = conn.execute_rpc(rpc) + reply = fromstring(reply) + + data = conn.get_capabilities() + json.loads(data) + + conn.load_configuration(config=[''set system ntp server 1.1.1.1''], action='set', format='text') + """ + + __rpc__ = ['rpc', 'get_config', 'get', 'edit_config', 'validate', 'copy_config', 'dispatch', 'lock', 'unlock', + 'discard_changes', 'commit', 'get_schema', 'delete_config', 'get_device_operations'] + + def __init__(self, connection): + super(NetconfBase, self).__init__() + self._connection = connection + + @property + def m(self): + return self._connection.manager + + def rpc(self, name): + """ + RPC to be execute on remote device + :param name: Name of rpc in string format + :return: Received rpc response from remote host + """ + try: + obj = to_ele(name) + resp = self.m.rpc(obj) + return resp.data_xml if hasattr(resp, 'data_xml') else resp.xml + except RPCError as exc: + msg = exc.xml + raise Exception(to_xml(msg)) + + def get_config(self, source=None, filter=None): + """ + Retrieve all or part of a specified configuration + (by default entire configuration is retrieved). + :param source: Name of the configuration datastore being queried, defaults to running datastore + :param filter: This argument specifies the portion of the configuration data to retrieve + :return: Returns xml string containing the RPC response received from remote host + """ + if isinstance(filter, list): + filter = tuple(filter) + + if not source: + source = 'running' + resp = self.m.get_config(source=source, filter=filter) + return resp.data_xml if hasattr(resp, 'data_xml') else resp.xml + + def get(self, filter=None, with_defaults=None): + """ + Retrieve device configuration and state information. + :param filter: This argument specifies the portion of the state data to retrieve + (by default entire state data is retrieved) + :param with_defaults: defines an explicit method of retrieving default values + from the configuration + :return: Returns xml string containing the RPC response received from remote host + """ + if isinstance(filter, list): + filter = tuple(filter) + resp = self.m.get(filter=filter, with_defaults=with_defaults) + response = resp.data_xml if hasattr(resp, 'data_xml') else resp.xml + return response + + def edit_config(self, config=None, format='xml', target='candidate', default_operation=None, test_option=None, error_option=None): + """ + Loads all or part of the specified *config* to the *target* configuration datastore. + :param config: Is the configuration, which must be rooted in the `config` element. + It can be specified either as a string or an :class:`~xml.etree.ElementTree.Element`. + :param format: The format of configuration eg. xml, text + :param target: Is the name of the configuration datastore being edited + :param default_operation: If specified must be one of { `"merge"`, `"replace"`, or `"none"` } + :param test_option: If specified must be one of { `"test_then_set"`, `"set"` } + :param error_option: If specified must be one of { `"stop-on-error"`, `"continue-on-error"`, `"rollback-on-error"` } + The `"rollback-on-error"` *error_option* depends on the `:rollback-on-error` capability. + :return: Returns xml string containing the RPC response received from remote host + """ + if config is None: + raise ValueError('config value must be provided') + resp = self.m.edit_config(config, format=format, target=target, default_operation=default_operation, test_option=test_option, + error_option=error_option) + return resp.data_xml if hasattr(resp, 'data_xml') else resp.xml + + def validate(self, source='candidate'): + """ + Validate the contents of the specified configuration. + :param source: Is the name of the configuration datastore being validated or `config` element + containing the configuration subtree to be validated + :return: Returns xml string containing the RPC response received from remote host + """ + resp = self.m.validate(source=source) + return resp.data_xml if hasattr(resp, 'data_xml') else resp.xml + + def copy_config(self, source, target): + """ + Create or replace an entire configuration datastore with the contents of another complete configuration datastore. + :param source: Is the name of the configuration datastore to use as the source of the copy operation or `config` + element containing the configuration subtree to copy + :param target: Is the name of the configuration datastore to use as the destination of the copy operation + :return: Returns xml string containing the RPC response received from remote host + """ + resp = self.m.copy_config(source, target) + return resp.data_xml if hasattr(resp, 'data_xml') else resp.xml + + def dispatch(self, rpc_command=None, source=None, filter=None): + """ + Execute rpc on the remote device eg. dispatch('clear-arp-table') + :param rpc_command: specifies rpc command to be dispatched either in plain text or in xml element format (depending on command) + :param source: name of the configuration datastore being queried + :param filter: specifies the portion of the configuration to retrieve (by default entire configuration is retrieved) + :return: Returns xml string containing the RPC response received from remote host + """ + if rpc_command is None: + raise ValueError('rpc_command value must be provided') + + resp = self.m.dispatch(fromstring(rpc_command), source=source, filter=filter) + + if isinstance(resp, NCElement): + # In case xml reply is transformed or namespace is removed in + # ncclient device specific handler return modified xml response + result = resp.data_xml + elif hasattr(resp, 'data_ele') and resp.data_ele: + # if data node is present in xml response return the xml string + # with data node as root + result = resp.data_xml + else: + # return raw xml string received from host with rpc-reply as the root node + result = resp.xml + + return result + + def lock(self, target="candidate"): + """ + Allows the client to lock the configuration system of a device. + :param target: is the name of the configuration datastore to lock, + defaults to candidate datastore + :return: Returns xml string containing the RPC response received from remote host + """ + resp = self.m.lock(target=target) + return resp.data_xml if hasattr(resp, 'data_xml') else resp.xml + + def unlock(self, target="candidate"): + """ + Release a configuration lock, previously obtained with the lock operation. + :param target: is the name of the configuration datastore to unlock, + defaults to candidate datastore + :return: Returns xml string containing the RPC response received from remote host + """ + resp = self.m.unlock(target=target) + return resp.data_xml if hasattr(resp, 'data_xml') else resp.xml + + def discard_changes(self): + """ + Revert the candidate configuration to the currently running configuration. + Any uncommitted changes are discarded. + :return: Returns xml string containing the RPC response received from remote host + """ + resp = self.m.discard_changes() + return resp.data_xml if hasattr(resp, 'data_xml') else resp.xml + + def commit(self, confirmed=False, timeout=None, persist=None): + """ + Commit the candidate configuration as the device's new current configuration. + Depends on the `:candidate` capability. + A confirmed commit (i.e. if *confirmed* is `True`) is reverted if there is no + followup commit within the *timeout* interval. If no timeout is specified the + confirm timeout defaults to 600 seconds (10 minutes). + A confirming commit may have the *confirmed* parameter but this is not required. + Depends on the `:confirmed-commit` capability. + :param confirmed: whether this is a confirmed commit + :param timeout: specifies the confirm timeout in seconds + :param persist: make the confirmed commit survive a session termination, + and set a token on the ongoing confirmed commit + :return: Returns xml string containing the RPC response received from remote host + """ + resp = self.m.commit(confirmed=confirmed, timeout=timeout, persist=persist) + return resp.data_xml if hasattr(resp, 'data_xml') else resp.xml + + def get_schema(self, identifier=None, version=None, format=None): + """ + Retrieve a named schema, with optional revision and type. + :param identifier: name of the schema to be retrieved + :param version: version of schema to get + :param format: format of the schema to be retrieved, yang is the default + :return: Returns xml string containing the RPC response received from remote host + """ + resp = self.m.get_schema(identifier, version=version, format=format) + return resp.data_xml if hasattr(resp, 'data_xml') else resp.xml + + def delete_config(self, target): + """ + delete a configuration datastore + :param target: specifies the name or URL of configuration datastore to delete + :return: Returns xml string containing the RPC response received from remote host + """ + resp = self.m.delete_config(target) + return resp.data_xml if hasattr(resp, 'data_xml') else resp.xml + + def locked(self, target): + return self.m.locked(target) + + @abstractmethod + def get_capabilities(self): + """ + Retrieves device information and supported + rpc methods by device platform and return result + as a string + :return: Netconf session capability + """ + pass + + @staticmethod + def guess_network_os(obj): + """ + Identifies the operating system of network device. + :param obj: ncclient manager connection instance + :return: The name of network operating system. + """ + pass + + def get_base_rpc(self): + """ + Returns list of base rpc method supported by remote device + :return: List of RPC supported + """ + return self.__rpc__ + + def put_file(self, source, destination): + """ + Copies file to remote host + :param source: Source location of file + :param destination: Destination file path + :return: Returns xml string containing the RPC response received from remote host + """ + pass + + def fetch_file(self, source, destination): + """ + Fetch file from remote host + :param source: Source location of file + :param destination: Source location of file + :return: Returns xml string containing the RPC response received from remote host + """ + pass + + def get_device_operations(self, server_capabilities): + """ + Retrieve remote host capability from Netconf server hello message. + :param server_capabilities: Server capabilities received during Netconf session initialization + :return: Remote host capabilities in dictionary format + """ + operations = {} + capabilities = '\n'.join(server_capabilities) + operations['supports_commit'] = ':candidate' in capabilities + operations['supports_defaults'] = ':with-defaults' in capabilities + operations['supports_confirm_commit'] = ':confirmed-commit' in capabilities + operations['supports_startup'] = ':startup' in capabilities + operations['supports_xpath'] = ':xpath' in capabilities + operations['supports_writable_running'] = ':writable-running' in capabilities + operations['supports_validate'] = ':validate' in capabilities + + operations['lock_datastore'] = [] + if operations['supports_writable_running']: + operations['lock_datastore'].append('running') + + if operations['supports_commit']: + operations['lock_datastore'].append('candidate') + + if operations['supports_startup']: + operations['lock_datastore'].append('startup') + + operations['supports_lock'] = bool(operations['lock_datastore']) + + return operations + +# TODO Restore .xml, when ncclient supports it for all platforms diff --git a/lib/ansible/plugins/shell/__init__.py b/lib/ansible/plugins/shell/__init__.py new file mode 100644 index 0000000..d5db261 --- /dev/null +++ b/lib/ansible/plugins/shell/__init__.py @@ -0,0 +1,239 @@ +# (c) 2016 RedHat +# +# This file is part of Ansible. +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import os +import os.path +import random +import re +import shlex +import time + +from ansible.errors import AnsibleError +from ansible.module_utils._text import to_native +from ansible.module_utils.six import text_type, string_types +from ansible.module_utils.common._collections_compat import Mapping, Sequence +from ansible.plugins import AnsiblePlugin + +_USER_HOME_PATH_RE = re.compile(r'^~[_.A-Za-z0-9][-_.A-Za-z0-9]*$') + + +class ShellBase(AnsiblePlugin): + def __init__(self): + + super(ShellBase, self).__init__() + + self.env = {} + self.tmpdir = None + self.executable = None + + def _normalize_system_tmpdirs(self): + # Normalize the tmp directory strings. We don't use expanduser/expandvars because those + # can vary between remote user and become user. Therefore the safest practice will be for + # this to always be specified as full paths) + normalized_paths = [d.rstrip('/') for d in self.get_option('system_tmpdirs')] + + # Make sure all system_tmpdirs are absolute otherwise they'd be relative to the login dir + # which is almost certainly going to fail in a cornercase. + if not all(os.path.isabs(d) for d in normalized_paths): + raise AnsibleError('The configured system_tmpdirs contains a relative path: {0}. All' + ' system_tmpdirs must be absolute'.format(to_native(normalized_paths))) + + self.set_option('system_tmpdirs', normalized_paths) + + def set_options(self, task_keys=None, var_options=None, direct=None): + + super(ShellBase, self).set_options(task_keys=task_keys, var_options=var_options, direct=direct) + + # set env if needed, deal with environment's 'dual nature' list of dicts or dict + # TODO: config system should already resolve this so we should be able to just iterate over dicts + env = self.get_option('environment') + if isinstance(env, string_types): + raise AnsibleError('The "envirionment" keyword takes a list of dictionaries or a dictionary, not a string') + if not isinstance(env, Sequence): + env = [env] + for env_dict in env: + if not isinstance(env_dict, Mapping): + raise AnsibleError('The "envirionment" keyword takes a list of dictionaries (or single dictionary), but got a "%s" instead' % type(env_dict)) + self.env.update(env_dict) + + # We can remove the try: except in the future when we make ShellBase a proper subset of + # *all* shells. Right now powershell and third party shells which do not use the + # shell_common documentation fragment (and so do not have system_tmpdirs) will fail + try: + self._normalize_system_tmpdirs() + except KeyError: + pass + + @staticmethod + def _generate_temp_dir_name(): + return 'ansible-tmp-%s-%s-%s' % (time.time(), os.getpid(), random.randint(0, 2**48)) + + def env_prefix(self, **kwargs): + return ' '.join(['%s=%s' % (k, shlex.quote(text_type(v))) for k, v in kwargs.items()]) + + def join_path(self, *args): + return os.path.join(*args) + + # some shells (eg, powershell) are snooty about filenames/extensions, this lets the shell plugin have a say + def get_remote_filename(self, pathname): + base_name = os.path.basename(pathname.strip()) + return base_name.strip() + + def path_has_trailing_slash(self, path): + return path.endswith('/') + + def chmod(self, paths, mode): + cmd = ['chmod', mode] + cmd.extend(paths) + cmd = [shlex.quote(c) for c in cmd] + + return ' '.join(cmd) + + def chown(self, paths, user): + cmd = ['chown', user] + cmd.extend(paths) + cmd = [shlex.quote(c) for c in cmd] + + return ' '.join(cmd) + + def chgrp(self, paths, group): + cmd = ['chgrp', group] + cmd.extend(paths) + cmd = [shlex.quote(c) for c in cmd] + + return ' '.join(cmd) + + def set_user_facl(self, paths, user, mode): + """Only sets acls for users as that's really all we need""" + cmd = ['setfacl', '-m', 'u:%s:%s' % (user, mode)] + cmd.extend(paths) + cmd = [shlex.quote(c) for c in cmd] + + return ' '.join(cmd) + + def remove(self, path, recurse=False): + path = shlex.quote(path) + cmd = 'rm -f ' + if recurse: + cmd += '-r ' + return cmd + "%s %s" % (path, self._SHELL_REDIRECT_ALLNULL) + + def exists(self, path): + cmd = ['test', '-e', shlex.quote(path)] + return ' '.join(cmd) + + def mkdtemp(self, basefile=None, system=False, mode=0o700, tmpdir=None): + if not basefile: + basefile = self.__class__._generate_temp_dir_name() + + # When system is specified we have to create this in a directory where + # other users can read and access the tmp directory. + # This is because we use system to create tmp dirs for unprivileged users who are + # sudo'ing to a second unprivileged user. + # The 'system_tmpdirs' setting defines dirctories we can use for this purpose + # the default are, /tmp and /var/tmp. + # So we only allow one of those locations if system=True, using the + # passed in tmpdir if it is valid or the first one from the setting if not. + + if system: + if tmpdir: + tmpdir = tmpdir.rstrip('/') + + if tmpdir in self.get_option('system_tmpdirs'): + basetmpdir = tmpdir + else: + basetmpdir = self.get_option('system_tmpdirs')[0] + else: + if tmpdir is None: + basetmpdir = self.get_option('remote_tmp') + else: + basetmpdir = tmpdir + + basetmp = self.join_path(basetmpdir, basefile) + + # use mkdir -p to ensure parents exist, but mkdir fullpath to ensure last one is created by us + cmd = 'mkdir -p %s echo %s %s' % (self._SHELL_SUB_LEFT, basetmpdir, self._SHELL_SUB_RIGHT) + cmd += '%s mkdir %s echo %s %s' % (self._SHELL_AND, self._SHELL_SUB_LEFT, basetmp, self._SHELL_SUB_RIGHT) + cmd += ' %s echo %s=%s echo %s %s' % (self._SHELL_AND, basefile, self._SHELL_SUB_LEFT, basetmp, self._SHELL_SUB_RIGHT) + + # change the umask in a subshell to achieve the desired mode + # also for directories created with `mkdir -p` + if mode: + tmp_umask = 0o777 & ~mode + cmd = '%s umask %o %s %s %s' % (self._SHELL_GROUP_LEFT, tmp_umask, self._SHELL_AND, cmd, self._SHELL_GROUP_RIGHT) + + return cmd + + def expand_user(self, user_home_path, username=''): + ''' Return a command to expand tildes in a path + + It can be either "~" or "~username". We just ignore $HOME + We use the POSIX definition of a username: + http://pubs.opengroup.org/onlinepubs/000095399/basedefs/xbd_chap03.html#tag_03_426 + http://pubs.opengroup.org/onlinepubs/000095399/basedefs/xbd_chap03.html#tag_03_276 + + Falls back to 'current working directory' as we assume 'home is where the remote user ends up' + ''' + + # Check that the user_path to expand is safe + if user_home_path != '~': + if not _USER_HOME_PATH_RE.match(user_home_path): + # shlex.quote will make the shell return the string verbatim + user_home_path = shlex.quote(user_home_path) + elif username: + # if present the user name is appended to resolve "that user's home" + user_home_path += username + + return 'echo %s' % user_home_path + + def pwd(self): + """Return the working directory after connecting""" + return 'echo %spwd%s' % (self._SHELL_SUB_LEFT, self._SHELL_SUB_RIGHT) + + def build_module_command(self, env_string, shebang, cmd, arg_path=None): + # don't quote the cmd if it's an empty string, because this will break pipelining mode + if cmd.strip() != '': + cmd = shlex.quote(cmd) + + cmd_parts = [] + if shebang: + shebang = shebang.replace("#!", "").strip() + else: + shebang = "" + cmd_parts.extend([env_string.strip(), shebang, cmd]) + if arg_path is not None: + cmd_parts.append(arg_path) + new_cmd = " ".join(cmd_parts) + return new_cmd + + def append_command(self, cmd, cmd_to_append): + """Append an additional command if supported by the shell""" + + if self._SHELL_AND: + cmd += ' %s %s' % (self._SHELL_AND, cmd_to_append) + + return cmd + + def wrap_for_exec(self, cmd): + """wrap script execution with any necessary decoration (eg '&' for quoted powershell script paths)""" + return cmd + + def quote(self, cmd): + """Returns a shell-escaped string that can be safely used as one token in a shell command line""" + return shlex.quote(cmd) diff --git a/lib/ansible/plugins/shell/cmd.py b/lib/ansible/plugins/shell/cmd.py new file mode 100644 index 0000000..c1083dc --- /dev/null +++ b/lib/ansible/plugins/shell/cmd.py @@ -0,0 +1,57 @@ +# Copyright (c) 2019 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' +name: cmd +version_added: '2.8' +short_description: Windows Command Prompt +description: +- Used with the 'ssh' connection plugin and no C(DefaultShell) has been set on the Windows host. +extends_documentation_fragment: +- shell_windows +''' + +import re + +from ansible.plugins.shell.powershell import ShellModule as PSShellModule + +# these are the metachars that have a special meaning in cmd that we want to escape when quoting +_find_unsafe = re.compile(r'[\s\(\)\%\!^\"\<\>\&\|]').search + + +class ShellModule(PSShellModule): + + # Common shell filenames that this plugin handles + COMPATIBLE_SHELLS = frozenset() # type: frozenset[str] + # Family of shells this has. Must match the filename without extension + SHELL_FAMILY = 'cmd' + + _SHELL_REDIRECT_ALLNULL = '>nul 2>&1' + _SHELL_AND = '&&' + + # Used by various parts of Ansible to do Windows specific changes + _IS_WINDOWS = True + + def quote(self, s): + # cmd does not support single quotes that the shlex_quote uses. We need to override the quoting behaviour to + # better match cmd.exe. + # https://blogs.msdn.microsoft.com/twistylittlepassagesallalike/2011/04/23/everyone-quotes-command-line-arguments-the-wrong-way/ + + # Return an empty argument + if not s: + return '""' + + if _find_unsafe(s) is None: + return s + + # Escape the metachars as we are quoting the string to stop cmd from interpreting that metachar. For example + # 'file &whoami.exe' would result in 'file $(whoami.exe)' instead of the literal string + # https://stackoverflow.com/questions/3411771/multiple-character-replace-with-python + for c in '^()%!"<>&|': # '^' must be the first char that we scan and replace + if c in s: + # I can't find any docs that explicitly say this but to escape ", it needs to be prefixed with \^. + s = s.replace(c, ("\\^" if c == '"' else "^") + c) + + return '^"' + s + '^"' diff --git a/lib/ansible/plugins/shell/powershell.py b/lib/ansible/plugins/shell/powershell.py new file mode 100644 index 0000000..de5e705 --- /dev/null +++ b/lib/ansible/plugins/shell/powershell.py @@ -0,0 +1,287 @@ +# Copyright (c) 2014, Chris Church <chris@ninemoreminutes.com> +# Copyright (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' +name: powershell +version_added: historical +short_description: Windows PowerShell +description: +- The only option when using 'winrm' or 'psrp' as a connection plugin. +- Can also be used when using 'ssh' as a connection plugin and the C(DefaultShell) has been configured to PowerShell. +extends_documentation_fragment: +- shell_windows +''' + +import base64 +import os +import re +import shlex +import pkgutil +import xml.etree.ElementTree as ET +import ntpath + +from ansible.module_utils._text import to_bytes, to_text +from ansible.plugins.shell import ShellBase + + +_common_args = ['PowerShell', '-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Unrestricted'] + + +def _parse_clixml(data, stream="Error"): + """ + Takes a byte string like '#< CLIXML\r\n<Objs...' and extracts the stream + message encoded in the XML data. CLIXML is used by PowerShell to encode + multiple objects in stderr. + """ + lines = [] + + # There are some scenarios where the stderr contains a nested CLIXML element like + # '<# CLIXML\r\n<# CLIXML\r\n<Objs>...</Objs><Objs>...</Objs>'. + # Parse each individual <Objs> element and add the error strings to our stderr list. + # https://github.com/ansible/ansible/issues/69550 + while data: + end_idx = data.find(b"</Objs>") + 7 + current_element = data[data.find(b"<Objs "):end_idx] + data = data[end_idx:] + + clixml = ET.fromstring(current_element) + namespace_match = re.match(r'{(.*)}', clixml.tag) + namespace = "{%s}" % namespace_match.group(1) if namespace_match else "" + + strings = clixml.findall("./%sS" % namespace) + lines.extend([e.text.replace('_x000D__x000A_', '') for e in strings if e.attrib.get('S') == stream]) + + return to_bytes('\r\n'.join(lines)) + + +class ShellModule(ShellBase): + + # Common shell filenames that this plugin handles + # Powershell is handled differently. It's selected when winrm is the + # connection + COMPATIBLE_SHELLS = frozenset() # type: frozenset[str] + # Family of shells this has. Must match the filename without extension + SHELL_FAMILY = 'powershell' + + _SHELL_REDIRECT_ALLNULL = '> $null' + _SHELL_AND = ';' + + # Used by various parts of Ansible to do Windows specific changes + _IS_WINDOWS = True + + # TODO: add binary module support + + def env_prefix(self, **kwargs): + # powershell/winrm env handling is handled in the exec wrapper + return "" + + def join_path(self, *args): + # use normpath() to remove doubled slashed and convert forward to backslashes + parts = [ntpath.normpath(self._unquote(arg)) for arg in args] + + # Because ntpath.join treats any component that begins with a backslash as an absolute path, + # we have to strip slashes from at least the beginning, otherwise join will ignore all previous + # path components except for the drive. + return ntpath.join(parts[0], *[part.strip('\\') for part in parts[1:]]) + + def get_remote_filename(self, pathname): + # powershell requires that script files end with .ps1 + base_name = os.path.basename(pathname.strip()) + name, ext = os.path.splitext(base_name.strip()) + if ext.lower() not in ['.ps1', '.exe']: + return name + '.ps1' + + return base_name.strip() + + def path_has_trailing_slash(self, path): + # Allow Windows paths to be specified using either slash. + path = self._unquote(path) + return path.endswith('/') or path.endswith('\\') + + def chmod(self, paths, mode): + raise NotImplementedError('chmod is not implemented for Powershell') + + def chown(self, paths, user): + raise NotImplementedError('chown is not implemented for Powershell') + + def set_user_facl(self, paths, user, mode): + raise NotImplementedError('set_user_facl is not implemented for Powershell') + + def remove(self, path, recurse=False): + path = self._escape(self._unquote(path)) + if recurse: + return self._encode_script('''Remove-Item '%s' -Force -Recurse;''' % path) + else: + return self._encode_script('''Remove-Item '%s' -Force;''' % path) + + def mkdtemp(self, basefile=None, system=False, mode=None, tmpdir=None): + # Windows does not have an equivalent for the system temp files, so + # the param is ignored + if not basefile: + basefile = self.__class__._generate_temp_dir_name() + basefile = self._escape(self._unquote(basefile)) + basetmpdir = tmpdir if tmpdir else self.get_option('remote_tmp') + + script = ''' + $tmp_path = [System.Environment]::ExpandEnvironmentVariables('%s') + $tmp = New-Item -Type Directory -Path $tmp_path -Name '%s' + Write-Output -InputObject $tmp.FullName + ''' % (basetmpdir, basefile) + return self._encode_script(script.strip()) + + def expand_user(self, user_home_path, username=''): + # PowerShell only supports "~" (not "~username"). Resolve-Path ~ does + # not seem to work remotely, though by default we are always starting + # in the user's home directory. + user_home_path = self._unquote(user_home_path) + if user_home_path == '~': + script = 'Write-Output (Get-Location).Path' + elif user_home_path.startswith('~\\'): + script = "Write-Output ((Get-Location).Path + '%s')" % self._escape(user_home_path[1:]) + else: + script = "Write-Output '%s'" % self._escape(user_home_path) + return self._encode_script(script) + + def exists(self, path): + path = self._escape(self._unquote(path)) + script = ''' + If (Test-Path '%s') + { + $res = 0; + } + Else + { + $res = 1; + } + Write-Output '$res'; + Exit $res; + ''' % path + return self._encode_script(script) + + def checksum(self, path, *args, **kwargs): + path = self._escape(self._unquote(path)) + script = ''' + If (Test-Path -PathType Leaf '%(path)s') + { + $sp = new-object -TypeName System.Security.Cryptography.SHA1CryptoServiceProvider; + $fp = [System.IO.File]::Open('%(path)s', [System.IO.Filemode]::Open, [System.IO.FileAccess]::Read); + [System.BitConverter]::ToString($sp.ComputeHash($fp)).Replace("-", "").ToLower(); + $fp.Dispose(); + } + ElseIf (Test-Path -PathType Container '%(path)s') + { + Write-Output "3"; + } + Else + { + Write-Output "1"; + } + ''' % dict(path=path) + return self._encode_script(script) + + def build_module_command(self, env_string, shebang, cmd, arg_path=None): + bootstrap_wrapper = pkgutil.get_data("ansible.executor.powershell", "bootstrap_wrapper.ps1") + + # pipelining bypass + if cmd == '': + return self._encode_script(script=bootstrap_wrapper, strict_mode=False, preserve_rc=False) + + # non-pipelining + + cmd_parts = shlex.split(cmd, posix=False) + cmd_parts = list(map(to_text, cmd_parts)) + if shebang and shebang.lower() == '#!powershell': + if not self._unquote(cmd_parts[0]).lower().endswith('.ps1'): + # we're running a module via the bootstrap wrapper + cmd_parts[0] = '"%s.ps1"' % self._unquote(cmd_parts[0]) + wrapper_cmd = "type " + cmd_parts[0] + " | " + self._encode_script(script=bootstrap_wrapper, strict_mode=False, preserve_rc=False) + return wrapper_cmd + elif shebang and shebang.startswith('#!'): + cmd_parts.insert(0, shebang[2:]) + elif not shebang: + # The module is assumed to be a binary + cmd_parts[0] = self._unquote(cmd_parts[0]) + cmd_parts.append(arg_path) + script = ''' + Try + { + %s + %s + } + Catch + { + $_obj = @{ failed = $true } + If ($_.Exception.GetType) + { + $_obj.Add('msg', $_.Exception.Message) + } + Else + { + $_obj.Add('msg', $_.ToString()) + } + If ($_.InvocationInfo.PositionMessage) + { + $_obj.Add('exception', $_.InvocationInfo.PositionMessage) + } + ElseIf ($_.ScriptStackTrace) + { + $_obj.Add('exception', $_.ScriptStackTrace) + } + Try + { + $_obj.Add('error_record', ($_ | ConvertTo-Json | ConvertFrom-Json)) + } + Catch + { + } + Echo $_obj | ConvertTo-Json -Compress -Depth 99 + Exit 1 + } + ''' % (env_string, ' '.join(cmd_parts)) + return self._encode_script(script, preserve_rc=False) + + def wrap_for_exec(self, cmd): + return '& %s; exit $LASTEXITCODE' % cmd + + def _unquote(self, value): + '''Remove any matching quotes that wrap the given value.''' + value = to_text(value or '') + m = re.match(r'^\s*?\'(.*?)\'\s*?$', value) + if m: + return m.group(1) + m = re.match(r'^\s*?"(.*?)"\s*?$', value) + if m: + return m.group(1) + return value + + def _escape(self, value): + '''Return value escaped for use in PowerShell single quotes.''' + # There are 5 chars that need to be escaped in a single quote. + # https://github.com/PowerShell/PowerShell/blob/b7cb335f03fe2992d0cbd61699de9d9aafa1d7c1/src/System.Management.Automation/engine/parser/CharTraits.cs#L265-L272 + return re.compile(u"(['\u2018\u2019\u201a\u201b])").sub(u'\\1\\1', value) + + def _encode_script(self, script, as_list=False, strict_mode=True, preserve_rc=True): + '''Convert a PowerShell script to a single base64-encoded command.''' + script = to_text(script) + + if script == u'-': + cmd_parts = _common_args + ['-Command', '-'] + + else: + if strict_mode: + script = u'Set-StrictMode -Version Latest\r\n%s' % script + # try to propagate exit code if present- won't work with begin/process/end-style scripts (ala put_file) + # NB: the exit code returned may be incorrect in the case of a successful command followed by an invalid command + if preserve_rc: + script = u'%s\r\nIf (-not $?) { If (Get-Variable LASTEXITCODE -ErrorAction SilentlyContinue) { exit $LASTEXITCODE } Else { exit 1 } }\r\n'\ + % script + script = '\n'.join([x.strip() for x in script.splitlines() if x.strip()]) + encoded_script = to_text(base64.b64encode(script.encode('utf-16-le')), 'utf-8') + cmd_parts = _common_args + ['-EncodedCommand', encoded_script] + + if as_list: + return cmd_parts + return ' '.join(cmd_parts) diff --git a/lib/ansible/plugins/shell/sh.py b/lib/ansible/plugins/shell/sh.py new file mode 100644 index 0000000..146c466 --- /dev/null +++ b/lib/ansible/plugins/shell/sh.py @@ -0,0 +1,79 @@ +# Copyright (c) 2014, Chris Church <chris@ninemoreminutes.com> +# Copyright (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' +name: sh +short_description: "POSIX shell (/bin/sh)" +version_added: historical +description: + - This shell plugin is the one you want to use on most Unix systems, it is the most compatible and widely installed shell. +extends_documentation_fragment: + - shell_common +''' + +import shlex + +from ansible.plugins.shell import ShellBase + + +class ShellModule(ShellBase): + + # Common shell filenames that this plugin handles. + # Note: sh is the default shell plugin so this plugin may also be selected + # This code needs to be SH-compliant. BASH-isms will not work if /bin/sh points to a non-BASH shell. + + # if the filename is not listed in any Shell plugin. + COMPATIBLE_SHELLS = frozenset(('sh', 'zsh', 'bash', 'dash', 'ksh')) + # Family of shells this has. Must match the filename without extension + SHELL_FAMILY = 'sh' + + # commonly used + ECHO = 'echo' + COMMAND_SEP = ';' + + # How to end lines in a python script one-liner + _SHELL_EMBEDDED_PY_EOL = '\n' + _SHELL_REDIRECT_ALLNULL = '> /dev/null 2>&1' + _SHELL_AND = '&&' + _SHELL_OR = '||' + _SHELL_SUB_LEFT = '"`' + _SHELL_SUB_RIGHT = '`"' + _SHELL_GROUP_LEFT = '(' + _SHELL_GROUP_RIGHT = ')' + + def checksum(self, path, python_interp): + # In the following test, each condition is a check and logical + # comparison (|| or &&) that sets the rc value. Every check is run so + # the last check in the series to fail will be the rc that is returned. + # + # If a check fails we error before invoking the hash functions because + # hash functions may successfully take the hash of a directory on BSDs + # (UFS filesystem?) which is not what the rest of the ansible code expects + # + # If all of the available hashing methods fail we fail with an rc of 0. + # This logic is added to the end of the cmd at the bottom of this function. + + # Return codes: + # checksum: success! + # 0: Unknown error + # 1: Remote file does not exist + # 2: No read permissions on the file + # 3: File is a directory + # 4: No python interpreter + + # Quoting gets complex here. We're writing a python string that's + # used by a variety of shells on the remote host to invoke a python + # "one-liner". + shell_escaped_path = shlex.quote(path) + test = "rc=flag; [ -r %(p)s ] %(shell_or)s rc=2; [ -f %(p)s ] %(shell_or)s rc=1; [ -d %(p)s ] %(shell_and)s rc=3; %(i)s -V 2>/dev/null %(shell_or)s rc=4; [ x\"$rc\" != \"xflag\" ] %(shell_and)s echo \"${rc} \"%(p)s %(shell_and)s exit 0" % dict(p=shell_escaped_path, i=python_interp, shell_and=self._SHELL_AND, shell_or=self._SHELL_OR) # NOQA + csums = [ + u"({0} -c 'import hashlib; BLOCKSIZE = 65536; hasher = hashlib.sha1();{2}afile = open(\"'{1}'\", \"rb\"){2}buf = afile.read(BLOCKSIZE){2}while len(buf) > 0:{2}\thasher.update(buf){2}\tbuf = afile.read(BLOCKSIZE){2}afile.close(){2}print(hasher.hexdigest())' 2>/dev/null)".format(python_interp, shell_escaped_path, self._SHELL_EMBEDDED_PY_EOL), # NOQA Python > 2.4 (including python3) + u"({0} -c 'import sha; BLOCKSIZE = 65536; hasher = sha.sha();{2}afile = open(\"'{1}'\", \"rb\"){2}buf = afile.read(BLOCKSIZE){2}while len(buf) > 0:{2}\thasher.update(buf){2}\tbuf = afile.read(BLOCKSIZE){2}afile.close(){2}print(hasher.hexdigest())' 2>/dev/null)".format(python_interp, shell_escaped_path, self._SHELL_EMBEDDED_PY_EOL), # NOQA Python == 2.4 + ] + + cmd = (" %s " % self._SHELL_OR).join(csums) + cmd = "%s; %s %s (echo \'0 \'%s)" % (test, cmd, self._SHELL_OR, shell_escaped_path) + return cmd diff --git a/lib/ansible/plugins/strategy/__init__.py b/lib/ansible/plugins/strategy/__init__.py new file mode 100644 index 0000000..5cc05ee --- /dev/null +++ b/lib/ansible/plugins/strategy/__init__.py @@ -0,0 +1,1202 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import cmd +import functools +import os +import pprint +import queue +import sys +import threading +import time + +from collections import deque +from multiprocessing import Lock + +from jinja2.exceptions import UndefinedError + +from ansible import constants as C +from ansible import context +from ansible.errors import AnsibleError, AnsibleFileNotFound, AnsibleUndefinedVariable, AnsibleParserError +from ansible.executor import action_write_locks +from ansible.executor.play_iterator import IteratingStates +from ansible.executor.process.worker import WorkerProcess +from ansible.executor.task_result import TaskResult +from ansible.executor.task_queue_manager import CallbackSend, DisplaySend +from ansible.module_utils.six import string_types +from ansible.module_utils._text import to_text +from ansible.module_utils.connection import Connection, ConnectionError +from ansible.playbook.conditional import Conditional +from ansible.playbook.handler import Handler +from ansible.playbook.helpers import load_list_of_blocks +from ansible.playbook.task import Task +from ansible.playbook.task_include import TaskInclude +from ansible.plugins import loader as plugin_loader +from ansible.template import Templar +from ansible.utils.display import Display +from ansible.utils.fqcn import add_internal_fqcns +from ansible.utils.unsafe_proxy import wrap_var +from ansible.utils.vars import combine_vars, isidentifier +from ansible.vars.clean import strip_internal_keys, module_response_deepcopy + +display = Display() + +__all__ = ['StrategyBase'] + +# This list can be an exact match, or start of string bound +# does not accept regex +ALWAYS_DELEGATE_FACT_PREFIXES = frozenset(( + 'discovered_interpreter_', +)) + + +class StrategySentinel: + pass + + +_sentinel = StrategySentinel() + + +def post_process_whens(result, task, templar, task_vars): + cond = None + if task.changed_when: + with templar.set_temporary_context(available_variables=task_vars): + cond = Conditional(loader=templar._loader) + cond.when = task.changed_when + result['changed'] = cond.evaluate_conditional(templar, templar.available_variables) + + if task.failed_when: + with templar.set_temporary_context(available_variables=task_vars): + if cond is None: + cond = Conditional(loader=templar._loader) + cond.when = task.failed_when + failed_when_result = cond.evaluate_conditional(templar, templar.available_variables) + result['failed_when_result'] = result['failed'] = failed_when_result + + +def _get_item_vars(result, task): + item_vars = {} + if task.loop or task.loop_with: + loop_var = result.get('ansible_loop_var', 'item') + index_var = result.get('ansible_index_var') + if loop_var in result: + item_vars[loop_var] = result[loop_var] + if index_var and index_var in result: + item_vars[index_var] = result[index_var] + if '_ansible_item_label' in result: + item_vars['_ansible_item_label'] = result['_ansible_item_label'] + if 'ansible_loop' in result: + item_vars['ansible_loop'] = result['ansible_loop'] + return item_vars + + +def results_thread_main(strategy): + while True: + try: + result = strategy._final_q.get() + if isinstance(result, StrategySentinel): + break + elif isinstance(result, DisplaySend): + display.display(*result.args, **result.kwargs) + elif isinstance(result, CallbackSend): + for arg in result.args: + if isinstance(arg, TaskResult): + strategy.normalize_task_result(arg) + break + strategy._tqm.send_callback(result.method_name, *result.args, **result.kwargs) + elif isinstance(result, TaskResult): + strategy.normalize_task_result(result) + with strategy._results_lock: + strategy._results.append(result) + else: + display.warning('Received an invalid object (%s) in the result queue: %r' % (type(result), result)) + except (IOError, EOFError): + break + except queue.Empty: + pass + + +def debug_closure(func): + """Closure to wrap ``StrategyBase._process_pending_results`` and invoke the task debugger""" + @functools.wraps(func) + def inner(self, iterator, one_pass=False, max_passes=None): + status_to_stats_map = ( + ('is_failed', 'failures'), + ('is_unreachable', 'dark'), + ('is_changed', 'changed'), + ('is_skipped', 'skipped'), + ) + + # We don't know the host yet, copy the previous states, for lookup after we process new results + prev_host_states = iterator.host_states.copy() + + results = func(self, iterator, one_pass=one_pass, max_passes=max_passes) + _processed_results = [] + + for result in results: + task = result._task + host = result._host + _queued_task_args = self._queued_task_cache.pop((host.name, task._uuid), None) + task_vars = _queued_task_args['task_vars'] + play_context = _queued_task_args['play_context'] + # Try to grab the previous host state, if it doesn't exist use get_host_state to generate an empty state + try: + prev_host_state = prev_host_states[host.name] + except KeyError: + prev_host_state = iterator.get_host_state(host) + + while result.needs_debugger(globally_enabled=self.debugger_active): + next_action = NextAction() + dbg = Debugger(task, host, task_vars, play_context, result, next_action) + dbg.cmdloop() + + if next_action.result == NextAction.REDO: + # rollback host state + self._tqm.clear_failed_hosts() + if task.run_once and iterator._play.strategy in add_internal_fqcns(('linear',)) and result.is_failed(): + for host_name, state in prev_host_states.items(): + if host_name == host.name: + continue + iterator.set_state_for_host(host_name, state) + iterator._play._removed_hosts.remove(host_name) + iterator.set_state_for_host(host.name, prev_host_state) + for method, what in status_to_stats_map: + if getattr(result, method)(): + self._tqm._stats.decrement(what, host.name) + self._tqm._stats.decrement('ok', host.name) + + # redo + self._queue_task(host, task, task_vars, play_context) + + _processed_results.extend(debug_closure(func)(self, iterator, one_pass)) + break + elif next_action.result == NextAction.CONTINUE: + _processed_results.append(result) + break + elif next_action.result == NextAction.EXIT: + # Matches KeyboardInterrupt from bin/ansible + sys.exit(99) + else: + _processed_results.append(result) + + return _processed_results + return inner + + +class StrategyBase: + + ''' + This is the base class for strategy plugins, which contains some common + code useful to all strategies like running handlers, cleanup actions, etc. + ''' + + # by default, strategies should support throttling but we allow individual + # strategies to disable this and either forego supporting it or managing + # the throttling internally (as `free` does) + ALLOW_BASE_THROTTLING = True + + def __init__(self, tqm): + self._tqm = tqm + self._inventory = tqm.get_inventory() + self._workers = tqm._workers + self._variable_manager = tqm.get_variable_manager() + self._loader = tqm.get_loader() + self._final_q = tqm._final_q + self._step = context.CLIARGS.get('step', False) + self._diff = context.CLIARGS.get('diff', False) + + # the task cache is a dictionary of tuples of (host.name, task._uuid) + # used to find the original task object of in-flight tasks and to store + # the task args/vars and play context info used to queue the task. + self._queued_task_cache = {} + + # Backwards compat: self._display isn't really needed, just import the global display and use that. + self._display = display + + # internal counters + self._pending_results = 0 + self._cur_worker = 0 + + # this dictionary is used to keep track of hosts that have + # outstanding tasks still in queue + self._blocked_hosts = dict() + + self._results = deque() + self._results_lock = threading.Condition(threading.Lock()) + + # create the result processing thread for reading results in the background + self._results_thread = threading.Thread(target=results_thread_main, args=(self,)) + self._results_thread.daemon = True + self._results_thread.start() + + # holds the list of active (persistent) connections to be shutdown at + # play completion + self._active_connections = dict() + + # Caches for get_host calls, to avoid calling excessively + # These values should be set at the top of the ``run`` method of each + # strategy plugin. Use ``_set_hosts_cache`` to set these values + self._hosts_cache = [] + self._hosts_cache_all = [] + + self.debugger_active = C.ENABLE_TASK_DEBUGGER + + def _set_hosts_cache(self, play, refresh=True): + """Responsible for setting _hosts_cache and _hosts_cache_all + + See comment in ``__init__`` for the purpose of these caches + """ + if not refresh and all((self._hosts_cache, self._hosts_cache_all)): + return + + if not play.finalized and Templar(None).is_template(play.hosts): + _pattern = 'all' + else: + _pattern = play.hosts or 'all' + self._hosts_cache_all = [h.name for h in self._inventory.get_hosts(pattern=_pattern, ignore_restrictions=True)] + self._hosts_cache = [h.name for h in self._inventory.get_hosts(play.hosts, order=play.order)] + + def cleanup(self): + # close active persistent connections + for sock in self._active_connections.values(): + try: + conn = Connection(sock) + conn.reset() + except ConnectionError as e: + # most likely socket is already closed + display.debug("got an error while closing persistent connection: %s" % e) + self._final_q.put(_sentinel) + self._results_thread.join() + + def run(self, iterator, play_context, result=0): + # execute one more pass through the iterator without peeking, to + # make sure that all of the hosts are advanced to their final task. + # This should be safe, as everything should be IteratingStates.COMPLETE by + # this point, though the strategy may not advance the hosts itself. + + for host in self._hosts_cache: + if host not in self._tqm._unreachable_hosts: + try: + iterator.get_next_task_for_host(self._inventory.hosts[host]) + except KeyError: + iterator.get_next_task_for_host(self._inventory.get_host(host)) + + # return the appropriate code, depending on the status hosts after the run + if not isinstance(result, bool) and result != self._tqm.RUN_OK: + return result + elif len(self._tqm._unreachable_hosts.keys()) > 0: + return self._tqm.RUN_UNREACHABLE_HOSTS + elif len(iterator.get_failed_hosts()) > 0: + return self._tqm.RUN_FAILED_HOSTS + else: + return self._tqm.RUN_OK + + def get_hosts_remaining(self, play): + self._set_hosts_cache(play, refresh=False) + ignore = set(self._tqm._failed_hosts).union(self._tqm._unreachable_hosts) + return [host for host in self._hosts_cache if host not in ignore] + + def get_failed_hosts(self, play): + self._set_hosts_cache(play, refresh=False) + return [host for host in self._hosts_cache if host in self._tqm._failed_hosts] + + def add_tqm_variables(self, vars, play): + ''' + Base class method to add extra variables/information to the list of task + vars sent through the executor engine regarding the task queue manager state. + ''' + vars['ansible_current_hosts'] = self.get_hosts_remaining(play) + vars['ansible_failed_hosts'] = self.get_failed_hosts(play) + + def _queue_task(self, host, task, task_vars, play_context): + ''' handles queueing the task up to be sent to a worker ''' + + display.debug("entering _queue_task() for %s/%s" % (host.name, task.action)) + + # Add a write lock for tasks. + # Maybe this should be added somewhere further up the call stack but + # this is the earliest in the code where we have task (1) extracted + # into its own variable and (2) there's only a single code path + # leading to the module being run. This is called by two + # functions: linear.py::run(), and + # free.py::run() so we'd have to add to both to do it there. + # The next common higher level is __init__.py::run() and that has + # tasks inside of play_iterator so we'd have to extract them to do it + # there. + + if task.action not in action_write_locks.action_write_locks: + display.debug('Creating lock for %s' % task.action) + action_write_locks.action_write_locks[task.action] = Lock() + + # create a templar and template things we need later for the queuing process + templar = Templar(loader=self._loader, variables=task_vars) + + try: + throttle = int(templar.template(task.throttle)) + except Exception as e: + raise AnsibleError("Failed to convert the throttle value to an integer.", obj=task._ds, orig_exc=e) + + # and then queue the new task + try: + # Determine the "rewind point" of the worker list. This means we start + # iterating over the list of workers until the end of the list is found. + # Normally, that is simply the length of the workers list (as determined + # by the forks or serial setting), however a task/block/play may "throttle" + # that limit down. + rewind_point = len(self._workers) + if throttle > 0 and self.ALLOW_BASE_THROTTLING: + if task.run_once: + display.debug("Ignoring 'throttle' as 'run_once' is also set for '%s'" % task.get_name()) + else: + if throttle <= rewind_point: + display.debug("task: %s, throttle: %d" % (task.get_name(), throttle)) + rewind_point = throttle + + queued = False + starting_worker = self._cur_worker + while True: + if self._cur_worker >= rewind_point: + self._cur_worker = 0 + + worker_prc = self._workers[self._cur_worker] + if worker_prc is None or not worker_prc.is_alive(): + self._queued_task_cache[(host.name, task._uuid)] = { + 'host': host, + 'task': task, + 'task_vars': task_vars, + 'play_context': play_context + } + + worker_prc = WorkerProcess(self._final_q, task_vars, host, task, play_context, self._loader, self._variable_manager, plugin_loader) + self._workers[self._cur_worker] = worker_prc + self._tqm.send_callback('v2_runner_on_start', host, task) + worker_prc.start() + display.debug("worker is %d (out of %d available)" % (self._cur_worker + 1, len(self._workers))) + queued = True + + self._cur_worker += 1 + + if self._cur_worker >= rewind_point: + self._cur_worker = 0 + + if queued: + break + elif self._cur_worker == starting_worker: + time.sleep(0.0001) + + self._pending_results += 1 + except (EOFError, IOError, AssertionError) as e: + # most likely an abort + display.debug("got an error while queuing: %s" % e) + return + display.debug("exiting _queue_task() for %s/%s" % (host.name, task.action)) + + def get_task_hosts(self, iterator, task_host, task): + if task.run_once: + host_list = [host for host in self._hosts_cache if host not in self._tqm._unreachable_hosts] + else: + host_list = [task_host.name] + return host_list + + def get_delegated_hosts(self, result, task): + host_name = result.get('_ansible_delegated_vars', {}).get('ansible_delegated_host', None) + return [host_name or task.delegate_to] + + def _set_always_delegated_facts(self, result, task): + """Sets host facts for ``delegate_to`` hosts for facts that should + always be delegated + + This operation mutates ``result`` to remove the always delegated facts + + See ``ALWAYS_DELEGATE_FACT_PREFIXES`` + """ + if task.delegate_to is None: + return + + facts = result['ansible_facts'] + always_keys = set() + _add = always_keys.add + for fact_key in facts: + for always_key in ALWAYS_DELEGATE_FACT_PREFIXES: + if fact_key.startswith(always_key): + _add(fact_key) + if always_keys: + _pop = facts.pop + always_facts = { + 'ansible_facts': dict((k, _pop(k)) for k in list(facts) if k in always_keys) + } + host_list = self.get_delegated_hosts(result, task) + _set_host_facts = self._variable_manager.set_host_facts + for target_host in host_list: + _set_host_facts(target_host, always_facts) + + def normalize_task_result(self, task_result): + """Normalize a TaskResult to reference actual Host and Task objects + when only given the ``Host.name``, or the ``Task._uuid`` + + Only the ``Host.name`` and ``Task._uuid`` are commonly sent back from + the ``TaskExecutor`` or ``WorkerProcess`` due to performance concerns + + Mutates the original object + """ + + if isinstance(task_result._host, string_types): + # If the value is a string, it is ``Host.name`` + task_result._host = self._inventory.get_host(to_text(task_result._host)) + + if isinstance(task_result._task, string_types): + # If the value is a string, it is ``Task._uuid`` + queue_cache_entry = (task_result._host.name, task_result._task) + try: + found_task = self._queued_task_cache[queue_cache_entry]['task'] + except KeyError: + # This should only happen due to an implicit task created by the + # TaskExecutor, restrict this behavior to the explicit use case + # of an implicit async_status task + if task_result._task_fields.get('action') != 'async_status': + raise + original_task = Task() + else: + original_task = found_task.copy(exclude_parent=True, exclude_tasks=True) + original_task._parent = found_task._parent + original_task.from_attrs(task_result._task_fields) + task_result._task = original_task + + return task_result + + @debug_closure + def _process_pending_results(self, iterator, one_pass=False, max_passes=None): + ''' + Reads results off the final queue and takes appropriate action + based on the result (executing callbacks, updating state, etc.). + ''' + + ret_results = [] + handler_templar = Templar(self._loader) + + def search_handler_blocks_by_name(handler_name, handler_blocks): + # iterate in reversed order since last handler loaded with the same name wins + for handler_block in reversed(handler_blocks): + for handler_task in handler_block.block: + if handler_task.name: + try: + if not handler_task.cached_name: + if handler_templar.is_template(handler_task.name): + handler_templar.available_variables = self._variable_manager.get_vars(play=iterator._play, + task=handler_task, + _hosts=self._hosts_cache, + _hosts_all=self._hosts_cache_all) + handler_task.name = handler_templar.template(handler_task.name) + handler_task.cached_name = True + + # first we check with the full result of get_name(), which may + # include the role name (if the handler is from a role). If that + # is not found, we resort to the simple name field, which doesn't + # have anything extra added to it. + candidates = ( + handler_task.name, + handler_task.get_name(include_role_fqcn=False), + handler_task.get_name(include_role_fqcn=True), + ) + + if handler_name in candidates: + return handler_task + except (UndefinedError, AnsibleUndefinedVariable) as e: + # We skip this handler due to the fact that it may be using + # a variable in the name that was conditionally included via + # set_fact or some other method, and we don't want to error + # out unnecessarily + if not handler_task.listen: + display.warning( + "Handler '%s' is unusable because it has no listen topics and " + "the name could not be templated (host-specific variables are " + "not supported in handler names). The error: %s" % (handler_task.name, to_text(e)) + ) + continue + + cur_pass = 0 + while True: + try: + self._results_lock.acquire() + task_result = self._results.popleft() + except IndexError: + break + finally: + self._results_lock.release() + + original_host = task_result._host + original_task = task_result._task + + # all host status messages contain 2 entries: (msg, task_result) + role_ran = False + if task_result.is_failed(): + role_ran = True + ignore_errors = original_task.ignore_errors + if not ignore_errors: + # save the current state before failing it for later inspection + state_when_failed = iterator.get_state_for_host(original_host.name) + display.debug("marking %s as failed" % original_host.name) + if original_task.run_once: + # if we're using run_once, we have to fail every host here + for h in self._inventory.get_hosts(iterator._play.hosts): + if h.name not in self._tqm._unreachable_hosts: + iterator.mark_host_failed(h) + else: + iterator.mark_host_failed(original_host) + + state, _ = iterator.get_next_task_for_host(original_host, peek=True) + + if iterator.is_failed(original_host) and state and state.run_state == IteratingStates.COMPLETE: + self._tqm._failed_hosts[original_host.name] = True + + # if we're iterating on the rescue portion of a block then + # we save the failed task in a special var for use + # within the rescue/always + if iterator.is_any_block_rescuing(state_when_failed): + self._tqm._stats.increment('rescued', original_host.name) + iterator._play._removed_hosts.remove(original_host.name) + self._variable_manager.set_nonpersistent_facts( + original_host.name, + dict( + ansible_failed_task=wrap_var(original_task.serialize()), + ansible_failed_result=task_result._result, + ), + ) + else: + self._tqm._stats.increment('failures', original_host.name) + else: + self._tqm._stats.increment('ok', original_host.name) + self._tqm._stats.increment('ignored', original_host.name) + if 'changed' in task_result._result and task_result._result['changed']: + self._tqm._stats.increment('changed', original_host.name) + self._tqm.send_callback('v2_runner_on_failed', task_result, ignore_errors=ignore_errors) + elif task_result.is_unreachable(): + ignore_unreachable = original_task.ignore_unreachable + if not ignore_unreachable: + self._tqm._unreachable_hosts[original_host.name] = True + iterator._play._removed_hosts.append(original_host.name) + self._tqm._stats.increment('dark', original_host.name) + else: + self._tqm._stats.increment('ok', original_host.name) + self._tqm._stats.increment('ignored', original_host.name) + self._tqm.send_callback('v2_runner_on_unreachable', task_result) + elif task_result.is_skipped(): + self._tqm._stats.increment('skipped', original_host.name) + self._tqm.send_callback('v2_runner_on_skipped', task_result) + else: + role_ran = True + + if original_task.loop: + # this task had a loop, and has more than one result, so + # loop over all of them instead of a single result + result_items = task_result._result.get('results', []) + else: + result_items = [task_result._result] + + for result_item in result_items: + if '_ansible_notify' in result_item: + if task_result.is_changed(): + # The shared dictionary for notified handlers is a proxy, which + # does not detect when sub-objects within the proxy are modified. + # So, per the docs, we reassign the list so the proxy picks up and + # notifies all other threads + for handler_name in result_item['_ansible_notify']: + found = False + # Find the handler using the above helper. First we look up the + # dependency chain of the current task (if it's from a role), otherwise + # we just look through the list of handlers in the current play/all + # roles and use the first one that matches the notify name + target_handler = search_handler_blocks_by_name(handler_name, iterator._play.handlers) + if target_handler is not None: + found = True + if target_handler.notify_host(original_host): + self._tqm.send_callback('v2_playbook_on_notify', target_handler, original_host) + + for listening_handler_block in iterator._play.handlers: + for listening_handler in listening_handler_block.block: + listeners = getattr(listening_handler, 'listen', []) or [] + if not listeners: + continue + + listeners = listening_handler.get_validated_value( + 'listen', listening_handler.fattributes.get('listen'), listeners, handler_templar + ) + if handler_name not in listeners: + continue + else: + found = True + + if listening_handler.notify_host(original_host): + self._tqm.send_callback('v2_playbook_on_notify', listening_handler, original_host) + + # and if none were found, then we raise an error + if not found: + msg = ("The requested handler '%s' was not found in either the main handlers list nor in the listening " + "handlers list" % handler_name) + if C.ERROR_ON_MISSING_HANDLER: + raise AnsibleError(msg) + else: + display.warning(msg) + + if 'add_host' in result_item: + # this task added a new host (add_host module) + new_host_info = result_item.get('add_host', dict()) + self._inventory.add_dynamic_host(new_host_info, result_item) + # ensure host is available for subsequent plays + if result_item.get('changed') and new_host_info['host_name'] not in self._hosts_cache_all: + self._hosts_cache_all.append(new_host_info['host_name']) + + elif 'add_group' in result_item: + # this task added a new group (group_by module) + self._inventory.add_dynamic_group(original_host, result_item) + + if 'add_host' in result_item or 'add_group' in result_item: + item_vars = _get_item_vars(result_item, original_task) + found_task_vars = self._queued_task_cache.get((original_host.name, task_result._task._uuid))['task_vars'] + if item_vars: + all_task_vars = combine_vars(found_task_vars, item_vars) + else: + all_task_vars = found_task_vars + all_task_vars[original_task.register] = wrap_var(result_item) + post_process_whens(result_item, original_task, handler_templar, all_task_vars) + if original_task.loop or original_task.loop_with: + new_item_result = TaskResult( + task_result._host, + task_result._task, + result_item, + task_result._task_fields, + ) + self._tqm.send_callback('v2_runner_item_on_ok', new_item_result) + if result_item.get('changed', False): + task_result._result['changed'] = True + if result_item.get('failed', False): + task_result._result['failed'] = True + + if 'ansible_facts' in result_item and original_task.action not in C._ACTION_DEBUG: + # if delegated fact and we are delegating facts, we need to change target host for them + if original_task.delegate_to is not None and original_task.delegate_facts: + host_list = self.get_delegated_hosts(result_item, original_task) + else: + # Set facts that should always be on the delegated hosts + self._set_always_delegated_facts(result_item, original_task) + + host_list = self.get_task_hosts(iterator, original_host, original_task) + + if original_task.action in C._ACTION_INCLUDE_VARS: + for (var_name, var_value) in result_item['ansible_facts'].items(): + # find the host we're actually referring too here, which may + # be a host that is not really in inventory at all + for target_host in host_list: + self._variable_manager.set_host_variable(target_host, var_name, var_value) + else: + cacheable = result_item.pop('_ansible_facts_cacheable', False) + for target_host in host_list: + # so set_fact is a misnomer but 'cacheable = true' was meant to create an 'actual fact' + # to avoid issues with precedence and confusion with set_fact normal operation, + # we set BOTH fact and nonpersistent_facts (aka hostvar) + # when fact is retrieved from cache in subsequent operations it will have the lower precedence, + # but for playbook setting it the 'higher' precedence is kept + is_set_fact = original_task.action in C._ACTION_SET_FACT + if not is_set_fact or cacheable: + self._variable_manager.set_host_facts(target_host, result_item['ansible_facts'].copy()) + if is_set_fact: + self._variable_manager.set_nonpersistent_facts(target_host, result_item['ansible_facts'].copy()) + + if 'ansible_stats' in result_item and 'data' in result_item['ansible_stats'] and result_item['ansible_stats']['data']: + + if 'per_host' not in result_item['ansible_stats'] or result_item['ansible_stats']['per_host']: + host_list = self.get_task_hosts(iterator, original_host, original_task) + else: + host_list = [None] + + data = result_item['ansible_stats']['data'] + aggregate = 'aggregate' in result_item['ansible_stats'] and result_item['ansible_stats']['aggregate'] + for myhost in host_list: + for k in data.keys(): + if aggregate: + self._tqm._stats.update_custom_stats(k, data[k], myhost) + else: + self._tqm._stats.set_custom_stats(k, data[k], myhost) + + if 'diff' in task_result._result: + if self._diff or getattr(original_task, 'diff', False): + self._tqm.send_callback('v2_on_file_diff', task_result) + + if not isinstance(original_task, TaskInclude): + self._tqm._stats.increment('ok', original_host.name) + if 'changed' in task_result._result and task_result._result['changed']: + self._tqm._stats.increment('changed', original_host.name) + + # finally, send the ok for this task + self._tqm.send_callback('v2_runner_on_ok', task_result) + + # register final results + if original_task.register: + + if not isidentifier(original_task.register): + raise AnsibleError("Invalid variable name in 'register' specified: '%s'" % original_task.register) + + host_list = self.get_task_hosts(iterator, original_host, original_task) + + clean_copy = strip_internal_keys(module_response_deepcopy(task_result._result)) + if 'invocation' in clean_copy: + del clean_copy['invocation'] + + for target_host in host_list: + self._variable_manager.set_nonpersistent_facts(target_host, {original_task.register: clean_copy}) + + self._pending_results -= 1 + if original_host.name in self._blocked_hosts: + del self._blocked_hosts[original_host.name] + + # If this is a role task, mark the parent role as being run (if + # the task was ok or failed, but not skipped or unreachable) + if original_task._role is not None and role_ran: # TODO: and original_task.action not in C._ACTION_INCLUDE_ROLE:? + # lookup the role in the ROLE_CACHE to make sure we're dealing + # with the correct object and mark it as executed + for (entry, role_obj) in iterator._play.ROLE_CACHE[original_task._role.get_name()].items(): + if role_obj._uuid == original_task._role._uuid: + role_obj._had_task_run[original_host.name] = True + + ret_results.append(task_result) + + if isinstance(original_task, Handler): + for handler in (h for b in iterator._play.handlers for h in b.block if h._uuid == original_task._uuid): + handler.remove_host(original_host) + + if one_pass or max_passes is not None and (cur_pass + 1) >= max_passes: + break + + cur_pass += 1 + + return ret_results + + def _wait_on_pending_results(self, iterator): + ''' + Wait for the shared counter to drop to zero, using a short sleep + between checks to ensure we don't spin lock + ''' + + ret_results = [] + + display.debug("waiting for pending results...") + while self._pending_results > 0 and not self._tqm._terminated: + + if self._tqm.has_dead_workers(): + raise AnsibleError("A worker was found in a dead state") + + results = self._process_pending_results(iterator) + ret_results.extend(results) + if self._pending_results > 0: + time.sleep(C.DEFAULT_INTERNAL_POLL_INTERVAL) + + display.debug("no more pending results, returning what we have") + + return ret_results + + def _copy_included_file(self, included_file): + ''' + A proven safe and performant way to create a copy of an included file + ''' + ti_copy = included_file._task.copy(exclude_parent=True) + ti_copy._parent = included_file._task._parent + + temp_vars = ti_copy.vars | included_file._vars + + ti_copy.vars = temp_vars + + return ti_copy + + def _load_included_file(self, included_file, iterator, is_handler=False): + ''' + Loads an included YAML file of tasks, applying the optional set of variables. + + Raises AnsibleError exception in case of a failure during including a file, + in such case the caller is responsible for marking the host(s) as failed + using PlayIterator.mark_host_failed(). + ''' + display.debug("loading included file: %s" % included_file._filename) + try: + data = self._loader.load_from_file(included_file._filename) + if data is None: + return [] + elif not isinstance(data, list): + raise AnsibleError("included task files must contain a list of tasks") + + ti_copy = self._copy_included_file(included_file) + + block_list = load_list_of_blocks( + data, + play=iterator._play, + parent_block=ti_copy.build_parent_block(), + role=included_file._task._role, + use_handlers=is_handler, + loader=self._loader, + variable_manager=self._variable_manager, + ) + + # since we skip incrementing the stats when the task result is + # first processed, we do so now for each host in the list + for host in included_file._hosts: + self._tqm._stats.increment('ok', host.name) + except AnsibleParserError: + raise + except AnsibleError as e: + if isinstance(e, AnsibleFileNotFound): + reason = "Could not find or access '%s' on the Ansible Controller." % to_text(e.file_name) + else: + reason = to_text(e) + + for r in included_file._results: + r._result['failed'] = True + + for host in included_file._hosts: + tr = TaskResult(host=host, task=included_file._task, return_data=dict(failed=True, reason=reason)) + self._tqm._stats.increment('failures', host.name) + self._tqm.send_callback('v2_runner_on_failed', tr) + raise AnsibleError(reason) from e + + # finally, send the callback and return the list of blocks loaded + self._tqm.send_callback('v2_playbook_on_include', included_file) + display.debug("done processing included file") + return block_list + + def _take_step(self, task, host=None): + + ret = False + msg = u'Perform task: %s ' % task + if host: + msg += u'on %s ' % host + msg += u'(N)o/(y)es/(c)ontinue: ' + resp = display.prompt(msg) + + if resp.lower() in ['y', 'yes']: + display.debug("User ran task") + ret = True + elif resp.lower() in ['c', 'continue']: + display.debug("User ran task and canceled step mode") + self._step = False + ret = True + else: + display.debug("User skipped task") + + display.banner(msg) + + return ret + + def _cond_not_supported_warn(self, task_name): + display.warning("%s task does not support when conditional" % task_name) + + def _execute_meta(self, task, play_context, iterator, target_host): + + # meta tasks store their args in the _raw_params field of args, + # since they do not use k=v pairs, so get that + meta_action = task.args.get('_raw_params') + + def _evaluate_conditional(h): + all_vars = self._variable_manager.get_vars(play=iterator._play, host=h, task=task, + _hosts=self._hosts_cache, _hosts_all=self._hosts_cache_all) + templar = Templar(loader=self._loader, variables=all_vars) + return task.evaluate_conditional(templar, all_vars) + + skipped = False + msg = meta_action + skip_reason = '%s conditional evaluated to False' % meta_action + if isinstance(task, Handler): + self._tqm.send_callback('v2_playbook_on_handler_task_start', task) + else: + self._tqm.send_callback('v2_playbook_on_task_start', task, is_conditional=False) + + # These don't support "when" conditionals + if meta_action in ('noop', 'refresh_inventory', 'reset_connection') and task.when: + self._cond_not_supported_warn(meta_action) + + if meta_action == 'noop': + msg = "noop" + elif meta_action == 'flush_handlers': + if _evaluate_conditional(target_host): + host_state = iterator.get_state_for_host(target_host.name) + if host_state.run_state == IteratingStates.HANDLERS: + raise AnsibleError('flush_handlers cannot be used as a handler') + if target_host.name not in self._tqm._unreachable_hosts: + host_state.pre_flushing_run_state = host_state.run_state + host_state.run_state = IteratingStates.HANDLERS + msg = "triggered running handlers for %s" % target_host.name + else: + skipped = True + skip_reason += ', not running handlers for %s' % target_host.name + elif meta_action == 'refresh_inventory': + self._inventory.refresh_inventory() + self._set_hosts_cache(iterator._play) + msg = "inventory successfully refreshed" + elif meta_action == 'clear_facts': + if _evaluate_conditional(target_host): + for host in self._inventory.get_hosts(iterator._play.hosts): + hostname = host.get_name() + self._variable_manager.clear_facts(hostname) + msg = "facts cleared" + else: + skipped = True + skip_reason += ', not clearing facts and fact cache for %s' % target_host.name + elif meta_action == 'clear_host_errors': + if _evaluate_conditional(target_host): + for host in self._inventory.get_hosts(iterator._play.hosts): + self._tqm._failed_hosts.pop(host.name, False) + self._tqm._unreachable_hosts.pop(host.name, False) + iterator.clear_host_errors(host) + msg = "cleared host errors" + else: + skipped = True + skip_reason += ', not clearing host error state for %s' % target_host.name + elif meta_action == 'end_batch': + if _evaluate_conditional(target_host): + for host in self._inventory.get_hosts(iterator._play.hosts): + if host.name not in self._tqm._unreachable_hosts: + iterator.set_run_state_for_host(host.name, IteratingStates.COMPLETE) + msg = "ending batch" + else: + skipped = True + skip_reason += ', continuing current batch' + elif meta_action == 'end_play': + if _evaluate_conditional(target_host): + for host in self._inventory.get_hosts(iterator._play.hosts): + if host.name not in self._tqm._unreachable_hosts: + iterator.set_run_state_for_host(host.name, IteratingStates.COMPLETE) + # end_play is used in PlaybookExecutor/TQM to indicate that + # the whole play is supposed to be ended as opposed to just a batch + iterator.end_play = True + msg = "ending play" + else: + skipped = True + skip_reason += ', continuing play' + elif meta_action == 'end_host': + if _evaluate_conditional(target_host): + iterator.set_run_state_for_host(target_host.name, IteratingStates.COMPLETE) + iterator._play._removed_hosts.append(target_host.name) + msg = "ending play for %s" % target_host.name + else: + skipped = True + skip_reason += ", continuing execution for %s" % target_host.name + # TODO: Nix msg here? Left for historical reasons, but skip_reason exists now. + msg = "end_host conditional evaluated to false, continuing execution for %s" % target_host.name + elif meta_action == 'role_complete': + # Allow users to use this in a play as reported in https://github.com/ansible/ansible/issues/22286? + # How would this work with allow_duplicates?? + if task.implicit: + if target_host.name in task._role._had_task_run: + task._role._completed[target_host.name] = True + msg = 'role_complete for %s' % target_host.name + elif meta_action == 'reset_connection': + all_vars = self._variable_manager.get_vars(play=iterator._play, host=target_host, task=task, + _hosts=self._hosts_cache, _hosts_all=self._hosts_cache_all) + templar = Templar(loader=self._loader, variables=all_vars) + + # apply the given task's information to the connection info, + # which may override some fields already set by the play or + # the options specified on the command line + play_context = play_context.set_task_and_variable_override(task=task, variables=all_vars, templar=templar) + + # fields set from the play/task may be based on variables, so we have to + # do the same kind of post validation step on it here before we use it. + play_context.post_validate(templar=templar) + + # now that the play context is finalized, if the remote_addr is not set + # default to using the host's address field as the remote address + if not play_context.remote_addr: + play_context.remote_addr = target_host.address + + # We also add "magic" variables back into the variables dict to make sure + # a certain subset of variables exist. This 'mostly' works here cause meta + # disregards the loop, but should not really use play_context at all + play_context.update_vars(all_vars) + + if target_host in self._active_connections: + connection = Connection(self._active_connections[target_host]) + del self._active_connections[target_host] + else: + connection = plugin_loader.connection_loader.get(play_context.connection, play_context, os.devnull) + connection.set_options(task_keys=task.dump_attrs(), var_options=all_vars) + play_context.set_attributes_from_plugin(connection) + + if connection: + try: + connection.reset() + msg = 'reset connection' + except ConnectionError as e: + # most likely socket is already closed + display.debug("got an error while closing persistent connection: %s" % e) + else: + msg = 'no connection, nothing to reset' + else: + raise AnsibleError("invalid meta action requested: %s" % meta_action, obj=task._ds) + + result = {'msg': msg} + if skipped: + result['skipped'] = True + result['skip_reason'] = skip_reason + else: + result['changed'] = False + + if not task.implicit: + header = skip_reason if skipped else msg + display.vv(f"META: {header}") + + if isinstance(task, Handler): + task.remove_host(target_host) + + res = TaskResult(target_host, task, result) + if skipped: + self._tqm.send_callback('v2_runner_on_skipped', res) + return [res] + + def get_hosts_left(self, iterator): + ''' returns list of available hosts for this iterator by filtering out unreachables ''' + + hosts_left = [] + for host in self._hosts_cache: + if host not in self._tqm._unreachable_hosts: + try: + hosts_left.append(self._inventory.hosts[host]) + except KeyError: + hosts_left.append(self._inventory.get_host(host)) + return hosts_left + + def update_active_connections(self, results): + ''' updates the current active persistent connections ''' + for r in results: + if 'args' in r._task_fields: + socket_path = r._task_fields['args'].get('_ansible_socket') + if socket_path: + if r._host not in self._active_connections: + self._active_connections[r._host] = socket_path + + +class NextAction(object): + """ The next action after an interpreter's exit. """ + REDO = 1 + CONTINUE = 2 + EXIT = 3 + + def __init__(self, result=EXIT): + self.result = result + + +class Debugger(cmd.Cmd): + prompt_continuous = '> ' # multiple lines + + def __init__(self, task, host, task_vars, play_context, result, next_action): + # cmd.Cmd is old-style class + cmd.Cmd.__init__(self) + + self.prompt = '[%s] %s (debug)> ' % (host, task) + self.intro = None + self.scope = {} + self.scope['task'] = task + self.scope['task_vars'] = task_vars + self.scope['host'] = host + self.scope['play_context'] = play_context + self.scope['result'] = result + self.next_action = next_action + + def cmdloop(self): + try: + cmd.Cmd.cmdloop(self) + except KeyboardInterrupt: + pass + + do_h = cmd.Cmd.do_help + + def do_EOF(self, args): + """Quit""" + return self.do_quit(args) + + def do_quit(self, args): + """Quit""" + display.display('User interrupted execution') + self.next_action.result = NextAction.EXIT + return True + + do_q = do_quit + + def do_continue(self, args): + """Continue to next result""" + self.next_action.result = NextAction.CONTINUE + return True + + do_c = do_continue + + def do_redo(self, args): + """Schedule task for re-execution. The re-execution may not be the next result""" + self.next_action.result = NextAction.REDO + return True + + do_r = do_redo + + def do_update_task(self, args): + """Recreate the task from ``task._ds``, and template with updated ``task_vars``""" + templar = Templar(None, variables=self.scope['task_vars']) + task = self.scope['task'] + task = task.load_data(task._ds) + task.post_validate(templar) + self.scope['task'] = task + + do_u = do_update_task + + def evaluate(self, args): + try: + return eval(args, globals(), self.scope) + except Exception: + t, v = sys.exc_info()[:2] + if isinstance(t, str): + exc_type_name = t + else: + exc_type_name = t.__name__ + display.display('***%s:%s' % (exc_type_name, repr(v))) + raise + + def do_pprint(self, args): + """Pretty Print""" + try: + result = self.evaluate(args) + display.display(pprint.pformat(result)) + except Exception: + pass + + do_p = do_pprint + + def execute(self, args): + try: + code = compile(args + '\n', '<stdin>', 'single') + exec(code, globals(), self.scope) + except Exception: + t, v = sys.exc_info()[:2] + if isinstance(t, str): + exc_type_name = t + else: + exc_type_name = t.__name__ + display.display('***%s:%s' % (exc_type_name, repr(v))) + raise + + def default(self, line): + try: + self.execute(line) + except Exception: + pass diff --git a/lib/ansible/plugins/strategy/debug.py b/lib/ansible/plugins/strategy/debug.py new file mode 100644 index 0000000..f808bcf --- /dev/null +++ b/lib/ansible/plugins/strategy/debug.py @@ -0,0 +1,37 @@ +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' + name: debug + short_description: Executes tasks in interactive debug session. + description: + - Task execution is 'linear' but controlled by an interactive debug session. + version_added: "2.1" + author: Kishin Yagami (!UNKNOWN) +''' + +import cmd +import pprint +import sys + +from ansible.plugins.strategy.linear import StrategyModule as LinearStrategyModule + + +class StrategyModule(LinearStrategyModule): + def __init__(self, tqm): + super(StrategyModule, self).__init__(tqm) + self.debugger_active = True diff --git a/lib/ansible/plugins/strategy/free.py b/lib/ansible/plugins/strategy/free.py new file mode 100644 index 0000000..6f45114 --- /dev/null +++ b/lib/ansible/plugins/strategy/free.py @@ -0,0 +1,303 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' + name: free + short_description: Executes tasks without waiting for all hosts + description: + - Task execution is as fast as possible per batch as defined by C(serial) (default all). + Ansible will not wait for other hosts to finish the current task before queuing more tasks for other hosts. + All hosts are still attempted for the current task, but it prevents blocking new tasks for hosts that have already finished. + - With the free strategy, unlike the default linear strategy, a host that is slow or stuck on a specific task + won't hold up the rest of the hosts and tasks. + version_added: "2.0" + author: Ansible Core Team +''' + +import time + +from ansible import constants as C +from ansible.errors import AnsibleError, AnsibleParserError +from ansible.playbook.handler import Handler +from ansible.playbook.included_file import IncludedFile +from ansible.plugins.loader import action_loader +from ansible.plugins.strategy import StrategyBase +from ansible.template import Templar +from ansible.module_utils._text import to_text +from ansible.utils.display import Display + +display = Display() + + +class StrategyModule(StrategyBase): + + # This strategy manages throttling on its own, so we don't want it done in queue_task + ALLOW_BASE_THROTTLING = False + + def __init__(self, tqm): + super(StrategyModule, self).__init__(tqm) + self._host_pinned = False + + def run(self, iterator, play_context): + ''' + The "free" strategy is a bit more complex, in that it allows tasks to + be sent to hosts as quickly as they can be processed. This means that + some hosts may finish very quickly if run tasks result in little or no + work being done versus other systems. + + The algorithm used here also tries to be more "fair" when iterating + through hosts by remembering the last host in the list to be given a task + and starting the search from there as opposed to the top of the hosts + list again, which would end up favoring hosts near the beginning of the + list. + ''' + + # the last host to be given a task + last_host = 0 + + result = self._tqm.RUN_OK + + # start with all workers being counted as being free + workers_free = len(self._workers) + + self._set_hosts_cache(iterator._play) + + if iterator._play.max_fail_percentage is not None: + display.warning("Using max_fail_percentage with the free strategy is not supported, as tasks are executed independently on each host") + + work_to_do = True + while work_to_do and not self._tqm._terminated: + + hosts_left = self.get_hosts_left(iterator) + + if len(hosts_left) == 0: + self._tqm.send_callback('v2_playbook_on_no_hosts_remaining') + result = False + break + + work_to_do = False # assume we have no more work to do + starting_host = last_host # save current position so we know when we've looped back around and need to break + + # try and find an unblocked host with a task to run + host_results = [] + while True: + host = hosts_left[last_host] + display.debug("next free host: %s" % host) + host_name = host.get_name() + + # peek at the next task for the host, to see if there's + # anything to do do for this host + (state, task) = iterator.get_next_task_for_host(host, peek=True) + display.debug("free host state: %s" % state, host=host_name) + display.debug("free host task: %s" % task, host=host_name) + + # check if there is work to do, either there is a task or the host is still blocked which could + # mean that it is processing an include task and after its result is processed there might be + # more tasks to run + if (task or self._blocked_hosts.get(host_name, False)) and not self._tqm._unreachable_hosts.get(host_name, False): + display.debug("this host has work to do", host=host_name) + # set the flag so the outer loop knows we've still found + # some work which needs to be done + work_to_do = True + + if not self._tqm._unreachable_hosts.get(host_name, False) and task: + # check to see if this host is blocked (still executing a previous task) + if not self._blocked_hosts.get(host_name, False): + display.debug("getting variables", host=host_name) + task_vars = self._variable_manager.get_vars(play=iterator._play, host=host, task=task, + _hosts=self._hosts_cache, + _hosts_all=self._hosts_cache_all) + self.add_tqm_variables(task_vars, play=iterator._play) + templar = Templar(loader=self._loader, variables=task_vars) + display.debug("done getting variables", host=host_name) + + try: + throttle = int(templar.template(task.throttle)) + except Exception as e: + raise AnsibleError("Failed to convert the throttle value to an integer.", obj=task._ds, orig_exc=e) + + if throttle > 0: + same_tasks = 0 + for worker in self._workers: + if worker and worker.is_alive() and worker._task._uuid == task._uuid: + same_tasks += 1 + + display.debug("task: %s, same_tasks: %d" % (task.get_name(), same_tasks)) + if same_tasks >= throttle: + break + + # advance the host, mark the host blocked, and queue it + self._blocked_hosts[host_name] = True + iterator.set_state_for_host(host.name, state) + + try: + action = action_loader.get(task.action, class_only=True, collection_list=task.collections) + except KeyError: + # we don't care here, because the action may simply not have a + # corresponding action plugin + action = None + + try: + task.name = to_text(templar.template(task.name, fail_on_undefined=False), nonstring='empty') + display.debug("done templating", host=host_name) + except Exception: + # just ignore any errors during task name templating, + # we don't care if it just shows the raw name + display.debug("templating failed for some reason", host=host_name) + + run_once = templar.template(task.run_once) or action and getattr(action, 'BYPASS_HOST_LOOP', False) + if run_once: + if action and getattr(action, 'BYPASS_HOST_LOOP', False): + raise AnsibleError("The '%s' module bypasses the host loop, which is currently not supported in the free strategy " + "and would instead execute for every host in the inventory list." % task.action, obj=task._ds) + else: + display.warning("Using run_once with the free strategy is not currently supported. This task will still be " + "executed for every host in the inventory list.") + + # check to see if this task should be skipped, due to it being a member of a + # role which has already run (and whether that role allows duplicate execution) + if not isinstance(task, Handler) and task._role and task._role.has_run(host): + # If there is no metadata, the default behavior is to not allow duplicates, + # if there is metadata, check to see if the allow_duplicates flag was set to true + if task._role._metadata is None or task._role._metadata and not task._role._metadata.allow_duplicates: + display.debug("'%s' skipped because role has already run" % task, host=host_name) + del self._blocked_hosts[host_name] + continue + + if task.action in C._ACTION_META: + self._execute_meta(task, play_context, iterator, target_host=host) + self._blocked_hosts[host_name] = False + else: + # handle step if needed, skip meta actions as they are used internally + if not self._step or self._take_step(task, host_name): + if task.any_errors_fatal: + display.warning("Using any_errors_fatal with the free strategy is not supported, " + "as tasks are executed independently on each host") + if isinstance(task, Handler): + self._tqm.send_callback('v2_playbook_on_handler_task_start', task) + else: + self._tqm.send_callback('v2_playbook_on_task_start', task, is_conditional=False) + self._queue_task(host, task, task_vars, play_context) + # each task is counted as a worker being busy + workers_free -= 1 + del task_vars + else: + display.debug("%s is blocked, skipping for now" % host_name) + + # all workers have tasks to do (and the current host isn't done with the play). + # loop back to starting host and break out + if self._host_pinned and workers_free == 0 and work_to_do: + last_host = starting_host + break + + # move on to the next host and make sure we + # haven't gone past the end of our hosts list + last_host += 1 + if last_host > len(hosts_left) - 1: + last_host = 0 + + # if we've looped around back to the start, break out + if last_host == starting_host: + break + + results = self._process_pending_results(iterator) + host_results.extend(results) + + # each result is counted as a worker being free again + workers_free += len(results) + + self.update_active_connections(results) + + included_files = IncludedFile.process_include_results( + host_results, + iterator=iterator, + loader=self._loader, + variable_manager=self._variable_manager + ) + + if len(included_files) > 0: + all_blocks = dict((host, []) for host in hosts_left) + failed_includes_hosts = set() + for included_file in included_files: + display.debug("collecting new blocks for %s" % included_file) + is_handler = False + try: + if included_file._is_role: + new_ir = self._copy_included_file(included_file) + + new_blocks, handler_blocks = new_ir.get_block_list( + play=iterator._play, + variable_manager=self._variable_manager, + loader=self._loader, + ) + else: + is_handler = isinstance(included_file._task, Handler) + new_blocks = self._load_included_file(included_file, iterator=iterator, is_handler=is_handler) + + # let PlayIterator know about any new handlers included via include_role or + # import_role within include_role/include_taks + iterator.handlers = [h for b in iterator._play.handlers for h in b.block] + except AnsibleParserError: + raise + except AnsibleError as e: + if included_file._is_role: + # include_role does not have on_include callback so display the error + display.error(to_text(e), wrap_text=False) + for r in included_file._results: + r._result['failed'] = True + failed_includes_hosts.add(r._host) + continue + + for new_block in new_blocks: + if is_handler: + for task in new_block.block: + task.notified_hosts = included_file._hosts[:] + final_block = new_block + else: + task_vars = self._variable_manager.get_vars( + play=iterator._play, + task=new_block.get_first_parent_include(), + _hosts=self._hosts_cache, + _hosts_all=self._hosts_cache_all, + ) + final_block = new_block.filter_tagged_tasks(task_vars) + for host in hosts_left: + if host in included_file._hosts: + all_blocks[host].append(final_block) + display.debug("done collecting new blocks for %s" % included_file) + + for host in failed_includes_hosts: + self._tqm._failed_hosts[host.name] = True + iterator.mark_host_failed(host) + + display.debug("adding all collected blocks from %d included file(s) to iterator" % len(included_files)) + for host in hosts_left: + iterator.add_tasks(host, all_blocks[host]) + display.debug("done adding collected blocks to iterator") + + # pause briefly so we don't spin lock + time.sleep(C.DEFAULT_INTERNAL_POLL_INTERVAL) + + # collect all the final results + results = self._wait_on_pending_results(iterator) + + # run the base class run() method, which executes the cleanup function + # and runs any outstanding handlers which have been triggered + return super(StrategyModule, self).run(iterator, play_context, result) diff --git a/lib/ansible/plugins/strategy/host_pinned.py b/lib/ansible/plugins/strategy/host_pinned.py new file mode 100644 index 0000000..70f22eb --- /dev/null +++ b/lib/ansible/plugins/strategy/host_pinned.py @@ -0,0 +1,45 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' + name: host_pinned + short_description: Executes tasks on each host without interruption + description: + - Task execution is as fast as possible per host in batch as defined by C(serial) (default all). + Ansible will not start a play for a host unless the play can be finished without interruption by tasks for another host, + i.e. the number of hosts with an active play does not exceed the number of forks. + Ansible will not wait for other hosts to finish the current task before queuing the next task for a host that has finished. + Once a host is done with the play, it opens it's slot to a new host that was waiting to start. + Other than that, it behaves just like the "free" strategy. + version_added: "2.7" + author: Ansible Core Team +''' + +from ansible.plugins.strategy.free import StrategyModule as FreeStrategyModule +from ansible.utils.display import Display + +display = Display() + + +class StrategyModule(FreeStrategyModule): + + def __init__(self, tqm): + super(StrategyModule, self).__init__(tqm) + self._host_pinned = True diff --git a/lib/ansible/plugins/strategy/linear.py b/lib/ansible/plugins/strategy/linear.py new file mode 100644 index 0000000..a3c91c2 --- /dev/null +++ b/lib/ansible/plugins/strategy/linear.py @@ -0,0 +1,406 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' + name: linear + short_description: Executes tasks in a linear fashion + description: + - Task execution is in lockstep per host batch as defined by C(serial) (default all). + Up to the fork limit of hosts will execute each task at the same time and then + the next series of hosts until the batch is done, before going on to the next task. + version_added: "2.0" + notes: + - This was the default Ansible behaviour before 'strategy plugins' were introduced in 2.0. + author: Ansible Core Team +''' + +from ansible import constants as C +from ansible.errors import AnsibleError, AnsibleAssertionError, AnsibleParserError +from ansible.executor.play_iterator import IteratingStates, FailedStates +from ansible.module_utils._text import to_text +from ansible.playbook.handler import Handler +from ansible.playbook.included_file import IncludedFile +from ansible.playbook.task import Task +from ansible.plugins.loader import action_loader +from ansible.plugins.strategy import StrategyBase +from ansible.template import Templar +from ansible.utils.display import Display + +display = Display() + + +class StrategyModule(StrategyBase): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # used for the lockstep to indicate to run handlers + self._in_handlers = False + + def _get_next_task_lockstep(self, hosts, iterator): + ''' + Returns a list of (host, task) tuples, where the task may + be a noop task to keep the iterator in lock step across + all hosts. + ''' + noop_task = Task() + noop_task.action = 'meta' + noop_task.args['_raw_params'] = 'noop' + noop_task.implicit = True + noop_task.set_loader(iterator._play._loader) + + state_task_per_host = {} + for host in hosts: + state, task = iterator.get_next_task_for_host(host, peek=True) + if task is not None: + state_task_per_host[host] = state, task + + if not state_task_per_host: + return [(h, None) for h in hosts] + + if self._in_handlers and not any(filter( + lambda rs: rs == IteratingStates.HANDLERS, + (s.run_state for s, _ in state_task_per_host.values())) + ): + self._in_handlers = False + + if self._in_handlers: + lowest_cur_handler = min( + s.cur_handlers_task for s, t in state_task_per_host.values() + if s.run_state == IteratingStates.HANDLERS + ) + else: + task_uuids = [t._uuid for s, t in state_task_per_host.values()] + _loop_cnt = 0 + while _loop_cnt <= 1: + try: + cur_task = iterator.all_tasks[iterator.cur_task] + except IndexError: + # pick up any tasks left after clear_host_errors + iterator.cur_task = 0 + _loop_cnt += 1 + else: + iterator.cur_task += 1 + if cur_task._uuid in task_uuids: + break + else: + # prevent infinite loop + raise AnsibleAssertionError( + 'BUG: There seems to be a mismatch between tasks in PlayIterator and HostStates.' + ) + + host_tasks = [] + for host, (state, task) in state_task_per_host.items(): + if ((self._in_handlers and lowest_cur_handler == state.cur_handlers_task) or + (not self._in_handlers and cur_task._uuid == task._uuid)): + iterator.set_state_for_host(host.name, state) + host_tasks.append((host, task)) + else: + host_tasks.append((host, noop_task)) + + # once hosts synchronize on 'flush_handlers' lockstep enters + # '_in_handlers' phase where handlers are run instead of tasks + # until at least one host is in IteratingStates.HANDLERS + if (not self._in_handlers and cur_task.action in C._ACTION_META and + cur_task.args.get('_raw_params') == 'flush_handlers'): + self._in_handlers = True + + return host_tasks + + def run(self, iterator, play_context): + ''' + The linear strategy is simple - get the next task and queue + it for all hosts, then wait for the queue to drain before + moving on to the next task + ''' + + # iterate over each task, while there is one left to run + result = self._tqm.RUN_OK + work_to_do = True + + self._set_hosts_cache(iterator._play) + + while work_to_do and not self._tqm._terminated: + + try: + display.debug("getting the remaining hosts for this loop") + hosts_left = self.get_hosts_left(iterator) + display.debug("done getting the remaining hosts for this loop") + + # queue up this task for each host in the inventory + callback_sent = False + work_to_do = False + + host_tasks = self._get_next_task_lockstep(hosts_left, iterator) + + # skip control + skip_rest = False + choose_step = True + + # flag set if task is set to any_errors_fatal + any_errors_fatal = False + + results = [] + for (host, task) in host_tasks: + if not task: + continue + + if self._tqm._terminated: + break + + run_once = False + work_to_do = True + + # check to see if this task should be skipped, due to it being a member of a + # role which has already run (and whether that role allows duplicate execution) + if not isinstance(task, Handler) and task._role and task._role.has_run(host): + # If there is no metadata, the default behavior is to not allow duplicates, + # if there is metadata, check to see if the allow_duplicates flag was set to true + if task._role._metadata is None or task._role._metadata and not task._role._metadata.allow_duplicates: + display.debug("'%s' skipped because role has already run" % task) + continue + + display.debug("getting variables") + task_vars = self._variable_manager.get_vars(play=iterator._play, host=host, task=task, + _hosts=self._hosts_cache, _hosts_all=self._hosts_cache_all) + self.add_tqm_variables(task_vars, play=iterator._play) + templar = Templar(loader=self._loader, variables=task_vars) + display.debug("done getting variables") + + # test to see if the task across all hosts points to an action plugin which + # sets BYPASS_HOST_LOOP to true, or if it has run_once enabled. If so, we + # will only send this task to the first host in the list. + + task_action = templar.template(task.action) + + try: + action = action_loader.get(task_action, class_only=True, collection_list=task.collections) + except KeyError: + # we don't care here, because the action may simply not have a + # corresponding action plugin + action = None + + if task_action in C._ACTION_META: + # for the linear strategy, we run meta tasks just once and for + # all hosts currently being iterated over rather than one host + results.extend(self._execute_meta(task, play_context, iterator, host)) + if task.args.get('_raw_params', None) not in ('noop', 'reset_connection', 'end_host', 'role_complete', 'flush_handlers'): + run_once = True + if (task.any_errors_fatal or run_once) and not task.ignore_errors: + any_errors_fatal = True + else: + # handle step if needed, skip meta actions as they are used internally + if self._step and choose_step: + if self._take_step(task): + choose_step = False + else: + skip_rest = True + break + + run_once = templar.template(task.run_once) or action and getattr(action, 'BYPASS_HOST_LOOP', False) + + if (task.any_errors_fatal or run_once) and not task.ignore_errors: + any_errors_fatal = True + + if not callback_sent: + display.debug("sending task start callback, copying the task so we can template it temporarily") + saved_name = task.name + display.debug("done copying, going to template now") + try: + task.name = to_text(templar.template(task.name, fail_on_undefined=False), nonstring='empty') + display.debug("done templating") + except Exception: + # just ignore any errors during task name templating, + # we don't care if it just shows the raw name + display.debug("templating failed for some reason") + display.debug("here goes the callback...") + if isinstance(task, Handler): + self._tqm.send_callback('v2_playbook_on_handler_task_start', task) + else: + self._tqm.send_callback('v2_playbook_on_task_start', task, is_conditional=False) + task.name = saved_name + callback_sent = True + display.debug("sending task start callback") + + self._blocked_hosts[host.get_name()] = True + self._queue_task(host, task, task_vars, play_context) + del task_vars + + # if we're bypassing the host loop, break out now + if run_once: + break + + results.extend(self._process_pending_results(iterator, max_passes=max(1, int(len(self._tqm._workers) * 0.1)))) + + # go to next host/task group + if skip_rest: + continue + + display.debug("done queuing things up, now waiting for results queue to drain") + if self._pending_results > 0: + results.extend(self._wait_on_pending_results(iterator)) + + self.update_active_connections(results) + + included_files = IncludedFile.process_include_results( + results, + iterator=iterator, + loader=self._loader, + variable_manager=self._variable_manager + ) + + if len(included_files) > 0: + display.debug("we have included files to process") + + display.debug("generating all_blocks data") + all_blocks = dict((host, []) for host in hosts_left) + display.debug("done generating all_blocks data") + included_tasks = [] + failed_includes_hosts = set() + for included_file in included_files: + display.debug("processing included file: %s" % included_file._filename) + is_handler = False + try: + if included_file._is_role: + new_ir = self._copy_included_file(included_file) + + new_blocks, handler_blocks = new_ir.get_block_list( + play=iterator._play, + variable_manager=self._variable_manager, + loader=self._loader, + ) + else: + is_handler = isinstance(included_file._task, Handler) + new_blocks = self._load_included_file(included_file, iterator=iterator, is_handler=is_handler) + + # let PlayIterator know about any new handlers included via include_role or + # import_role within include_role/include_taks + iterator.handlers = [h for b in iterator._play.handlers for h in b.block] + + display.debug("iterating over new_blocks loaded from include file") + for new_block in new_blocks: + if is_handler: + for task in new_block.block: + task.notified_hosts = included_file._hosts[:] + final_block = new_block + else: + task_vars = self._variable_manager.get_vars( + play=iterator._play, + task=new_block.get_first_parent_include(), + _hosts=self._hosts_cache, + _hosts_all=self._hosts_cache_all, + ) + display.debug("filtering new block on tags") + final_block = new_block.filter_tagged_tasks(task_vars) + display.debug("done filtering new block on tags") + + included_tasks.extend(final_block.get_tasks()) + + for host in hosts_left: + if host in included_file._hosts: + all_blocks[host].append(final_block) + + display.debug("done iterating over new_blocks loaded from include file") + except AnsibleParserError: + raise + except AnsibleError as e: + if included_file._is_role: + # include_role does not have on_include callback so display the error + display.error(to_text(e), wrap_text=False) + for r in included_file._results: + r._result['failed'] = True + failed_includes_hosts.add(r._host) + continue + + for host in failed_includes_hosts: + self._tqm._failed_hosts[host.name] = True + iterator.mark_host_failed(host) + + # finally go through all of the hosts and append the + # accumulated blocks to their list of tasks + display.debug("extending task lists for all hosts with included blocks") + + for host in hosts_left: + iterator.add_tasks(host, all_blocks[host]) + + iterator.all_tasks[iterator.cur_task:iterator.cur_task] = included_tasks + + display.debug("done extending task lists") + display.debug("done processing included files") + + display.debug("results queue empty") + + display.debug("checking for any_errors_fatal") + failed_hosts = [] + unreachable_hosts = [] + for res in results: + # execute_meta() does not set 'failed' in the TaskResult + # so we skip checking it with the meta tasks and look just at the iterator + if (res.is_failed() or res._task.action in C._ACTION_META) and iterator.is_failed(res._host): + failed_hosts.append(res._host.name) + elif res.is_unreachable(): + unreachable_hosts.append(res._host.name) + + # if any_errors_fatal and we had an error, mark all hosts as failed + if any_errors_fatal and (len(failed_hosts) > 0 or len(unreachable_hosts) > 0): + dont_fail_states = frozenset([IteratingStates.RESCUE, IteratingStates.ALWAYS]) + for host in hosts_left: + (s, _) = iterator.get_next_task_for_host(host, peek=True) + # the state may actually be in a child state, use the get_active_state() + # method in the iterator to figure out the true active state + s = iterator.get_active_state(s) + if s.run_state not in dont_fail_states or \ + s.run_state == IteratingStates.RESCUE and s.fail_state & FailedStates.RESCUE != 0: + self._tqm._failed_hosts[host.name] = True + result |= self._tqm.RUN_FAILED_BREAK_PLAY + display.debug("done checking for any_errors_fatal") + + display.debug("checking for max_fail_percentage") + if iterator._play.max_fail_percentage is not None and len(results) > 0: + percentage = iterator._play.max_fail_percentage / 100.0 + + if (len(self._tqm._failed_hosts) / iterator.batch_size) > percentage: + for host in hosts_left: + # don't double-mark hosts, or the iterator will potentially + # fail them out of the rescue/always states + if host.name not in failed_hosts: + self._tqm._failed_hosts[host.name] = True + iterator.mark_host_failed(host) + self._tqm.send_callback('v2_playbook_on_no_hosts_remaining') + result |= self._tqm.RUN_FAILED_BREAK_PLAY + display.debug('(%s failed / %s total )> %s max fail' % (len(self._tqm._failed_hosts), iterator.batch_size, percentage)) + display.debug("done checking for max_fail_percentage") + + display.debug("checking to see if all hosts have failed and the running result is not ok") + if result != self._tqm.RUN_OK and len(self._tqm._failed_hosts) >= len(hosts_left): + display.debug("^ not ok, so returning result now") + self._tqm.send_callback('v2_playbook_on_no_hosts_remaining') + return result + display.debug("done checking to see if all hosts have failed") + + except (IOError, EOFError) as e: + display.debug("got IOError/EOFError in task loop: %s" % e) + # most likely an abort, return failed + return self._tqm.RUN_UNKNOWN_ERROR + + # run the base class run() method, which executes the cleanup function + # and runs any outstanding handlers which have been triggered + + return super(StrategyModule, self).run(iterator, play_context, result) diff --git a/lib/ansible/plugins/terminal/__init__.py b/lib/ansible/plugins/terminal/__init__.py new file mode 100644 index 0000000..d464b07 --- /dev/null +++ b/lib/ansible/plugins/terminal/__init__.py @@ -0,0 +1,133 @@ +# +# (c) 2016 Red Hat Inc. +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. +# +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import re + +from abc import ABC, abstractmethod + +from ansible.errors import AnsibleConnectionFailure + + +class TerminalBase(ABC): + ''' + A base class for implementing cli connections + + .. note:: Unlike most of Ansible, nearly all strings in + :class:`TerminalBase` plugins are byte strings. This is because of + how close to the underlying platform these plugins operate. Remember + to mark literal strings as byte string (``b"string"``) and to use + :func:`~ansible.module_utils._text.to_bytes` and + :func:`~ansible.module_utils._text.to_text` to avoid unexpected + problems. + ''' + + #: compiled bytes regular expressions as stdout + terminal_stdout_re = [] # type: list[re.Pattern] + + #: compiled bytes regular expressions as stderr + terminal_stderr_re = [] # type: list[re.Pattern] + + #: compiled bytes regular expressions to remove ANSI codes + ansi_re = [ + re.compile(br'\x1b\[\?1h\x1b='), # CSI ? 1 h ESC = + re.compile(br'\x08.'), # [Backspace] . + re.compile(br"\x1b\[m"), # ANSI reset code + ] + + #: terminal initial prompt + terminal_initial_prompt = None + + #: terminal initial answer + terminal_initial_answer = None + + #: Send newline after prompt match + terminal_inital_prompt_newline = True + + def __init__(self, connection): + self._connection = connection + + def _exec_cli_command(self, cmd, check_rc=True): + ''' + Executes the CLI command on the remote device and returns the output + + :arg cmd: Byte string command to be executed + ''' + return self._connection.exec_command(cmd) + + def _get_prompt(self): + """ + Returns the current prompt from the device + + :returns: A byte string of the prompt + """ + return self._connection.get_prompt() + + def on_open_shell(self): + """Called after the SSH session is established + + This method is called right after the invoke_shell() is called from + the Paramiko SSHClient instance. It provides an opportunity to setup + terminal parameters such as disbling paging for instance. + """ + pass + + def on_close_shell(self): + """Called before the connection is closed + + This method gets called once the connection close has been requested + but before the connection is actually closed. It provides an + opportunity to clean up any terminal resources before the shell is + actually closed + """ + pass + + def on_become(self, passwd=None): + """Called when privilege escalation is requested + + :kwarg passwd: String containing the password + + This method is called when the privilege is requested to be elevated + in the play context by setting become to True. It is the responsibility + of the terminal plugin to actually do the privilege escalation such + as entering `enable` mode for instance + """ + pass + + def on_unbecome(self): + """Called when privilege deescalation is requested + + This method is called when the privilege changed from escalated + (become=True) to non escalated (become=False). It is the responsibility + of this method to actually perform the deauthorization procedure + """ + pass + + def on_authorize(self, passwd=None): + """Deprecated method for privilege escalation + + :kwarg passwd: String containing the password + """ + return self.on_become(passwd) + + def on_deauthorize(self): + """Deprecated method for privilege deescalation + """ + return self.on_unbecome() diff --git a/lib/ansible/plugins/test/__init__.py b/lib/ansible/plugins/test/__init__.py new file mode 100644 index 0000000..1400316 --- /dev/null +++ b/lib/ansible/plugins/test/__init__.py @@ -0,0 +1,13 @@ +# (c) Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible.plugins import AnsibleJinja2Plugin + + +class AnsibleJinja2Test(AnsibleJinja2Plugin): + + def _no_options(self, *args, **kwargs): + raise NotImplementedError("Jinaj2 test plugins do not support option functions, they use direct arguments instead.") diff --git a/lib/ansible/plugins/test/abs.yml b/lib/ansible/plugins/test/abs.yml new file mode 100644 index 0000000..46f7f70 --- /dev/null +++ b/lib/ansible/plugins/test/abs.yml @@ -0,0 +1,23 @@ +DOCUMENTATION: + name: abs + author: Ansible Core + version_added: "2.5" + short_description: is the path absolute + aliases: [is_abs] + description: + - Check if the provided path is absolute, not relative. + - An absolute path expresses the location of a filesystem object starting at the filesystem root and requires no context. + - A relative path does not start at the filesystem root and requires a 'current' directory as a context to resolve. + options: + _input: + description: A path. + type: path + +EXAMPLES: | + is_path_absolute: "{{ '/etc/hosts' is abs }}}" + relative_paths: "{{ all_paths | reject('abs') }}" + +RETURN: + _value: + description: Returns C(True) if the path is absolute, C(False) if it is relative. + type: boolean diff --git a/lib/ansible/plugins/test/all.yml b/lib/ansible/plugins/test/all.yml new file mode 100644 index 0000000..e227d6e --- /dev/null +++ b/lib/ansible/plugins/test/all.yml @@ -0,0 +1,23 @@ +DOCUMENTATION: + name: all + author: Ansible Core + version_added: "2.4" + short_description: are all conditions in a list true + description: + - This test checks each condition in a list for truthiness. + - Same as the C(all) Python function. + options: + _input: + description: List of conditions, each can be a boolean or conditional expression that results in a boolean value. + type: list + elements: raw + required: True +EXAMPLES: | + varexpression: "{{ 3 == 3 }}" + # are all statements true? + {{ [true, booleanvar, varexpression] is all }} + +RETURN: + _value: + description: Returns C(True) if all elements of the list were True, C(False) otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/any.yml b/lib/ansible/plugins/test/any.yml new file mode 100644 index 0000000..0ce9e48 --- /dev/null +++ b/lib/ansible/plugins/test/any.yml @@ -0,0 +1,23 @@ +DOCUMENTATION: + name: any + author: Ansible Core + version_added: "2.4" + short_description: is any conditions in a list true + description: + - This test checks each condition in a list for truthiness. + - Same as the C(any) Python function. + options: + _input: + description: List of conditions, each can be a boolean or conditional expression that results in a boolean value. + type: list + elements: raw + required: True +EXAMPLES: | + varexpression: "{{ 3 == 3 }}" + # are all statements true? + {{ [false, booleanvar, varexpression] is any}} + +RETURN: + _value: + description: Returns C(True) if any element of the list was true, C(False) otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/change.yml b/lib/ansible/plugins/test/change.yml new file mode 100644 index 0000000..1fb1e5e --- /dev/null +++ b/lib/ansible/plugins/test/change.yml @@ -0,0 +1,22 @@ +DOCUMENTATION: + name: changed + author: Ansible Core + version_added: "1.9" + short_description: did the task require changes + aliases: [change] + description: + - Tests if task required changes to complete + - This test checks for the existance of a C(changed) key in the input dictionary and that it is C(True) if present + options: + _input: + description: registered result from an Ansible task + type: dictionary + required: True +EXAMPLES: | + # test 'status' to know how to respond + {{ (taskresults is changed }} + +RETURN: + _value: + description: Returns C(True) if the task was required changes, C(False) otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/changed.yml b/lib/ansible/plugins/test/changed.yml new file mode 100644 index 0000000..1fb1e5e --- /dev/null +++ b/lib/ansible/plugins/test/changed.yml @@ -0,0 +1,22 @@ +DOCUMENTATION: + name: changed + author: Ansible Core + version_added: "1.9" + short_description: did the task require changes + aliases: [change] + description: + - Tests if task required changes to complete + - This test checks for the existance of a C(changed) key in the input dictionary and that it is C(True) if present + options: + _input: + description: registered result from an Ansible task + type: dictionary + required: True +EXAMPLES: | + # test 'status' to know how to respond + {{ (taskresults is changed }} + +RETURN: + _value: + description: Returns C(True) if the task was required changes, C(False) otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/contains.yml b/lib/ansible/plugins/test/contains.yml new file mode 100644 index 0000000..68741da --- /dev/null +++ b/lib/ansible/plugins/test/contains.yml @@ -0,0 +1,49 @@ +DOCUMENTATION: + name: contains + author: Ansible Core + version_added: "2.4" + short_description: does the list contain this element + description: + - Checks the supplied element against the input list to see if it exists within it. + options: + _input: + description: List of elements to compare. + type: list + elements: raw + required: True + _contained: + description: Element to test for. + type: raw + required: True +EXAMPLES: | + # simple expression + {{ listofthings is contains('this') }} + + # as a selector + - action: module=doessomething + when: lacp_groups|selectattr('interfaces', 'contains', 'em1')|first).master + vars: + lacp_groups: + - master: lacp0 + network: 10.65.100.0/24 + gateway: 10.65.100.1 + dns4: + - 10.65.100.10 + - 10.65.100.11 + interfaces: + - em1 + - em2 + + - master: lacp1 + network: 10.65.120.0/24 + gateway: 10.65.120.1 + dns4: + - 10.65.100.10 + - 10.65.100.11 + interfaces: + - em3 + - em4 +RETURN: + _value: + description: Returns C(True) if the specified element is contained in the supplied sequence, C(False) otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/core.py b/lib/ansible/plugins/test/core.py new file mode 100644 index 0000000..d9e7e8b --- /dev/null +++ b/lib/ansible/plugins/test/core.py @@ -0,0 +1,287 @@ +# (c) 2012, Jeroen Hoekx <jeroen@hoekx.be> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import re +import operator as py_operator + +from collections.abc import MutableMapping, MutableSequence + +from ansible.module_utils.compat.version import LooseVersion, StrictVersion + +from ansible import errors +from ansible.module_utils._text import to_native, to_text +from ansible.module_utils.parsing.convert_bool import boolean +from ansible.utils.display import Display +from ansible.utils.version import SemanticVersion + +try: + from packaging.version import Version as PEP440Version + HAS_PACKAGING = True +except ImportError: + HAS_PACKAGING = False + +display = Display() + + +def failed(result): + ''' Test if task result yields failed ''' + if not isinstance(result, MutableMapping): + raise errors.AnsibleFilterError("The 'failed' test expects a dictionary") + return result.get('failed', False) + + +def success(result): + ''' Test if task result yields success ''' + return not failed(result) + + +def unreachable(result): + ''' Test if task result yields unreachable ''' + if not isinstance(result, MutableMapping): + raise errors.AnsibleFilterError("The 'unreachable' test expects a dictionary") + return result.get('unreachable', False) + + +def reachable(result): + ''' Test if task result yields reachable ''' + return not unreachable(result) + + +def changed(result): + ''' Test if task result yields changed ''' + if not isinstance(result, MutableMapping): + raise errors.AnsibleFilterError("The 'changed' test expects a dictionary") + if 'changed' not in result: + changed = False + if ( + 'results' in result and # some modules return a 'results' key + isinstance(result['results'], MutableSequence) and + isinstance(result['results'][0], MutableMapping) + ): + for res in result['results']: + if res.get('changed', False): + changed = True + break + else: + changed = result.get('changed', False) + return changed + + +def skipped(result): + ''' Test if task result yields skipped ''' + if not isinstance(result, MutableMapping): + raise errors.AnsibleFilterError("The 'skipped' test expects a dictionary") + return result.get('skipped', False) + + +def started(result): + ''' Test if async task has started ''' + if not isinstance(result, MutableMapping): + raise errors.AnsibleFilterError("The 'started' test expects a dictionary") + if 'started' in result: + # For async tasks, return status + # NOTE: The value of started is 0 or 1, not False or True :-/ + return result.get('started', 0) == 1 + else: + # For non-async tasks, warn user, but return as if started + display.warning("The 'started' test expects an async task, but a non-async task was tested") + return True + + +def finished(result): + ''' Test if async task has finished ''' + if not isinstance(result, MutableMapping): + raise errors.AnsibleFilterError("The 'finished' test expects a dictionary") + if 'finished' in result: + # For async tasks, return status + # NOTE: The value of finished is 0 or 1, not False or True :-/ + return result.get('finished', 0) == 1 + else: + # For non-async tasks, warn user, but return as if finished + display.warning("The 'finished' test expects an async task, but a non-async task was tested") + return True + + +def regex(value='', pattern='', ignorecase=False, multiline=False, match_type='search'): + ''' Expose `re` as a boolean filter using the `search` method by default. + This is likely only useful for `search` and `match` which already + have their own filters. + ''' + # In addition to ensuring the correct type, to_text here will ensure + # _fail_with_undefined_error happens if the value is Undefined + value = to_text(value, errors='surrogate_or_strict') + flags = 0 + if ignorecase: + flags |= re.I + if multiline: + flags |= re.M + _re = re.compile(pattern, flags=flags) + return bool(getattr(_re, match_type, 'search')(value)) + + +def vault_encrypted(value): + """Evaulate whether a variable is a single vault encrypted value + + .. versionadded:: 2.10 + """ + return getattr(value, '__ENCRYPTED__', False) and value.is_encrypted() + + +def match(value, pattern='', ignorecase=False, multiline=False): + ''' Perform a `re.match` returning a boolean ''' + return regex(value, pattern, ignorecase, multiline, 'match') + + +def search(value, pattern='', ignorecase=False, multiline=False): + ''' Perform a `re.search` returning a boolean ''' + return regex(value, pattern, ignorecase, multiline, 'search') + + +def version_compare(value, version, operator='eq', strict=None, version_type=None): + ''' Perform a version comparison on a value ''' + op_map = { + '==': 'eq', '=': 'eq', 'eq': 'eq', + '<': 'lt', 'lt': 'lt', + '<=': 'le', 'le': 'le', + '>': 'gt', 'gt': 'gt', + '>=': 'ge', 'ge': 'ge', + '!=': 'ne', '<>': 'ne', 'ne': 'ne' + } + + type_map = { + 'loose': LooseVersion, + 'strict': StrictVersion, + 'semver': SemanticVersion, + 'semantic': SemanticVersion, + 'pep440': PEP440Version, + } + + if strict is not None and version_type is not None: + raise errors.AnsibleFilterError("Cannot specify both 'strict' and 'version_type'") + + if not value: + raise errors.AnsibleFilterError("Input version value cannot be empty") + + if not version: + raise errors.AnsibleFilterError("Version parameter to compare against cannot be empty") + + if version_type == 'pep440' and not HAS_PACKAGING: + raise errors.AnsibleFilterError("The pep440 version_type requires the Python 'packaging' library") + + Version = LooseVersion + if strict: + Version = StrictVersion + elif version_type: + try: + Version = type_map[version_type] + except KeyError: + raise errors.AnsibleFilterError( + "Invalid version type (%s). Must be one of %s" % (version_type, ', '.join(map(repr, type_map))) + ) + + if operator in op_map: + operator = op_map[operator] + else: + raise errors.AnsibleFilterError( + 'Invalid operator type (%s). Must be one of %s' % (operator, ', '.join(map(repr, op_map))) + ) + + try: + method = getattr(py_operator, operator) + return method(Version(to_text(value)), Version(to_text(version))) + except Exception as e: + raise errors.AnsibleFilterError('Version comparison failed: %s' % to_native(e)) + + +def truthy(value, convert_bool=False): + """Evaluate as value for truthiness using python ``bool`` + + Optionally, attempt to do a conversion to bool from boolean like values + such as ``"false"``, ``"true"``, ``"yes"``, ``"no"``, ``"on"``, ``"off"``, etc. + + .. versionadded:: 2.10 + """ + if convert_bool: + try: + value = boolean(value) + except TypeError: + pass + + return bool(value) + + +def falsy(value, convert_bool=False): + """Evaluate as value for falsiness using python ``bool`` + + Optionally, attempt to do a conversion to bool from boolean like values + such as ``"false"``, ``"true"``, ``"yes"``, ``"no"``, ``"on"``, ``"off"``, etc. + + .. versionadded:: 2.10 + """ + return not truthy(value, convert_bool=convert_bool) + + +class TestModule(object): + ''' Ansible core jinja2 tests ''' + + def tests(self): + return { + # failure testing + 'failed': failed, + 'failure': failed, + 'succeeded': success, + 'success': success, + 'successful': success, + 'reachable': reachable, + 'unreachable': unreachable, + + # changed testing + 'changed': changed, + 'change': changed, + + # skip testing + 'skipped': skipped, + 'skip': skipped, + + # async testing + 'finished': finished, + 'started': started, + + # regex + 'match': match, + 'search': search, + 'regex': regex, + + # version comparison + 'version_compare': version_compare, + 'version': version_compare, + + # lists + 'any': any, + 'all': all, + + # truthiness + 'truthy': truthy, + 'falsy': falsy, + + # vault + 'vault_encrypted': vault_encrypted, + } diff --git a/lib/ansible/plugins/test/directory.yml b/lib/ansible/plugins/test/directory.yml new file mode 100644 index 0000000..5d7fa78 --- /dev/null +++ b/lib/ansible/plugins/test/directory.yml @@ -0,0 +1,21 @@ +DOCUMENTATION: + name: directory + author: Ansible Core + version_added: "2.5" + short_description: does the path resolve to an existing directory + description: + - Check if the provided path maps to an existing directory on the controller's filesystem (localhost). + options: + _input: + description: A path. + type: path + +EXAMPLES: | + vars: + my_etc_hosts_not_a_dir: "{{ '/etc/hosts' is directory}}" + list_of_files: "{{ list_of_paths | reject('directory') }}" + +RETURN: + _value: + description: Returns C(True) if the path corresponds to an existing directory on the filesystem on the controller, c(False) if otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/exists.yml b/lib/ansible/plugins/test/exists.yml new file mode 100644 index 0000000..85f9108 --- /dev/null +++ b/lib/ansible/plugins/test/exists.yml @@ -0,0 +1,22 @@ +DOCUMENTATION: + name: exists + author: Ansible Core + version_added: "2.5" + short_description: does the path exist, follow symlinks + description: + - Check if the provided path maps to an existing filesystem object on the controller (localhost). + - Follows symlinks and checks the target of the symlink instead of the link itself, use the C(link) or C(link_exists) tests to check on the link. + options: + _input: + description: a path + type: path + +EXAMPLES: | + vars: + my_etc_hosts_exists: "{{ '/etc/hosts' is exist }}" + list_of_local_files_to_copy_to_remote: "{{ list_of_all_possible_files | select('exists') }}" + +RETURN: + _value: + description: Returns C(True) if the path corresponds to an existing filesystem object on the controller (after following symlinks), C(False) if otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/failed.yml b/lib/ansible/plugins/test/failed.yml new file mode 100644 index 0000000..b8a9b3e --- /dev/null +++ b/lib/ansible/plugins/test/failed.yml @@ -0,0 +1,23 @@ +DOCUMENTATION: + name: failed + author: Ansible Core + version_added: "1.9" + short_description: did the task fail + aliases: [failure] + description: + - Tests if task finished in failure, opposite of C(succeeded). + - This test checks for the existance of a C(failed) key in the input dictionary and that it is C(True) if present. + - Tasks that get skipped or not executed due to other failures (syntax, templating, unreachable host, etc) do not return a 'failed' status. + options: + _input: + description: registered result from an Ansible task + type: dictionary + required: True +EXAMPLES: | + # test 'status' to know how to respond + {{ taskresults is failed }} + +RETURN: + _value: + description: Returns C(True) if the task was failed, C(False) otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/failure.yml b/lib/ansible/plugins/test/failure.yml new file mode 100644 index 0000000..b8a9b3e --- /dev/null +++ b/lib/ansible/plugins/test/failure.yml @@ -0,0 +1,23 @@ +DOCUMENTATION: + name: failed + author: Ansible Core + version_added: "1.9" + short_description: did the task fail + aliases: [failure] + description: + - Tests if task finished in failure, opposite of C(succeeded). + - This test checks for the existance of a C(failed) key in the input dictionary and that it is C(True) if present. + - Tasks that get skipped or not executed due to other failures (syntax, templating, unreachable host, etc) do not return a 'failed' status. + options: + _input: + description: registered result from an Ansible task + type: dictionary + required: True +EXAMPLES: | + # test 'status' to know how to respond + {{ taskresults is failed }} + +RETURN: + _value: + description: Returns C(True) if the task was failed, C(False) otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/falsy.yml b/lib/ansible/plugins/test/falsy.yml new file mode 100644 index 0000000..49a198f --- /dev/null +++ b/lib/ansible/plugins/test/falsy.yml @@ -0,0 +1,24 @@ +DOCUMENTATION: + name: falsy + author: Ansible Core + version_added: "2.10" + short_description: Pythonic false + description: + - This check is a more Python version of what is 'false'. + - It is the opposite of 'truthy'. + options: + _input: + description: An expression that can be expressed in a boolean context. + type: string + required: True + convert_bool: + description: Attempts to convert the result to a strict Python boolean vs normally acceptable values (C(yes)/C(no), C(on)/C(off), C(0)/C(1), etc). + type: bool + default: false +EXAMPLES: | + thisisfalse: '{{ "any string" is falsy }}' + thisistrue: '{{ "" is falsy }}' +RETURN: + _value: + description: Returns C(False) if the condition is not "Python truthy", C(True) otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/file.yml b/lib/ansible/plugins/test/file.yml new file mode 100644 index 0000000..8b79c07 --- /dev/null +++ b/lib/ansible/plugins/test/file.yml @@ -0,0 +1,22 @@ +DOCUMENTATION: + name: file + author: Ansible Core + version_added: "2.5" + short_description: does the path resolve to an existing file + description: + - Check if the provided path maps to an existing file on the controller's filesystem (localhost) + aliases: [is_file] + options: + _input: + description: A path. + type: path + +EXAMPLES: | + vars: + my_etc_hosts_is_a_file: "{{ '/etc/hosts' is file }}" + list_of_files: "{{ list_of_paths | select('file') }}" + +RETURN: + _value: + description: Returns C(True) if the path corresponds to an existing file on the filesystem on the controller, C(False) if otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/files.py b/lib/ansible/plugins/test/files.py new file mode 100644 index 0000000..35761a4 --- /dev/null +++ b/lib/ansible/plugins/test/files.py @@ -0,0 +1,48 @@ +# (c) 2015, Ansible, Inc +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from os.path import isdir, isfile, isabs, exists, lexists, islink, samefile, ismount +from ansible import errors + + +class TestModule(object): + ''' Ansible file jinja2 tests ''' + + def tests(self): + return { + # file testing + 'directory': isdir, + 'is_dir': isdir, + 'file': isfile, + 'is_file': isfile, + 'link': islink, + 'is_link': islink, + 'exists': exists, + 'link_exists': lexists, + + # path testing + 'abs': isabs, + 'is_abs': isabs, + 'same_file': samefile, + 'is_same_file': samefile, + 'mount': ismount, + 'is_mount': ismount, + } diff --git a/lib/ansible/plugins/test/finished.yml b/lib/ansible/plugins/test/finished.yml new file mode 100644 index 0000000..b01b132 --- /dev/null +++ b/lib/ansible/plugins/test/finished.yml @@ -0,0 +1,21 @@ +DOCUMENTATION: + name: finished + author: Ansible Core + version_added: "1.9" + short_description: Did async task finish + description: + - Used to test if an async task has finished, it will aslo work with normal tasks but will issue a warning. + - This test checks for the existance of a C(finished) key in the input dictionary and that it is C(1) if present + options: + _input: + description: registered result from an Ansible task + type: dictionary + required: True +EXAMPLES: | + # test 'status' to know how to respond + {{ (asynctaskpoll is finished}} + +RETURN: + _value: + description: Returns C(True) if the aysnc task has finished, C(False) otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/is_abs.yml b/lib/ansible/plugins/test/is_abs.yml new file mode 100644 index 0000000..46f7f70 --- /dev/null +++ b/lib/ansible/plugins/test/is_abs.yml @@ -0,0 +1,23 @@ +DOCUMENTATION: + name: abs + author: Ansible Core + version_added: "2.5" + short_description: is the path absolute + aliases: [is_abs] + description: + - Check if the provided path is absolute, not relative. + - An absolute path expresses the location of a filesystem object starting at the filesystem root and requires no context. + - A relative path does not start at the filesystem root and requires a 'current' directory as a context to resolve. + options: + _input: + description: A path. + type: path + +EXAMPLES: | + is_path_absolute: "{{ '/etc/hosts' is abs }}}" + relative_paths: "{{ all_paths | reject('abs') }}" + +RETURN: + _value: + description: Returns C(True) if the path is absolute, C(False) if it is relative. + type: boolean diff --git a/lib/ansible/plugins/test/is_dir.yml b/lib/ansible/plugins/test/is_dir.yml new file mode 100644 index 0000000..5d7fa78 --- /dev/null +++ b/lib/ansible/plugins/test/is_dir.yml @@ -0,0 +1,21 @@ +DOCUMENTATION: + name: directory + author: Ansible Core + version_added: "2.5" + short_description: does the path resolve to an existing directory + description: + - Check if the provided path maps to an existing directory on the controller's filesystem (localhost). + options: + _input: + description: A path. + type: path + +EXAMPLES: | + vars: + my_etc_hosts_not_a_dir: "{{ '/etc/hosts' is directory}}" + list_of_files: "{{ list_of_paths | reject('directory') }}" + +RETURN: + _value: + description: Returns C(True) if the path corresponds to an existing directory on the filesystem on the controller, c(False) if otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/is_file.yml b/lib/ansible/plugins/test/is_file.yml new file mode 100644 index 0000000..8b79c07 --- /dev/null +++ b/lib/ansible/plugins/test/is_file.yml @@ -0,0 +1,22 @@ +DOCUMENTATION: + name: file + author: Ansible Core + version_added: "2.5" + short_description: does the path resolve to an existing file + description: + - Check if the provided path maps to an existing file on the controller's filesystem (localhost) + aliases: [is_file] + options: + _input: + description: A path. + type: path + +EXAMPLES: | + vars: + my_etc_hosts_is_a_file: "{{ '/etc/hosts' is file }}" + list_of_files: "{{ list_of_paths | select('file') }}" + +RETURN: + _value: + description: Returns C(True) if the path corresponds to an existing file on the filesystem on the controller, C(False) if otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/is_link.yml b/lib/ansible/plugins/test/is_link.yml new file mode 100644 index 0000000..27af41f --- /dev/null +++ b/lib/ansible/plugins/test/is_link.yml @@ -0,0 +1,21 @@ +DOCUMENTATION: + name: link + author: Ansible Core + version_added: "2.5" + short_description: does the path reference existing symbolic link + aliases: [islink] + description: + - Check if the provided path maps to an existing symlink on the controller's filesystem (localhost). + options: + _input: + description: A path. + type: path + +EXAMPLES: | + ismyhostsalink: "{{ '/etc/hosts' is link}}" + list_of_symlinks: "{{ list_of_paths | select('link') }}" + +RETURN: + _value: + description: Returns C(True) if the path corresponds to an existing symlink on the filesystem on the controller, C(False) if otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/is_mount.yml b/lib/ansible/plugins/test/is_mount.yml new file mode 100644 index 0000000..23f19b6 --- /dev/null +++ b/lib/ansible/plugins/test/is_mount.yml @@ -0,0 +1,22 @@ +DOCUMENTATION: + name: mount + author: Ansible Core + version_added: "2.5" + short_description: does the path resolve to mount point + description: + - Check if the provided path maps to a filesystem mount point on the controller (localhost). + aliases: [is_mount] + options: + _input: + description: A path. + type: path + +EXAMPLES: | + vars: + ihopefalse: "{{ '/etc/hosts' is mount }}" + normallytrue: "{{ '/tmp' is mount }}" + +RETURN: + _value: + description: Returns C(True) if the path corresponds to a mount point on the controller, C(False) if otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/is_same_file.yml b/lib/ansible/plugins/test/is_same_file.yml new file mode 100644 index 0000000..a10a36a --- /dev/null +++ b/lib/ansible/plugins/test/is_same_file.yml @@ -0,0 +1,24 @@ +DOCUMENTATION: + name: same_file + author: Ansible Core + version_added: "2.5" + short_description: compares two paths to see if they resolve to the same filesystem object + description: Check if the provided paths map to the same location on the controller's filesystem (localhost). + aliases: [is_file] + options: + _input: + description: A path. + type: path + required: true + _path2: + description: Another path. + type: path + required: true + +EXAMPLES: | + amionelevelfromroot: "{{ '/etc/hosts' is same_file('../etc/hosts') }}" + +RETURN: + _value: + description: Returns C(True) if the paths correspond to the same location on the filesystem on the controller, C(False) if otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/isnan.yml b/lib/ansible/plugins/test/isnan.yml new file mode 100644 index 0000000..3c1055b --- /dev/null +++ b/lib/ansible/plugins/test/isnan.yml @@ -0,0 +1,20 @@ +DOCUMENTATION: + name: nan + author: Ansible Core + version_added: "2.5" + short_description: is this not a number (NaN) + description: + - Whether the input is a special floating point number called L(not a number, https://en.wikipedia.org/wiki/NaN). + aliases: [is_file] + options: + _input: + description: Possible number representation or string that can be converted into one. + type: raw + required: true +EXAMPLES: | + isnan: "{{ '42' is nan }}" + +RETURN: + _value: + description: Returns C(True) if the input is NaN, C(False) if otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/issubset.yml b/lib/ansible/plugins/test/issubset.yml new file mode 100644 index 0000000..d57d05b --- /dev/null +++ b/lib/ansible/plugins/test/issubset.yml @@ -0,0 +1,28 @@ +DOCUMENTATION: + name: subset + author: Ansible Core + version_added: "2.4" + aliases: [issubset] + short_description: is the list a subset of this other list + description: + - Validate if the first list is a sub set (is included) of the second list. + - Same as the C(all) Python function. + options: + _input: + description: List. + type: list + elements: raw + required: True + _superset: + description: List to test against. + type: list + elements: raw + required: True +EXAMPLES: | + big: [1,2,3,4,5] + sml: [3,4] + issmallinbig: '{{ small is subset(big) }}' +RETURN: + _value: + description: Returns C(True) if the specified list is a subset of the provided list, C(False) otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/issuperset.yml b/lib/ansible/plugins/test/issuperset.yml new file mode 100644 index 0000000..72be3d5 --- /dev/null +++ b/lib/ansible/plugins/test/issuperset.yml @@ -0,0 +1,28 @@ +DOCUMENTATION: + name: superset + author: Ansible Core + version_added: "2.4" + short_description: is the list a superset of this other list + aliases: [issuperset] + description: + - Validate if the first list is a super set (includes) the second list. + - Same as the C(all) Python function. + options: + _input: + description: List. + type: list + elements: raw + required: True + _subset: + description: List to test against. + type: list + elements: raw + required: True +EXAMPLES: | + big: [1,2,3,4,5] + sml: [3,4] + issmallinbig: '{{ big is superset(small) }}' +RETURN: + _value: + description: Returns C(True) if the specified list is a superset of the provided list, C(False) otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/link.yml b/lib/ansible/plugins/test/link.yml new file mode 100644 index 0000000..27af41f --- /dev/null +++ b/lib/ansible/plugins/test/link.yml @@ -0,0 +1,21 @@ +DOCUMENTATION: + name: link + author: Ansible Core + version_added: "2.5" + short_description: does the path reference existing symbolic link + aliases: [islink] + description: + - Check if the provided path maps to an existing symlink on the controller's filesystem (localhost). + options: + _input: + description: A path. + type: path + +EXAMPLES: | + ismyhostsalink: "{{ '/etc/hosts' is link}}" + list_of_symlinks: "{{ list_of_paths | select('link') }}" + +RETURN: + _value: + description: Returns C(True) if the path corresponds to an existing symlink on the filesystem on the controller, C(False) if otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/link_exists.yml b/lib/ansible/plugins/test/link_exists.yml new file mode 100644 index 0000000..f75a699 --- /dev/null +++ b/lib/ansible/plugins/test/link_exists.yml @@ -0,0 +1,21 @@ +DOCUMENTATION: + name: link_exists + author: Ansible Core + version_added: "2.5" + short_description: does the path exist, no follow + description: + - Check if the provided path maps to an existing symlink on the controller's filesystem (localhost). + - Does not follow symlinks, so it only verifies that the link itself exists. + options: + _input: + description: A path. + type: path + +EXAMPLES: | + ismyhostsalink: "{{ '/etc/hosts' is link_exists}}" + list_of_symlinks: "{{ list_of_paths | select('link_exists') }}" + +RETURN: + _value: + description: Returns C(True) if the path corresponds to an existing filesystem object on the controller, C(False) if otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/match.yml b/lib/ansible/plugins/test/match.yml new file mode 100644 index 0000000..ecb4ae6 --- /dev/null +++ b/lib/ansible/plugins/test/match.yml @@ -0,0 +1,32 @@ +DOCUMENTATION: + name: match + author: Ansible Core + short_description: Does string match regular expression from the start + description: + - Compare string against regular expression using Python's match function, + this means the regex is automatically anchored at the start of the string. + options: + _input: + description: String to match. + type: string + required: True + pattern: + description: Regex to match against. + type: string + required: True + ignorecase: + description: Use case insenstive matching. + type: boolean + default: False + multiline: + description: Match against mulitple lines in string. + type: boolean + default: False +EXAMPLES: | + url: "https://example.com/users/foo/resources/bar" + foundmatch: url is match("https://example.com/users/.*/resources") + nomatch: url is match("/users/.*/resources") +RETURN: + _value: + description: Returns C(True) if there is a match, C(False) otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/mathstuff.py b/lib/ansible/plugins/test/mathstuff.py new file mode 100644 index 0000000..9a3f467 --- /dev/null +++ b/lib/ansible/plugins/test/mathstuff.py @@ -0,0 +1,62 @@ +# (c) 2016, Ansible, Inc +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import math + + +def issubset(a, b): + return set(a) <= set(b) + + +def issuperset(a, b): + return set(a) >= set(b) + + +def isnotanumber(x): + try: + return math.isnan(x) + except TypeError: + return False + + +def contains(seq, value): + '''Opposite of the ``in`` test, allowing use as a test in filters like ``selectattr`` + + .. versionadded:: 2.8 + ''' + return value in seq + + +class TestModule: + ''' Ansible math jinja2 tests ''' + + def tests(self): + return { + # set theory + 'subset': issubset, + 'issubset': issubset, + 'superset': issuperset, + 'issuperset': issuperset, + 'contains': contains, + + # numbers + 'nan': isnotanumber, + 'isnan': isnotanumber, + } diff --git a/lib/ansible/plugins/test/mount.yml b/lib/ansible/plugins/test/mount.yml new file mode 100644 index 0000000..23f19b6 --- /dev/null +++ b/lib/ansible/plugins/test/mount.yml @@ -0,0 +1,22 @@ +DOCUMENTATION: + name: mount + author: Ansible Core + version_added: "2.5" + short_description: does the path resolve to mount point + description: + - Check if the provided path maps to a filesystem mount point on the controller (localhost). + aliases: [is_mount] + options: + _input: + description: A path. + type: path + +EXAMPLES: | + vars: + ihopefalse: "{{ '/etc/hosts' is mount }}" + normallytrue: "{{ '/tmp' is mount }}" + +RETURN: + _value: + description: Returns C(True) if the path corresponds to a mount point on the controller, C(False) if otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/nan.yml b/lib/ansible/plugins/test/nan.yml new file mode 100644 index 0000000..3c1055b --- /dev/null +++ b/lib/ansible/plugins/test/nan.yml @@ -0,0 +1,20 @@ +DOCUMENTATION: + name: nan + author: Ansible Core + version_added: "2.5" + short_description: is this not a number (NaN) + description: + - Whether the input is a special floating point number called L(not a number, https://en.wikipedia.org/wiki/NaN). + aliases: [is_file] + options: + _input: + description: Possible number representation or string that can be converted into one. + type: raw + required: true +EXAMPLES: | + isnan: "{{ '42' is nan }}" + +RETURN: + _value: + description: Returns C(True) if the input is NaN, C(False) if otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/reachable.yml b/lib/ansible/plugins/test/reachable.yml new file mode 100644 index 0000000..8cb1ce3 --- /dev/null +++ b/lib/ansible/plugins/test/reachable.yml @@ -0,0 +1,21 @@ +DOCUMENTATION: + name: reachable + author: Ansible Core + version_added: "1.9" + short_description: Task did not end due to unreachable host + description: + - Tests if task was able to reach the host for execution + - This test checks for the existance of a C(unreachable) key in the input dictionary and that it is C(False) if present + options: + _input: + description: registered result from an Ansible task + type: dictionary + required: True +EXAMPLES: | + # test 'status' to know how to respond + {{ (taskresults is reachable }} + +RETURN: + _value: + description: Returns C(True) if the task did not flag the host as unreachable, C(False) otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/regex.yml b/lib/ansible/plugins/test/regex.yml new file mode 100644 index 0000000..90ca786 --- /dev/null +++ b/lib/ansible/plugins/test/regex.yml @@ -0,0 +1,37 @@ +DOCUMENTATION: + name: regex + author: Ansible Core + short_description: Does string match regular expression from the start + description: + - Compare string against regular expression using Python's match or search functions. + options: + _input: + description: String to match. + type: string + required: True + pattern: + description: Regex to match against. + type: string + required: True + ignorecase: + description: Use case insenstive matching. + type: boolean + default: False + multiline: + description: Match against multiple lines in string. + type: boolean + default: False + match_type: + description: Decide which function to be used to do the matching. + type: string + choices: [match, search] + default: search + +EXAMPLES: | + url: "https://example.com/users/foo/resources/bar" + foundmatch: url is regex("example\.com/\w+/foo") + +RETURN: + _value: + description: Returns C(True) if there is a match, C(False) otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/same_file.yml b/lib/ansible/plugins/test/same_file.yml new file mode 100644 index 0000000..a10a36a --- /dev/null +++ b/lib/ansible/plugins/test/same_file.yml @@ -0,0 +1,24 @@ +DOCUMENTATION: + name: same_file + author: Ansible Core + version_added: "2.5" + short_description: compares two paths to see if they resolve to the same filesystem object + description: Check if the provided paths map to the same location on the controller's filesystem (localhost). + aliases: [is_file] + options: + _input: + description: A path. + type: path + required: true + _path2: + description: Another path. + type: path + required: true + +EXAMPLES: | + amionelevelfromroot: "{{ '/etc/hosts' is same_file('../etc/hosts') }}" + +RETURN: + _value: + description: Returns C(True) if the paths correspond to the same location on the filesystem on the controller, C(False) if otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/search.yml b/lib/ansible/plugins/test/search.yml new file mode 100644 index 0000000..4578bde --- /dev/null +++ b/lib/ansible/plugins/test/search.yml @@ -0,0 +1,33 @@ +DOCUMENTATION: + name: search + author: Ansible Core + short_description: Does string match a regular expression + description: + - Compare string against regular expression using Python's C(search) function. + options: + _input: + description: String to match. + type: string + required: True + pattern: + description: Regex to match against. + type: string + required: True + ignorecase: + description: Use case insenstive matching. + type: boolean + default: False + multiline: + description: Match against mulitple lines in string. + type: boolean + default: False + +EXAMPLES: | + url: "https://example.com/users/foo/resources/bar" + foundmatch: url is search("https://example.com/users/.*/resources") + alsomatch: url is search("users/.*/resources") + +RETURN: + _value: + description: Returns C(True) if there is a match, C(False) otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/skip.yml b/lib/ansible/plugins/test/skip.yml new file mode 100644 index 0000000..9727172 --- /dev/null +++ b/lib/ansible/plugins/test/skip.yml @@ -0,0 +1,22 @@ +DOCUMENTATION: + name: skipped + author: Ansible Core + version_added: "1.9" + short_description: Was task skipped + aliases: [skip] + description: + - Tests if task was skipped + - This test checks for the existance of a C(skipped) key in the input dictionary and that it is C(True) if present + options: + _input: + description: registered result from an Ansible task + type: dictionary + required: True +EXAMPLES: | + # test 'status' to know how to respond + {{ (taskresults is skipped}} + +RETURN: + _value: + description: Returns C(True) if the task was skipped, C(False) otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/skipped.yml b/lib/ansible/plugins/test/skipped.yml new file mode 100644 index 0000000..9727172 --- /dev/null +++ b/lib/ansible/plugins/test/skipped.yml @@ -0,0 +1,22 @@ +DOCUMENTATION: + name: skipped + author: Ansible Core + version_added: "1.9" + short_description: Was task skipped + aliases: [skip] + description: + - Tests if task was skipped + - This test checks for the existance of a C(skipped) key in the input dictionary and that it is C(True) if present + options: + _input: + description: registered result from an Ansible task + type: dictionary + required: True +EXAMPLES: | + # test 'status' to know how to respond + {{ (taskresults is skipped}} + +RETURN: + _value: + description: Returns C(True) if the task was skipped, C(False) otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/started.yml b/lib/ansible/plugins/test/started.yml new file mode 100644 index 0000000..0cb0602 --- /dev/null +++ b/lib/ansible/plugins/test/started.yml @@ -0,0 +1,21 @@ +DOCUMENTATION: + name: started + author: Ansible Core + version_added: "1.9" + short_description: Was async task started + description: + - Used to check if an async task has started, will also work with non async tasks but will issue a warning. + - This test checks for the existance of a C(started) key in the input dictionary and that it is C(1) if present + options: + _input: + description: registered result from an Ansible task + type: dictionary + required: True +EXAMPLES: | + # test 'status' to know how to respond + {{ (asynctaskpoll is started}} + +RETURN: + _value: + description: Returns C(True) if the task has started, C(False) otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/subset.yml b/lib/ansible/plugins/test/subset.yml new file mode 100644 index 0000000..d57d05b --- /dev/null +++ b/lib/ansible/plugins/test/subset.yml @@ -0,0 +1,28 @@ +DOCUMENTATION: + name: subset + author: Ansible Core + version_added: "2.4" + aliases: [issubset] + short_description: is the list a subset of this other list + description: + - Validate if the first list is a sub set (is included) of the second list. + - Same as the C(all) Python function. + options: + _input: + description: List. + type: list + elements: raw + required: True + _superset: + description: List to test against. + type: list + elements: raw + required: True +EXAMPLES: | + big: [1,2,3,4,5] + sml: [3,4] + issmallinbig: '{{ small is subset(big) }}' +RETURN: + _value: + description: Returns C(True) if the specified list is a subset of the provided list, C(False) otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/succeeded.yml b/lib/ansible/plugins/test/succeeded.yml new file mode 100644 index 0000000..4626f9f --- /dev/null +++ b/lib/ansible/plugins/test/succeeded.yml @@ -0,0 +1,22 @@ +DOCUMENTATION: + name: success + author: Ansible Core + version_added: "1.9" + short_description: check task success + aliases: [succeeded, successful] + description: + - Tests if task finished successfully, opposite of C(failed). + - This test checks for the existance of a C(failed) key in the input dictionary and that it is C(False) if present + options: + _input: + description: registered result from an Ansible task + type: dictionary + required: True +EXAMPLES: | + # test 'status' to know how to respond + {{ (taskresults is success }} + +RETURN: + _value: + description: Returns C(True) if the task was successfully completed, C(False) otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/success.yml b/lib/ansible/plugins/test/success.yml new file mode 100644 index 0000000..4626f9f --- /dev/null +++ b/lib/ansible/plugins/test/success.yml @@ -0,0 +1,22 @@ +DOCUMENTATION: + name: success + author: Ansible Core + version_added: "1.9" + short_description: check task success + aliases: [succeeded, successful] + description: + - Tests if task finished successfully, opposite of C(failed). + - This test checks for the existance of a C(failed) key in the input dictionary and that it is C(False) if present + options: + _input: + description: registered result from an Ansible task + type: dictionary + required: True +EXAMPLES: | + # test 'status' to know how to respond + {{ (taskresults is success }} + +RETURN: + _value: + description: Returns C(True) if the task was successfully completed, C(False) otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/successful.yml b/lib/ansible/plugins/test/successful.yml new file mode 100644 index 0000000..4626f9f --- /dev/null +++ b/lib/ansible/plugins/test/successful.yml @@ -0,0 +1,22 @@ +DOCUMENTATION: + name: success + author: Ansible Core + version_added: "1.9" + short_description: check task success + aliases: [succeeded, successful] + description: + - Tests if task finished successfully, opposite of C(failed). + - This test checks for the existance of a C(failed) key in the input dictionary and that it is C(False) if present + options: + _input: + description: registered result from an Ansible task + type: dictionary + required: True +EXAMPLES: | + # test 'status' to know how to respond + {{ (taskresults is success }} + +RETURN: + _value: + description: Returns C(True) if the task was successfully completed, C(False) otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/superset.yml b/lib/ansible/plugins/test/superset.yml new file mode 100644 index 0000000..72be3d5 --- /dev/null +++ b/lib/ansible/plugins/test/superset.yml @@ -0,0 +1,28 @@ +DOCUMENTATION: + name: superset + author: Ansible Core + version_added: "2.4" + short_description: is the list a superset of this other list + aliases: [issuperset] + description: + - Validate if the first list is a super set (includes) the second list. + - Same as the C(all) Python function. + options: + _input: + description: List. + type: list + elements: raw + required: True + _subset: + description: List to test against. + type: list + elements: raw + required: True +EXAMPLES: | + big: [1,2,3,4,5] + sml: [3,4] + issmallinbig: '{{ big is superset(small) }}' +RETURN: + _value: + description: Returns C(True) if the specified list is a superset of the provided list, C(False) otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/truthy.yml b/lib/ansible/plugins/test/truthy.yml new file mode 100644 index 0000000..01d5255 --- /dev/null +++ b/lib/ansible/plugins/test/truthy.yml @@ -0,0 +1,24 @@ +DOCUMENTATION: + name: truthy + author: Ansible Core + version_added: "2.10" + short_description: Pythonic true + description: + - This check is a more Python version of what is 'true'. + - It is the opposite of C(falsy). + options: + _input: + description: An expression that can be expressed in a boolean context. + type: string + required: True + convert_bool: + description: Attempts to convert to strict python boolean vs normally acceptable values (C(yes)/C(no), C(on)/C(off), C(0)/C(1), etc). + type: bool + default: false +EXAMPLES: | + thisistrue: '{{ "any string" is truthy }}' + thisisfalse: '{{ "" is truthy }}' +RETURN: + _value: + description: Returns C(True) if the condition is not "Python truthy", C(False) otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/unreachable.yml b/lib/ansible/plugins/test/unreachable.yml new file mode 100644 index 0000000..ed6c17e --- /dev/null +++ b/lib/ansible/plugins/test/unreachable.yml @@ -0,0 +1,21 @@ +DOCUMENTATION: + name: unreachable + author: Ansible Core + version_added: "1.9" + short_description: Did task end due to the host was unreachable + description: + - Tests if task was not able to reach the host for execution + - This test checks for the existance of a C(unreachable) key in the input dictionary and that it's value is C(True) + options: + _input: + description: registered result from an Ansible task + type: dictionary + required: True +EXAMPLES: | + # test 'status' to know how to respond + {{ (taskresults is unreachable }} + +RETURN: + _value: + description: Returns C(True) if the task flagged the host as unreachable, C(False) otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/uri.py b/lib/ansible/plugins/test/uri.py new file mode 100644 index 0000000..7ef3381 --- /dev/null +++ b/lib/ansible/plugins/test/uri.py @@ -0,0 +1,46 @@ +# (c) Ansible Project + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from urllib.parse import urlparse + + +def is_uri(value, schemes=None): + ''' Will verify that the string passed is a valid 'URI', if given a list of valid schemes it will match those ''' + try: + x = urlparse(value) + isit = all([x.scheme is not None, x.path is not None, not schemes or x.scheme in schemes]) + except Exception as e: + isit = False + return isit + + +def is_url(value, schemes=None): + ''' Will verify that the string passed is a valid 'URL' ''' + + isit = is_uri(value, schemes) + if isit: + try: + x = urlparse(value) + isit = bool(x.netloc or x.scheme == 'file') + except Exception as e: + isit = False + return isit + + +def is_urn(value): + return is_uri(value, ['urn']) + + +class TestModule(object): + ''' Ansible URI jinja2 test ''' + + def tests(self): + return { + # file testing + 'uri': is_uri, + 'url': is_url, + 'urn': is_urn, + } diff --git a/lib/ansible/plugins/test/uri.yml b/lib/ansible/plugins/test/uri.yml new file mode 100644 index 0000000..bb3b8bd --- /dev/null +++ b/lib/ansible/plugins/test/uri.yml @@ -0,0 +1,30 @@ +DOCUMENTATION: + name: uri + author: Ansible Core + version_added: "2.14" + short_description: is the string a valid URI + description: + - Validates that the input string conforms to the URI standard, optionally that is also in the list of schemas provided. + options: + _input: + description: Possible URI. + type: string + required: True + schemes: + description: Subset of URI schemas to validate against, otherwise B(any) scheme is considered valid. + type: list + elements: string + required: False +EXAMPLES: | + # URLs are URIs + {{ 'http://example.com' is uri }} + # but not all URIs are URLs + {{ 'mailto://nowone@example.com' is uri }} + # looking only for file transfers URIs + {{ 'mailto://nowone@example.com' is not uri(schemes=['ftp', 'ftps', 'sftp', 'file']) }} + # make sure URL conforms to the 'special schemas' + {{ 'http://nobody:secret@example.com' is uri(['ftp', 'ftps', 'http', 'https', 'ws', 'wss']) }} +RETURN: + _value: + description: Returns C(false) if the string is not a URI or the schema extracted does not match the supplied list. + type: boolean diff --git a/lib/ansible/plugins/test/url.yml b/lib/ansible/plugins/test/url.yml new file mode 100644 index 0000000..36b6c77 --- /dev/null +++ b/lib/ansible/plugins/test/url.yml @@ -0,0 +1,29 @@ +DOCUMENTATION: + name: url + author: Ansible Core + version_added: "2.14" + short_description: is the string a valid URL + description: + - Validates a string to conform to the URL standard. + options: + _input: + description: Possible URL. + type: string + required: True + schemes: + description: Subset of URI schemas to validate against, otherwise B(any) scheme is considered valid. + type: list + elements: string +EXAMPLES: | + # simple URL + {{ 'http://example.com' is url }} + # looking only for file transfers URIs + {{ 'mailto://nowone@example.com' is not uri(schemes=['ftp', 'ftps', 'sftp', 'file']) }} + # but it is according to standard + {{ 'mailto://nowone@example.com' is not uri }} + # more complex URL + {{ 'ftp://admin:secret@example.com/path/to/myfile.yml' is url }} +RETURN: + _value: + description: Returns C(false) if the string is not a URL, C(true) otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/urn.yml b/lib/ansible/plugins/test/urn.yml new file mode 100644 index 0000000..81a6686 --- /dev/null +++ b/lib/ansible/plugins/test/urn.yml @@ -0,0 +1,21 @@ +DOCUMENTATION: + name: urn + author: Ansible Core + version_added: "2.14" + short_description: is the string a valid URN + description: + - Validates that the input string conforms to the URN standard. + options: + _input: + description: Possible URN. + type: string + required: True +EXAMPLES: | + # ISBN in URN format + {{ 'urn:isbn:9780302376463' is urn }} + # this is URL/URI but not URN + {{ 'mailto://nowone@example.com' is not urn }} +RETURN: + _value: + description: Returns C(true) if the string is a URN and C(false) if it is not. + type: boolean diff --git a/lib/ansible/plugins/test/vault_encrypted.yml b/lib/ansible/plugins/test/vault_encrypted.yml new file mode 100644 index 0000000..58d79f1 --- /dev/null +++ b/lib/ansible/plugins/test/vault_encrypted.yml @@ -0,0 +1,19 @@ +DOCUMENTATION: + name: truthy + author: Ansible Core + version_added: "2.10" + short_description: Is this an encrypted vault + description: + - Verifies if the input is an Ansible vault. + options: + _input: + description: The possible vault. + type: string + required: True +EXAMPLES: | + thisisfalse: '{{ "any string" is ansible_vault }}' + thisistrue: '{{ "$ANSIBLE_VAULT;1.2;AES256;dev...." is ansible_vault }}' +RETURN: + _value: + description: Returns C(True) if the input is a valid ansible vault, C(False) otherwise. + type: boolean diff --git a/lib/ansible/plugins/test/version.yml b/lib/ansible/plugins/test/version.yml new file mode 100644 index 0000000..92b6048 --- /dev/null +++ b/lib/ansible/plugins/test/version.yml @@ -0,0 +1,82 @@ +DOCUMENTATION: + name: version + author: Ansible Core + version_added: "1.6" + short_description: compare version strings + aliases: [version_compare] + description: + - Compare version strings using various versioning schemes + options: + _input: + description: Left hand version to compare + type: string + required: True + version: + description: Right hand version to compare + type: string + required: True + operator: + description: Comparison operator + type: string + required: False + choices: + - == + - '=' + - eq + - < + - lt + - <= + - le + - '>' + - gt + - '>=' + - ge + - '!=' + - <> + - ne + default: eq + strict: + description: Whether to use strict version scheme. Mutually exclusive with C(version_type) + type: boolean + required: False + default: False + version_type: + description: Version scheme to use for comparison. Mutually exclusive with C(strict). See C(notes) for descriptions on the version types. + type: string + required: False + choices: + - loose + - strict + - semver + - semantic + - pep440 + default: loose + notes: + - C(loose) - This type corresponds to the Python C(distutils.version.LooseVersion) class. All version formats are valid for this type. The rules for comparison are simple and predictable, but may not always give expected results. + - C(strict) - This type corresponds to the Python C(distutils.version.StrictVersion) class. A version number consists of two or three dot-separated numeric components, with an optional "pre-release" tag on the end. The pre-release tag consists of a single letter C(a) or C(b) followed by a number. If the numeric components of two version numbers are equal, then one with a pre-release tag will always be deemed earlier (lesser) than one without. + - C(semver)/C(semantic) - This type implements the L(Semantic Version,https://semver.org) scheme for version comparison. + - C(pep440) - This type implements the Python L(PEP-440,https://peps.python.org/pep-0440/) versioning rules for version comparison. Added in version 2.14. +EXAMPLES: | + - name: version test examples + assert: + that: + - "'1.0' is version_compare('1.0', '==')" # old name + - "'1.0' is version('1.0', '==')" + - "'1.0' is version('2.0', '!=')" + - "'1.0' is version('2.0', '<')" + - "'2.0' is version('1.0', '>')" + - "'1.0' is version('1.0', '<=')" + - "'1.0' is version('1.0', '>=')" + - "'1.0' is version_compare('1.0', '==', strict=true)" # old name + - "'1.0' is version('1.0', '==', strict=true)" + - "'1.0' is version('2.0', '!=', strict=true)" + - "'1.0' is version('2.0', '<', strict=true)" + - "'2.0' is version('1.0', '>', strict=true)" + - "'1.0' is version('1.0', '<=', strict=true)" + - "'1.0' is version('1.0', '>=', strict=true)" + - "'1.2.3' is version('2.0.0', 'lt', version_type='semver')" + - "'2.14.0rc1' is version('2.14.0', 'lt', version_type='pep440')" +RETURN: + _value: + description: Returns C(True) or C(False) depending on the outcome of the comparison. + type: boolean diff --git a/lib/ansible/plugins/test/version_compare.yml b/lib/ansible/plugins/test/version_compare.yml new file mode 100644 index 0000000..92b6048 --- /dev/null +++ b/lib/ansible/plugins/test/version_compare.yml @@ -0,0 +1,82 @@ +DOCUMENTATION: + name: version + author: Ansible Core + version_added: "1.6" + short_description: compare version strings + aliases: [version_compare] + description: + - Compare version strings using various versioning schemes + options: + _input: + description: Left hand version to compare + type: string + required: True + version: + description: Right hand version to compare + type: string + required: True + operator: + description: Comparison operator + type: string + required: False + choices: + - == + - '=' + - eq + - < + - lt + - <= + - le + - '>' + - gt + - '>=' + - ge + - '!=' + - <> + - ne + default: eq + strict: + description: Whether to use strict version scheme. Mutually exclusive with C(version_type) + type: boolean + required: False + default: False + version_type: + description: Version scheme to use for comparison. Mutually exclusive with C(strict). See C(notes) for descriptions on the version types. + type: string + required: False + choices: + - loose + - strict + - semver + - semantic + - pep440 + default: loose + notes: + - C(loose) - This type corresponds to the Python C(distutils.version.LooseVersion) class. All version formats are valid for this type. The rules for comparison are simple and predictable, but may not always give expected results. + - C(strict) - This type corresponds to the Python C(distutils.version.StrictVersion) class. A version number consists of two or three dot-separated numeric components, with an optional "pre-release" tag on the end. The pre-release tag consists of a single letter C(a) or C(b) followed by a number. If the numeric components of two version numbers are equal, then one with a pre-release tag will always be deemed earlier (lesser) than one without. + - C(semver)/C(semantic) - This type implements the L(Semantic Version,https://semver.org) scheme for version comparison. + - C(pep440) - This type implements the Python L(PEP-440,https://peps.python.org/pep-0440/) versioning rules for version comparison. Added in version 2.14. +EXAMPLES: | + - name: version test examples + assert: + that: + - "'1.0' is version_compare('1.0', '==')" # old name + - "'1.0' is version('1.0', '==')" + - "'1.0' is version('2.0', '!=')" + - "'1.0' is version('2.0', '<')" + - "'2.0' is version('1.0', '>')" + - "'1.0' is version('1.0', '<=')" + - "'1.0' is version('1.0', '>=')" + - "'1.0' is version_compare('1.0', '==', strict=true)" # old name + - "'1.0' is version('1.0', '==', strict=true)" + - "'1.0' is version('2.0', '!=', strict=true)" + - "'1.0' is version('2.0', '<', strict=true)" + - "'2.0' is version('1.0', '>', strict=true)" + - "'1.0' is version('1.0', '<=', strict=true)" + - "'1.0' is version('1.0', '>=', strict=true)" + - "'1.2.3' is version('2.0.0', 'lt', version_type='semver')" + - "'2.14.0rc1' is version('2.14.0', 'lt', version_type='pep440')" +RETURN: + _value: + description: Returns C(True) or C(False) depending on the outcome of the comparison. + type: boolean diff --git a/lib/ansible/plugins/vars/__init__.py b/lib/ansible/plugins/vars/__init__.py new file mode 100644 index 0000000..2a7bafd --- /dev/null +++ b/lib/ansible/plugins/vars/__init__.py @@ -0,0 +1,41 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# (c) 2014, Serge van Ginderachter <serge@vanginderachter.be> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible.plugins import AnsiblePlugin +from ansible.utils.path import basedir +from ansible.utils.display import Display + +display = Display() + + +class BaseVarsPlugin(AnsiblePlugin): + + """ + Loads variables for groups and/or hosts + """ + + def __init__(self): + """ constructor """ + super(BaseVarsPlugin, self).__init__() + self._display = display + + def get_vars(self, loader, path, entities): + """ Gets variables. """ + self._basedir = basedir(path) diff --git a/lib/ansible/plugins/vars/host_group_vars.py b/lib/ansible/plugins/vars/host_group_vars.py new file mode 100644 index 0000000..521b3b6 --- /dev/null +++ b/lib/ansible/plugins/vars/host_group_vars.py @@ -0,0 +1,116 @@ +# Copyright 2017 RedHat, inc +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. +############################################# +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' + name: host_group_vars + version_added: "2.4" + short_description: In charge of loading group_vars and host_vars + requirements: + - Enabled in configuration + description: + - Loads YAML vars into corresponding groups/hosts in group_vars/ and host_vars/ directories. + - Files are restricted by extension to one of .yaml, .json, .yml or no extension. + - Hidden (starting with '.') and backup (ending with '~') files and directories are ignored. + - Only applies to inventory sources that are existing paths. + - Starting in 2.10, this plugin requires enabling and is enabled by default. + options: + stage: + ini: + - key: stage + section: vars_host_group_vars + env: + - name: ANSIBLE_VARS_PLUGIN_STAGE + _valid_extensions: + default: [".yml", ".yaml", ".json"] + description: + - "Check all of these extensions when looking for 'variable' files which should be YAML or JSON or vaulted versions of these." + - 'This affects vars_files, include_vars, inventory and vars plugins among others.' + env: + - name: ANSIBLE_YAML_FILENAME_EXT + ini: + - key: yaml_valid_extensions + section: defaults + type: list + elements: string + extends_documentation_fragment: + - vars_plugin_staging +''' + +import os +from ansible import constants as C +from ansible.errors import AnsibleParserError +from ansible.module_utils._text import to_bytes, to_native, to_text +from ansible.plugins.vars import BaseVarsPlugin +from ansible.inventory.host import Host +from ansible.inventory.group import Group +from ansible.utils.vars import combine_vars + +FOUND = {} # type: dict[str, list[str]] + + +class VarsModule(BaseVarsPlugin): + + REQUIRES_ENABLED = True + + def get_vars(self, loader, path, entities, cache=True): + ''' parses the inventory file ''' + + if not isinstance(entities, list): + entities = [entities] + + super(VarsModule, self).get_vars(loader, path, entities) + + data = {} + for entity in entities: + if isinstance(entity, Host): + subdir = 'host_vars' + elif isinstance(entity, Group): + subdir = 'group_vars' + else: + raise AnsibleParserError("Supplied entity must be Host or Group, got %s instead" % (type(entity))) + + # avoid 'chroot' type inventory hostnames /path/to/chroot + if not entity.name.startswith(os.path.sep): + try: + found_files = [] + # load vars + b_opath = os.path.realpath(to_bytes(os.path.join(self._basedir, subdir))) + opath = to_text(b_opath) + key = '%s.%s' % (entity.name, opath) + if cache and key in FOUND: + found_files = FOUND[key] + else: + # no need to do much if path does not exist for basedir + if os.path.exists(b_opath): + if os.path.isdir(b_opath): + self._display.debug("\tprocessing dir %s" % opath) + found_files = loader.find_vars_files(opath, entity.name) + FOUND[key] = found_files + else: + self._display.warning("Found %s that is not a directory, skipping: %s" % (subdir, opath)) + + for found in found_files: + new_data = loader.load_from_file(found, cache=True, unsafe=True) + if new_data: # ignore empty files + data = combine_vars(data, new_data) + + except Exception as e: + raise AnsibleParserError(to_native(e)) + return data |