diff options
Diffstat (limited to 'lib/ansible/parsing')
| -rw-r--r-- | lib/ansible/parsing/__init__.py | 20 | ||||
| -rw-r--r-- | lib/ansible/parsing/ajson.py | 42 | ||||
| -rw-r--r-- | lib/ansible/parsing/dataloader.py | 468 | ||||
| -rw-r--r-- | lib/ansible/parsing/mod_args.py | 345 | ||||
| -rw-r--r-- | lib/ansible/parsing/plugin_docs.py | 227 | ||||
| -rw-r--r-- | lib/ansible/parsing/quoting.py | 31 | ||||
| -rw-r--r-- | lib/ansible/parsing/splitter.py | 286 | ||||
| -rw-r--r-- | lib/ansible/parsing/utils/__init__.py | 20 | ||||
| -rw-r--r-- | lib/ansible/parsing/utils/addresses.py | 216 | ||||
| -rw-r--r-- | lib/ansible/parsing/utils/jsonify.py | 38 | ||||
| -rw-r--r-- | lib/ansible/parsing/utils/yaml.py | 84 | ||||
| -rw-r--r-- | lib/ansible/parsing/vault/__init__.py | 1289 | ||||
| -rw-r--r-- | lib/ansible/parsing/yaml/__init__.py | 20 | ||||
| -rw-r--r-- | lib/ansible/parsing/yaml/constructor.py | 178 | ||||
| -rw-r--r-- | lib/ansible/parsing/yaml/dumper.py | 122 | ||||
| -rw-r--r-- | lib/ansible/parsing/yaml/loader.py | 45 | ||||
| -rw-r--r-- | lib/ansible/parsing/yaml/objects.py | 365 |
17 files changed, 3796 insertions, 0 deletions
diff --git a/lib/ansible/parsing/__init__.py b/lib/ansible/parsing/__init__.py new file mode 100644 index 0000000..28634b1 --- /dev/null +++ b/lib/ansible/parsing/__init__.py @@ -0,0 +1,20 @@ +# (c) 2015, Toshio Kuratomi <tkuratomi@ansible.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type diff --git a/lib/ansible/parsing/ajson.py b/lib/ansible/parsing/ajson.py new file mode 100644 index 0000000..8049755 --- /dev/null +++ b/lib/ansible/parsing/ajson.py @@ -0,0 +1,42 @@ +# Copyright: (c) 2018, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import json + +# Imported for backwards compat +from ansible.module_utils.common.json import AnsibleJSONEncoder + +from ansible.parsing.vault import VaultLib +from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode +from ansible.utils.unsafe_proxy import wrap_var + + +class AnsibleJSONDecoder(json.JSONDecoder): + + _vaults = {} # type: dict[str, VaultLib] + + def __init__(self, *args, **kwargs): + kwargs['object_hook'] = self.object_hook + super(AnsibleJSONDecoder, self).__init__(*args, **kwargs) + + @classmethod + def set_secrets(cls, secrets): + cls._vaults['default'] = VaultLib(secrets=secrets) + + def object_hook(self, pairs): + for key in pairs: + value = pairs[key] + + if key == '__ansible_vault': + value = AnsibleVaultEncryptedUnicode(value) + if self._vaults: + value.vault = self._vaults['default'] + return value + elif key == '__ansible_unsafe': + return wrap_var(value) + + return pairs diff --git a/lib/ansible/parsing/dataloader.py b/lib/ansible/parsing/dataloader.py new file mode 100644 index 0000000..cbba966 --- /dev/null +++ b/lib/ansible/parsing/dataloader.py @@ -0,0 +1,468 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# Copyright: (c) 2017, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import copy +import os +import os.path +import re +import tempfile + +from ansible import constants as C +from ansible.errors import AnsibleFileNotFound, AnsibleParserError +from ansible.module_utils.basic import is_executable +from ansible.module_utils.six import binary_type, text_type +from ansible.module_utils._text import to_bytes, to_native, to_text +from ansible.parsing.quoting import unquote +from ansible.parsing.utils.yaml import from_yaml +from ansible.parsing.vault import VaultLib, b_HEADER, is_encrypted, is_encrypted_file, parse_vaulttext_envelope +from ansible.utils.path import unfrackpath +from ansible.utils.display import Display + +display = Display() + + +# Tries to determine if a path is inside a role, last dir must be 'tasks' +# this is not perfect but people should really avoid 'tasks' dirs outside roles when using Ansible. +RE_TASKS = re.compile(u'(?:^|%s)+tasks%s?$' % (os.path.sep, os.path.sep)) + + +class DataLoader: + + ''' + The DataLoader class is used to load and parse YAML or JSON content, + either from a given file name or from a string that was previously + read in through other means. A Vault password can be specified, and + any vault-encrypted files will be decrypted. + + Data read from files will also be cached, so the file will never be + read from disk more than once. + + Usage: + + dl = DataLoader() + # optionally: dl.set_vault_password('foo') + ds = dl.load('...') + ds = dl.load_from_file('/path/to/file') + ''' + + def __init__(self): + + self._basedir = '.' + + # NOTE: not effective with forks as the main copy does not get updated. + # avoids rereading files + self._FILE_CACHE = dict() + + # NOTE: not thread safe, also issues with forks not returning data to main proc + # so they need to be cleaned independently. See WorkerProcess for example. + # used to keep track of temp files for cleaning + self._tempfiles = set() + + # initialize the vault stuff with an empty password + # TODO: replace with a ref to something that can get the password + # a creds/auth provider + # self.set_vault_password(None) + self._vaults = {} + self._vault = VaultLib() + self.set_vault_secrets(None) + + # TODO: since we can query vault_secrets late, we could provide this to DataLoader init + def set_vault_secrets(self, vault_secrets): + self._vault.secrets = vault_secrets + + def load(self, data, file_name='<string>', show_content=True, json_only=False): + '''Backwards compat for now''' + return from_yaml(data, file_name, show_content, self._vault.secrets, json_only=json_only) + + def load_from_file(self, file_name, cache=True, unsafe=False, json_only=False): + ''' Loads data from a file, which can contain either JSON or YAML. ''' + + file_name = self.path_dwim(file_name) + display.debug("Loading data from %s" % file_name) + + # if the file has already been read in and cached, we'll + # return those results to avoid more file/vault operations + if cache and file_name in self._FILE_CACHE: + parsed_data = self._FILE_CACHE[file_name] + else: + # read the file contents and load the data structure from them + (b_file_data, show_content) = self._get_file_contents(file_name) + + file_data = to_text(b_file_data, errors='surrogate_or_strict') + parsed_data = self.load(data=file_data, file_name=file_name, show_content=show_content, json_only=json_only) + + # cache the file contents for next time + self._FILE_CACHE[file_name] = parsed_data + + if unsafe: + return parsed_data + else: + # return a deep copy here, so the cache is not affected + return copy.deepcopy(parsed_data) + + def path_exists(self, path): + path = self.path_dwim(path) + return os.path.exists(to_bytes(path, errors='surrogate_or_strict')) + + def is_file(self, path): + path = self.path_dwim(path) + return os.path.isfile(to_bytes(path, errors='surrogate_or_strict')) or path == os.devnull + + def is_directory(self, path): + path = self.path_dwim(path) + return os.path.isdir(to_bytes(path, errors='surrogate_or_strict')) + + def list_directory(self, path): + path = self.path_dwim(path) + return os.listdir(path) + + def is_executable(self, path): + '''is the given path executable?''' + path = self.path_dwim(path) + return is_executable(path) + + def _decrypt_if_vault_data(self, b_vault_data, b_file_name=None): + '''Decrypt b_vault_data if encrypted and return b_data and the show_content flag''' + + if not is_encrypted(b_vault_data): + show_content = True + return b_vault_data, show_content + + b_ciphertext, b_version, cipher_name, vault_id = parse_vaulttext_envelope(b_vault_data) + b_data = self._vault.decrypt(b_vault_data, filename=b_file_name) + + show_content = False + return b_data, show_content + + def _get_file_contents(self, file_name): + ''' + Reads the file contents from the given file name + + If the contents are vault-encrypted, it will decrypt them and return + the decrypted data + + :arg file_name: The name of the file to read. If this is a relative + path, it will be expanded relative to the basedir + :raises AnsibleFileNotFound: if the file_name does not refer to a file + :raises AnsibleParserError: if we were unable to read the file + :return: Returns a byte string of the file contents + ''' + if not file_name or not isinstance(file_name, (binary_type, text_type)): + raise AnsibleParserError("Invalid filename: '%s'" % to_native(file_name)) + + b_file_name = to_bytes(self.path_dwim(file_name)) + # This is what we really want but have to fix unittests to make it pass + # if not os.path.exists(b_file_name) or not os.path.isfile(b_file_name): + if not self.path_exists(b_file_name): + raise AnsibleFileNotFound("Unable to retrieve file contents", file_name=file_name) + + try: + with open(b_file_name, 'rb') as f: + data = f.read() + return self._decrypt_if_vault_data(data, b_file_name) + except (IOError, OSError) as e: + raise AnsibleParserError("an error occurred while trying to read the file '%s': %s" % (file_name, to_native(e)), orig_exc=e) + + def get_basedir(self): + ''' returns the current basedir ''' + return self._basedir + + def set_basedir(self, basedir): + ''' sets the base directory, used to find files when a relative path is given ''' + + if basedir is not None: + self._basedir = to_text(basedir) + + def path_dwim(self, given): + ''' + make relative paths work like folks expect. + ''' + + given = unquote(given) + given = to_text(given, errors='surrogate_or_strict') + + if given.startswith(to_text(os.path.sep)) or given.startswith(u'~'): + path = given + else: + basedir = to_text(self._basedir, errors='surrogate_or_strict') + path = os.path.join(basedir, given) + + return unfrackpath(path, follow=False) + + def _is_role(self, path): + ''' imperfect role detection, roles are still valid w/o tasks|meta/main.yml|yaml|etc ''' + + b_path = to_bytes(path, errors='surrogate_or_strict') + b_path_dirname = os.path.dirname(b_path) + b_upath = to_bytes(unfrackpath(path, follow=False), errors='surrogate_or_strict') + + untasked_paths = ( + os.path.join(b_path, b'main.yml'), + os.path.join(b_path, b'main.yaml'), + os.path.join(b_path, b'main'), + ) + tasked_paths = ( + os.path.join(b_upath, b'tasks/main.yml'), + os.path.join(b_upath, b'tasks/main.yaml'), + os.path.join(b_upath, b'tasks/main'), + os.path.join(b_upath, b'meta/main.yml'), + os.path.join(b_upath, b'meta/main.yaml'), + os.path.join(b_upath, b'meta/main'), + os.path.join(b_path_dirname, b'tasks/main.yml'), + os.path.join(b_path_dirname, b'tasks/main.yaml'), + os.path.join(b_path_dirname, b'tasks/main'), + os.path.join(b_path_dirname, b'meta/main.yml'), + os.path.join(b_path_dirname, b'meta/main.yaml'), + os.path.join(b_path_dirname, b'meta/main'), + ) + + exists_untasked = map(os.path.exists, untasked_paths) + exists_tasked = map(os.path.exists, tasked_paths) + if RE_TASKS.search(path) and any(exists_untasked) or any(exists_tasked): + return True + + return False + + def path_dwim_relative(self, path, dirname, source, is_role=False): + ''' + find one file in either a role or playbook dir with or without + explicitly named dirname subdirs + + Used in action plugins and lookups to find supplemental files that + could be in either place. + ''' + + search = [] + source = to_text(source, errors='surrogate_or_strict') + + # I have full path, nothing else needs to be looked at + if source.startswith(to_text(os.path.sep)) or source.startswith(u'~'): + search.append(unfrackpath(source, follow=False)) + else: + # base role/play path + templates/files/vars + relative filename + search.append(os.path.join(path, dirname, source)) + basedir = unfrackpath(path, follow=False) + + # not told if role, but detect if it is a role and if so make sure you get correct base path + if not is_role: + is_role = self._is_role(path) + + if is_role and RE_TASKS.search(path): + basedir = unfrackpath(os.path.dirname(path), follow=False) + + cur_basedir = self._basedir + self.set_basedir(basedir) + # resolved base role/play path + templates/files/vars + relative filename + search.append(unfrackpath(os.path.join(basedir, dirname, source), follow=False)) + self.set_basedir(cur_basedir) + + if is_role and not source.endswith(dirname): + # look in role's tasks dir w/o dirname + search.append(unfrackpath(os.path.join(basedir, 'tasks', source), follow=False)) + + # try to create absolute path for loader basedir + templates/files/vars + filename + search.append(unfrackpath(os.path.join(dirname, source), follow=False)) + + # try to create absolute path for loader basedir + search.append(unfrackpath(os.path.join(basedir, source), follow=False)) + + # try to create absolute path for dirname + filename + search.append(self.path_dwim(os.path.join(dirname, source))) + + # try to create absolute path for filename + search.append(self.path_dwim(source)) + + for candidate in search: + if os.path.exists(to_bytes(candidate, errors='surrogate_or_strict')): + break + + return candidate + + def path_dwim_relative_stack(self, paths, dirname, source, is_role=False): + ''' + find one file in first path in stack taking roles into account and adding play basedir as fallback + + :arg paths: A list of text strings which are the paths to look for the filename in. + :arg dirname: A text string representing a directory. The directory + is prepended to the source to form the path to search for. + :arg source: A text string which is the filename to search for + :rtype: A text string + :returns: An absolute path to the filename ``source`` if found + :raises: An AnsibleFileNotFound Exception if the file is found to exist in the search paths + ''' + b_dirname = to_bytes(dirname, errors='surrogate_or_strict') + b_source = to_bytes(source, errors='surrogate_or_strict') + + result = None + search = [] + if source is None: + display.warning('Invalid request to find a file that matches a "null" value') + elif source and (source.startswith('~') or source.startswith(os.path.sep)): + # path is absolute, no relative needed, check existence and return source + test_path = unfrackpath(b_source, follow=False) + if os.path.exists(to_bytes(test_path, errors='surrogate_or_strict')): + result = test_path + else: + display.debug(u'evaluation_path:\n\t%s' % '\n\t'.join(paths)) + for path in paths: + upath = unfrackpath(path, follow=False) + b_upath = to_bytes(upath, errors='surrogate_or_strict') + b_pb_base_dir = os.path.dirname(b_upath) + + # if path is in role and 'tasks' not there already, add it into the search + if (is_role or self._is_role(path)) and b_pb_base_dir.endswith(b'/tasks'): + search.append(os.path.join(os.path.dirname(b_pb_base_dir), b_dirname, b_source)) + search.append(os.path.join(b_pb_base_dir, b_source)) + else: + # don't add dirname if user already is using it in source + if b_source.split(b'/')[0] != dirname: + search.append(os.path.join(b_upath, b_dirname, b_source)) + search.append(os.path.join(b_upath, b_source)) + + # always append basedir as last resort + # don't add dirname if user already is using it in source + if b_source.split(b'/')[0] != dirname: + search.append(os.path.join(to_bytes(self.get_basedir(), errors='surrogate_or_strict'), b_dirname, b_source)) + search.append(os.path.join(to_bytes(self.get_basedir(), errors='surrogate_or_strict'), b_source)) + + display.debug(u'search_path:\n\t%s' % to_text(b'\n\t'.join(search))) + for b_candidate in search: + display.vvvvv(u'looking for "%s" at "%s"' % (source, to_text(b_candidate))) + if os.path.exists(b_candidate): + result = to_text(b_candidate) + break + + if result is None: + raise AnsibleFileNotFound(file_name=source, paths=[to_native(p) for p in search]) + + return result + + def _create_content_tempfile(self, content): + ''' Create a tempfile containing defined content ''' + fd, content_tempfile = tempfile.mkstemp(dir=C.DEFAULT_LOCAL_TMP) + f = os.fdopen(fd, 'wb') + content = to_bytes(content) + try: + f.write(content) + except Exception as err: + os.remove(content_tempfile) + raise Exception(err) + finally: + f.close() + return content_tempfile + + def get_real_file(self, file_path, decrypt=True): + """ + If the file is vault encrypted return a path to a temporary decrypted file + If the file is not encrypted then the path is returned + Temporary files are cleanup in the destructor + """ + + if not file_path or not isinstance(file_path, (binary_type, text_type)): + raise AnsibleParserError("Invalid filename: '%s'" % to_native(file_path)) + + b_file_path = to_bytes(file_path, errors='surrogate_or_strict') + if not self.path_exists(b_file_path) or not self.is_file(b_file_path): + raise AnsibleFileNotFound(file_name=file_path) + + real_path = self.path_dwim(file_path) + + try: + if decrypt: + with open(to_bytes(real_path), 'rb') as f: + # Limit how much of the file is read since we do not know + # whether this is a vault file and therefore it could be very + # large. + if is_encrypted_file(f, count=len(b_HEADER)): + # if the file is encrypted and no password was specified, + # the decrypt call would throw an error, but we check first + # since the decrypt function doesn't know the file name + data = f.read() + if not self._vault.secrets: + raise AnsibleParserError("A vault password or secret must be specified to decrypt %s" % to_native(file_path)) + + data = self._vault.decrypt(data, filename=real_path) + # Make a temp file + real_path = self._create_content_tempfile(data) + self._tempfiles.add(real_path) + + return real_path + + except (IOError, OSError) as e: + raise AnsibleParserError("an error occurred while trying to read the file '%s': %s" % (to_native(real_path), to_native(e)), orig_exc=e) + + def cleanup_tmp_file(self, file_path): + """ + Removes any temporary files created from a previous call to + get_real_file. file_path must be the path returned from a + previous call to get_real_file. + """ + if file_path in self._tempfiles: + os.unlink(file_path) + self._tempfiles.remove(file_path) + + def cleanup_all_tmp_files(self): + """ + Removes all temporary files that DataLoader has created + NOTE: not thread safe, forks also need special handling see __init__ for details. + """ + for f in list(self._tempfiles): + try: + self.cleanup_tmp_file(f) + except Exception as e: + display.warning("Unable to cleanup temp files: %s" % to_text(e)) + + def find_vars_files(self, path, name, extensions=None, allow_dir=True): + """ + Find vars files in a given path with specified name. This will find + files in a dir named <name>/ or a file called <name> ending in known + extensions. + """ + + b_path = to_bytes(os.path.join(path, name)) + found = [] + + if extensions is None: + # Look for file with no extension first to find dir before file + extensions = [''] + C.YAML_FILENAME_EXTENSIONS + # add valid extensions to name + for ext in extensions: + + if '.' in ext: + full_path = b_path + to_bytes(ext) + elif ext: + full_path = b'.'.join([b_path, to_bytes(ext)]) + else: + full_path = b_path + + if self.path_exists(full_path): + if self.is_directory(full_path): + if allow_dir: + found.extend(self._get_dir_vars_files(to_text(full_path), extensions)) + else: + continue + else: + found.append(full_path) + break + return found + + def _get_dir_vars_files(self, path, extensions): + found = [] + for spath in sorted(self.list_directory(path)): + if not spath.startswith(u'.') and not spath.endswith(u'~'): # skip hidden and backups + + ext = os.path.splitext(spath)[-1] + full_spath = os.path.join(path, spath) + + if self.is_directory(full_spath) and not ext: # recursive search if dir + found.extend(self._get_dir_vars_files(full_spath, extensions)) + elif self.is_file(full_spath) and (not ext or to_text(ext) in extensions): + # only consider files with valid extensions or no extension + found.append(full_spath) + + return found diff --git a/lib/ansible/parsing/mod_args.py b/lib/ansible/parsing/mod_args.py new file mode 100644 index 0000000..aeb58b0 --- /dev/null +++ b/lib/ansible/parsing/mod_args.py @@ -0,0 +1,345 @@ +# (c) 2014 Michael DeHaan, <michael@ansible.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import ansible.constants as C +from ansible.errors import AnsibleParserError, AnsibleError, AnsibleAssertionError +from ansible.module_utils.six import string_types +from ansible.module_utils._text import to_text +from ansible.parsing.splitter import parse_kv, split_args +from ansible.plugins.loader import module_loader, action_loader +from ansible.template import Templar +from ansible.utils.fqcn import add_internal_fqcns +from ansible.utils.sentinel import Sentinel + + +# For filtering out modules correctly below +FREEFORM_ACTIONS = frozenset(C.MODULE_REQUIRE_ARGS) + +RAW_PARAM_MODULES = FREEFORM_ACTIONS.union(add_internal_fqcns(( + 'include', + 'include_vars', + 'include_tasks', + 'include_role', + 'import_tasks', + 'import_role', + 'add_host', + 'group_by', + 'set_fact', + 'meta', +))) + +BUILTIN_TASKS = frozenset(add_internal_fqcns(( + 'meta', + 'include', + 'include_tasks', + 'include_role', + 'import_tasks', + 'import_role' +))) + + +class ModuleArgsParser: + + """ + There are several ways a module and argument set can be expressed: + + # legacy form (for a shell command) + - action: shell echo hi + + # common shorthand for local actions vs delegate_to + - local_action: shell echo hi + + # most commonly: + - copy: src=a dest=b + + # legacy form + - action: copy src=a dest=b + + # complex args form, for passing structured data + - copy: + src: a + dest: b + + # gross, but technically legal + - action: + module: copy + args: + src: a + dest: b + + # Standard YAML form for command-type modules. In this case, the args specified + # will act as 'defaults' and will be overridden by any args specified + # in one of the other formats (complex args under the action, or + # parsed from the k=v string + - command: 'pwd' + args: + chdir: '/tmp' + + + This class has some of the logic to canonicalize these into the form + + - module: <module_name> + delegate_to: <optional> + args: <args> + + Args may also be munged for certain shell command parameters. + """ + + def __init__(self, task_ds=None, collection_list=None): + task_ds = {} if task_ds is None else task_ds + + if not isinstance(task_ds, dict): + raise AnsibleAssertionError("the type of 'task_ds' should be a dict, but is a %s" % type(task_ds)) + self._task_ds = task_ds + self._collection_list = collection_list + # delayed local imports to prevent circular import + from ansible.playbook.task import Task + from ansible.playbook.handler import Handler + # store the valid Task/Handler attrs for quick access + self._task_attrs = set(Task.fattributes) + self._task_attrs.update(set(Handler.fattributes)) + # HACK: why are these not FieldAttributes on task with a post-validate to check usage? + self._task_attrs.update(['local_action', 'static']) + self._task_attrs = frozenset(self._task_attrs) + + self.resolved_action = None + + def _split_module_string(self, module_string): + ''' + when module names are expressed like: + action: copy src=a dest=b + the first part of the string is the name of the module + and the rest are strings pertaining to the arguments. + ''' + + tokens = split_args(module_string) + if len(tokens) > 1: + return (tokens[0].strip(), " ".join(tokens[1:])) + else: + return (tokens[0].strip(), "") + + def _normalize_parameters(self, thing, action=None, additional_args=None): + ''' + arguments can be fuzzy. Deal with all the forms. + ''' + + additional_args = {} if additional_args is None else additional_args + + # final args are the ones we'll eventually return, so first update + # them with any additional args specified, which have lower priority + # than those which may be parsed/normalized next + final_args = dict() + if additional_args: + if isinstance(additional_args, string_types): + templar = Templar(loader=None) + if templar.is_template(additional_args): + final_args['_variable_params'] = additional_args + else: + raise AnsibleParserError("Complex args containing variables cannot use bare variables (without Jinja2 delimiters), " + "and must use the full variable style ('{{var_name}}')") + elif isinstance(additional_args, dict): + final_args.update(additional_args) + else: + raise AnsibleParserError('Complex args must be a dictionary or variable string ("{{var}}").') + + # how we normalize depends if we figured out what the module name is + # yet. If we have already figured it out, it's a 'new style' invocation. + # otherwise, it's not + + if action is not None: + args = self._normalize_new_style_args(thing, action) + else: + (action, args) = self._normalize_old_style_args(thing) + + # this can occasionally happen, simplify + if args and 'args' in args: + tmp_args = args.pop('args') + if isinstance(tmp_args, string_types): + tmp_args = parse_kv(tmp_args) + args.update(tmp_args) + + # only internal variables can start with an underscore, so + # we don't allow users to set them directly in arguments + if args and action not in FREEFORM_ACTIONS: + for arg in args: + arg = to_text(arg) + if arg.startswith('_ansible_'): + raise AnsibleError("invalid parameter specified for action '%s': '%s'" % (action, arg)) + + # finally, update the args we're going to return with the ones + # which were normalized above + if args: + final_args.update(args) + + return (action, final_args) + + def _normalize_new_style_args(self, thing, action): + ''' + deals with fuzziness in new style module invocations + accepting key=value pairs and dictionaries, and returns + a dictionary of arguments + + possible example inputs: + 'echo hi', 'shell' + {'region': 'xyz'}, 'ec2' + standardized outputs like: + { _raw_params: 'echo hi', _uses_shell: True } + ''' + + if isinstance(thing, dict): + # form is like: { xyz: { x: 2, y: 3 } } + args = thing + elif isinstance(thing, string_types): + # form is like: copy: src=a dest=b + check_raw = action in FREEFORM_ACTIONS + args = parse_kv(thing, check_raw=check_raw) + elif thing is None: + # this can happen with modules which take no params, like ping: + args = None + else: + raise AnsibleParserError("unexpected parameter type in action: %s" % type(thing), obj=self._task_ds) + return args + + def _normalize_old_style_args(self, thing): + ''' + deals with fuzziness in old-style (action/local_action) module invocations + returns tuple of (module_name, dictionary_args) + + possible example inputs: + { 'shell' : 'echo hi' } + 'shell echo hi' + {'module': 'ec2', 'x': 1 } + standardized outputs like: + ('ec2', { 'x': 1} ) + ''' + + action = None + args = None + + if isinstance(thing, dict): + # form is like: action: { module: 'copy', src: 'a', dest: 'b' } + thing = thing.copy() + if 'module' in thing: + action, module_args = self._split_module_string(thing['module']) + args = thing.copy() + check_raw = action in FREEFORM_ACTIONS + args.update(parse_kv(module_args, check_raw=check_raw)) + del args['module'] + + elif isinstance(thing, string_types): + # form is like: action: copy src=a dest=b + (action, args) = self._split_module_string(thing) + check_raw = action in FREEFORM_ACTIONS + args = parse_kv(args, check_raw=check_raw) + + else: + # need a dict or a string, so giving up + raise AnsibleParserError("unexpected parameter type in action: %s" % type(thing), obj=self._task_ds) + + return (action, args) + + def parse(self, skip_action_validation=False): + ''' + Given a task in one of the supported forms, parses and returns + returns the action, arguments, and delegate_to values for the + task, dealing with all sorts of levels of fuzziness. + ''' + + thing = None + + action = None + delegate_to = self._task_ds.get('delegate_to', Sentinel) + args = dict() + + # This is the standard YAML form for command-type modules. We grab + # the args and pass them in as additional arguments, which can/will + # be overwritten via dict updates from the other arg sources below + additional_args = self._task_ds.get('args', dict()) + + # We can have one of action, local_action, or module specified + # action + if 'action' in self._task_ds: + # an old school 'action' statement + thing = self._task_ds['action'] + action, args = self._normalize_parameters(thing, action=action, additional_args=additional_args) + + # local_action + if 'local_action' in self._task_ds: + # local_action is similar but also implies a delegate_to + if action is not None: + raise AnsibleParserError("action and local_action are mutually exclusive", obj=self._task_ds) + thing = self._task_ds.get('local_action', '') + delegate_to = 'localhost' + action, args = self._normalize_parameters(thing, action=action, additional_args=additional_args) + + # module: <stuff> is the more new-style invocation + + # filter out task attributes so we're only querying unrecognized keys as actions/modules + non_task_ds = dict((k, v) for k, v in self._task_ds.items() if (k not in self._task_attrs) and (not k.startswith('with_'))) + + # walk the filtered input dictionary to see if we recognize a module name + for item, value in non_task_ds.items(): + context = None + is_action_candidate = False + if item in BUILTIN_TASKS: + is_action_candidate = True + elif skip_action_validation: + is_action_candidate = True + else: + context = action_loader.find_plugin_with_context(item, collection_list=self._collection_list) + if not context.resolved: + context = module_loader.find_plugin_with_context(item, collection_list=self._collection_list) + + is_action_candidate = context.resolved and bool(context.redirect_list) + + if is_action_candidate: + # finding more than one module name is a problem + if action is not None: + raise AnsibleParserError("conflicting action statements: %s, %s" % (action, item), obj=self._task_ds) + + if context is not None and context.resolved: + self.resolved_action = context.resolved_fqcn + + action = item + thing = value + action, args = self._normalize_parameters(thing, action=action, additional_args=additional_args) + + # if we didn't see any module in the task at all, it's not a task really + if action is None: + if non_task_ds: # there was one non-task action, but we couldn't find it + bad_action = list(non_task_ds.keys())[0] + raise AnsibleParserError("couldn't resolve module/action '{0}'. This often indicates a " + "misspelling, missing collection, or incorrect module path.".format(bad_action), + obj=self._task_ds) + else: + raise AnsibleParserError("no module/action detected in task.", + obj=self._task_ds) + elif args.get('_raw_params', '') != '' and action not in RAW_PARAM_MODULES: + templar = Templar(loader=None) + raw_params = args.pop('_raw_params') + if templar.is_template(raw_params): + args['_variable_params'] = raw_params + else: + raise AnsibleParserError("this task '%s' has extra params, which is only allowed in the following modules: %s" % (action, + ", ".join(RAW_PARAM_MODULES)), + obj=self._task_ds) + + return (action, args, delegate_to) diff --git a/lib/ansible/parsing/plugin_docs.py b/lib/ansible/parsing/plugin_docs.py new file mode 100644 index 0000000..cda5463 --- /dev/null +++ b/lib/ansible/parsing/plugin_docs.py @@ -0,0 +1,227 @@ +# Copyright: (c) 2017, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import ast +import tokenize + +from ansible import constants as C +from ansible.errors import AnsibleError, AnsibleParserError +from ansible.module_utils._text import to_text, to_native +from ansible.parsing.yaml.loader import AnsibleLoader +from ansible.utils.display import Display + +display = Display() + + +string_to_vars = { + 'DOCUMENTATION': 'doc', + 'EXAMPLES': 'plainexamples', + 'RETURN': 'returndocs', + 'ANSIBLE_METADATA': 'metadata', # NOTE: now unused, but kept for backwards compat +} + + +def _var2string(value): + ''' reverse lookup of the dict above ''' + for k, v in string_to_vars.items(): + if v == value: + return k + + +def _init_doc_dict(): + ''' initialize a return dict for docs with the expected structure ''' + return {k: None for k in string_to_vars.values()} + + +def read_docstring_from_yaml_file(filename, verbose=True, ignore_errors=True): + ''' Read docs from 'sidecar' yaml file doc for a plugin ''' + + data = _init_doc_dict() + file_data = {} + + try: + with open(filename, 'rb') as yamlfile: + file_data = AnsibleLoader(yamlfile.read(), file_name=filename).get_single_data() + except Exception as e: + msg = "Unable to parse yaml file '%s': %s" % (filename, to_native(e)) + if not ignore_errors: + raise AnsibleParserError(msg, orig_exc=e) + elif verbose: + display.error(msg) + + if file_data: + for key in string_to_vars: + data[string_to_vars[key]] = file_data.get(key, None) + + return data + + +def read_docstring_from_python_module(filename, verbose=True, ignore_errors=True): + """ + Use tokenization to search for assignment of the documentation variables in the given file. + Parse from YAML and return the resulting python structure or None together with examples as plain text. + """ + + seen = set() + data = _init_doc_dict() + + next_string = None + with tokenize.open(filename) as f: + tokens = tokenize.generate_tokens(f.readline) + for token in tokens: + + # found lable that looks like variable + if token.type == tokenize.NAME: + + # label is expected value, in correct place and has not been seen before + if token.start == 1 and token.string in string_to_vars and token.string not in seen: + # next token that is string has the docs + next_string = string_to_vars[token.string] + continue + + # previous token indicated this string is a doc string + if next_string is not None and token.type == tokenize.STRING: + + # ensure we only process one case of it + seen.add(token.string) + + value = token.string + + # strip string modifiers/delimiters + if value.startswith(('r', 'b')): + value = value.lstrip('rb') + + if value.startswith(("'", '"')): + value = value.strip("'\"") + + # actually use the data + if next_string == 'plainexamples': + # keep as string, can be yaml, but we let caller deal with it + data[next_string] = to_text(value) + else: + # yaml load the data + try: + data[next_string] = AnsibleLoader(value, file_name=filename).get_single_data() + except Exception as e: + msg = "Unable to parse docs '%s' in python file '%s': %s" % (_var2string(next_string), filename, to_native(e)) + if not ignore_errors: + raise AnsibleParserError(msg, orig_exc=e) + elif verbose: + display.error(msg) + + next_string = None + + # if nothing else worked, fall back to old method + if not seen: + data = read_docstring_from_python_file(filename, verbose, ignore_errors) + + return data + + +def read_docstring_from_python_file(filename, verbose=True, ignore_errors=True): + """ + Use ast to search for assignment of the DOCUMENTATION and EXAMPLES variables in the given file. + Parse DOCUMENTATION from YAML and return the YAML doc or None together with EXAMPLES, as plain text. + """ + + data = _init_doc_dict() + + try: + with open(filename, 'rb') as b_module_data: + M = ast.parse(b_module_data.read()) + + for child in M.body: + if isinstance(child, ast.Assign): + for t in child.targets: + try: + theid = t.id + except AttributeError: + # skip errors can happen when trying to use the normal code + display.warning("Building documentation, failed to assign id for %s on %s, skipping" % (t, filename)) + continue + + if theid in string_to_vars: + varkey = string_to_vars[theid] + if isinstance(child.value, ast.Dict): + data[varkey] = ast.literal_eval(child.value) + else: + if theid == 'EXAMPLES': + # examples 'can' be yaml, but even if so, we dont want to parse as such here + # as it can create undesired 'objects' that don't display well as docs. + data[varkey] = to_text(child.value.s) + else: + # string should be yaml if already not a dict + data[varkey] = AnsibleLoader(child.value.s, file_name=filename).get_single_data() + + display.debug('Documentation assigned: %s' % varkey) + + except Exception as e: + msg = "Unable to parse documentation in python file '%s': %s" % (filename, to_native(e)) + if not ignore_errors: + raise AnsibleParserError(msg, orig_exc=e) + elif verbose: + display.error(msg) + + return data + + +def read_docstring(filename, verbose=True, ignore_errors=True): + ''' returns a documentation dictionary from Ansible plugin docstrings ''' + + # NOTE: adjacency of doc file to code file is responsibility of caller + if filename.endswith(C.YAML_DOC_EXTENSIONS): + docstring = read_docstring_from_yaml_file(filename, verbose=verbose, ignore_errors=ignore_errors) + elif filename.endswith(C.PYTHON_DOC_EXTENSIONS): + docstring = read_docstring_from_python_module(filename, verbose=verbose, ignore_errors=ignore_errors) + elif not ignore_errors: + raise AnsibleError("Unknown documentation format: %s" % to_native(filename)) + + if not docstring and not ignore_errors: + raise AnsibleError("Unable to parse documentation for: %s" % to_native(filename)) + + # cause seealso is specially processed from 'doc' later on + # TODO: stop any other 'overloaded' implementation in main doc + docstring['seealso'] = None + + return docstring + + +def read_docstub(filename): + """ + Quickly find short_description using string methods instead of node parsing. + This does not return a full set of documentation strings and is intended for + operations like ansible-doc -l. + """ + + in_documentation = False + capturing = False + indent_detection = '' + doc_stub = [] + + with open(filename, 'r') as t_module_data: + for line in t_module_data: + if in_documentation: + # start capturing the stub until indentation returns + if capturing and line.startswith(indent_detection): + doc_stub.append(line) + + elif capturing and not line.startswith(indent_detection): + break + + elif line.lstrip().startswith('short_description:'): + capturing = True + # Detect that the short_description continues on the next line if it's indented more + # than short_description itself. + indent_detection = ' ' * (len(line) - len(line.lstrip()) + 1) + doc_stub.append(line) + + elif line.startswith('DOCUMENTATION') and ('=' in line or ':' in line): + in_documentation = True + + short_description = r''.join(doc_stub).strip().rstrip('.') + data = AnsibleLoader(short_description, file_name=filename).get_single_data() + + return data diff --git a/lib/ansible/parsing/quoting.py b/lib/ansible/parsing/quoting.py new file mode 100644 index 0000000..d3a38d9 --- /dev/null +++ b/lib/ansible/parsing/quoting.py @@ -0,0 +1,31 @@ +# (c) 2014 James Cammarata, <jcammarata@ansible.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +def is_quoted(data): + return len(data) > 1 and data[0] == data[-1] and data[0] in ('"', "'") and data[-2] != '\\' + + +def unquote(data): + ''' removes first and last quotes from a string, if the string starts and ends with the same quotes ''' + if is_quoted(data): + return data[1:-1] + return data diff --git a/lib/ansible/parsing/splitter.py b/lib/ansible/parsing/splitter.py new file mode 100644 index 0000000..b68444f --- /dev/null +++ b/lib/ansible/parsing/splitter.py @@ -0,0 +1,286 @@ +# (c) 2014 James Cammarata, <jcammarata@ansible.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import codecs +import re + +from ansible.errors import AnsibleParserError +from ansible.module_utils._text import to_text +from ansible.parsing.quoting import unquote + + +# Decode escapes adapted from rspeer's answer here: +# http://stackoverflow.com/questions/4020539/process-escape-sequences-in-a-string-in-python +_HEXCHAR = '[a-fA-F0-9]' +_ESCAPE_SEQUENCE_RE = re.compile(r''' + ( \\U{0} # 8-digit hex escapes + | \\u{1} # 4-digit hex escapes + | \\x{2} # 2-digit hex escapes + | \\N\{{[^}}]+\}} # Unicode characters by name + | \\[\\'"abfnrtv] # Single-character escapes + )'''.format(_HEXCHAR * 8, _HEXCHAR * 4, _HEXCHAR * 2), re.UNICODE | re.VERBOSE) + + +def _decode_escapes(s): + def decode_match(match): + return codecs.decode(match.group(0), 'unicode-escape') + + return _ESCAPE_SEQUENCE_RE.sub(decode_match, s) + + +def parse_kv(args, check_raw=False): + ''' + Convert a string of key/value items to a dict. If any free-form params + are found and the check_raw option is set to True, they will be added + to a new parameter called '_raw_params'. If check_raw is not enabled, + they will simply be ignored. + ''' + + args = to_text(args, nonstring='passthru') + + options = {} + if args is not None: + try: + vargs = split_args(args) + except IndexError as e: + raise AnsibleParserError("Unable to parse argument string", orig_exc=e) + except ValueError as ve: + if 'no closing quotation' in str(ve).lower(): + raise AnsibleParserError("error parsing argument string, try quoting the entire line.", orig_exc=ve) + else: + raise + + raw_params = [] + for orig_x in vargs: + x = _decode_escapes(orig_x) + if "=" in x: + pos = 0 + try: + while True: + pos = x.index('=', pos + 1) + if pos > 0 and x[pos - 1] != '\\': + break + except ValueError: + # ran out of string, but we must have some escaped equals, + # so replace those and append this to the list of raw params + raw_params.append(x.replace('\\=', '=')) + continue + + k = x[:pos] + v = x[pos + 1:] + + # FIXME: make the retrieval of this list of shell/command options a function, so the list is centralized + if check_raw and k not in ('creates', 'removes', 'chdir', 'executable', 'warn', 'stdin', 'stdin_add_newline', 'strip_empty_ends'): + raw_params.append(orig_x) + else: + options[k.strip()] = unquote(v.strip()) + else: + raw_params.append(orig_x) + + # recombine the free-form params, if any were found, and assign + # them to a special option for use later by the shell/command module + if len(raw_params) > 0: + options[u'_raw_params'] = join_args(raw_params) + + return options + + +def _get_quote_state(token, quote_char): + ''' + the goal of this block is to determine if the quoted string + is unterminated in which case it needs to be put back together + ''' + # the char before the current one, used to see if + # the current character is escaped + prev_char = None + for idx, cur_char in enumerate(token): + if idx > 0: + prev_char = token[idx - 1] + if cur_char in '"\'' and prev_char != '\\': + if quote_char: + if cur_char == quote_char: + quote_char = None + else: + quote_char = cur_char + return quote_char + + +def _count_jinja2_blocks(token, cur_depth, open_token, close_token): + ''' + this function counts the number of opening/closing blocks for a + given opening/closing type and adjusts the current depth for that + block based on the difference + ''' + num_open = token.count(open_token) + num_close = token.count(close_token) + if num_open != num_close: + cur_depth += (num_open - num_close) + if cur_depth < 0: + cur_depth = 0 + return cur_depth + + +def join_args(s): + ''' + Join the original cmd based on manipulations by split_args(). + This retains the original newlines and whitespaces. + ''' + result = '' + for p in s: + if len(result) == 0 or result.endswith('\n'): + result += p + else: + result += ' ' + p + return result + + +def split_args(args): + ''' + Splits args on whitespace, but intelligently reassembles + those that may have been split over a jinja2 block or quotes. + + When used in a remote module, we won't ever have to be concerned about + jinja2 blocks, however this function is/will be used in the + core portions as well before the args are templated. + + example input: a=b c="foo bar" + example output: ['a=b', 'c="foo bar"'] + + Basically this is a variation shlex that has some more intelligence for + how Ansible needs to use it. + ''' + + # the list of params parsed out of the arg string + # this is going to be the result value when we are done + params = [] + + # Initial split on newlines + items = args.split('\n') + + # iterate over the tokens, and reassemble any that may have been + # split on a space inside a jinja2 block. + # ex if tokens are "{{", "foo", "}}" these go together + + # These variables are used + # to keep track of the state of the parsing, since blocks and quotes + # may be nested within each other. + + quote_char = None + inside_quotes = False + print_depth = 0 # used to count nested jinja2 {{ }} blocks + block_depth = 0 # used to count nested jinja2 {% %} blocks + comment_depth = 0 # used to count nested jinja2 {# #} blocks + + # now we loop over each split chunk, coalescing tokens if the white space + # split occurred within quotes or a jinja2 block of some kind + for (itemidx, item) in enumerate(items): + + # we split on spaces and newlines separately, so that we + # can tell which character we split on for reassembly + # inside quotation characters + tokens = item.split(' ') + + line_continuation = False + for (idx, token) in enumerate(tokens): + + # Empty entries means we have subsequent spaces + # We want to hold onto them so we can reconstruct them later + if len(token) == 0 and idx != 0: + params[-1] += ' ' + continue + + # if we hit a line continuation character, but + # we're not inside quotes, ignore it and continue + # on to the next token while setting a flag + if token == '\\' and not inside_quotes: + line_continuation = True + continue + + # store the previous quoting state for checking later + was_inside_quotes = inside_quotes + quote_char = _get_quote_state(token, quote_char) + inside_quotes = quote_char is not None + + # multiple conditions may append a token to the list of params, + # so we keep track with this flag to make sure it only happens once + # append means add to the end of the list, don't append means concatenate + # it to the end of the last token + appended = False + + # if we're inside quotes now, but weren't before, append the token + # to the end of the list, since we'll tack on more to it later + # otherwise, if we're inside any jinja2 block, inside quotes, or we were + # inside quotes (but aren't now) concat this token to the last param + if inside_quotes and not was_inside_quotes and not (print_depth or block_depth or comment_depth): + params.append(token) + appended = True + elif print_depth or block_depth or comment_depth or inside_quotes or was_inside_quotes: + if idx == 0 and was_inside_quotes: + params[-1] = "%s%s" % (params[-1], token) + elif len(tokens) > 1: + spacer = '' + if idx > 0: + spacer = ' ' + params[-1] = "%s%s%s" % (params[-1], spacer, token) + else: + params[-1] = "%s\n%s" % (params[-1], token) + appended = True + + # if the number of paired block tags is not the same, the depth has changed, so we calculate that here + # and may append the current token to the params (if we haven't previously done so) + prev_print_depth = print_depth + print_depth = _count_jinja2_blocks(token, print_depth, "{{", "}}") + if print_depth != prev_print_depth and not appended: + params.append(token) + appended = True + + prev_block_depth = block_depth + block_depth = _count_jinja2_blocks(token, block_depth, "{%", "%}") + if block_depth != prev_block_depth and not appended: + params.append(token) + appended = True + + prev_comment_depth = comment_depth + comment_depth = _count_jinja2_blocks(token, comment_depth, "{#", "#}") + if comment_depth != prev_comment_depth and not appended: + params.append(token) + appended = True + + # finally, if we're at zero depth for all blocks and not inside quotes, and have not + # yet appended anything to the list of params, we do so now + if not (print_depth or block_depth or comment_depth) and not inside_quotes and not appended and token != '': + params.append(token) + + # if this was the last token in the list, and we have more than + # one item (meaning we split on newlines), add a newline back here + # to preserve the original structure + if len(items) > 1 and itemidx != len(items) - 1 and not line_continuation: + params[-1] += '\n' + + # always clear the line continuation flag + line_continuation = False + + # If we're done and things are not at zero depth or we're still inside quotes, + # raise an error to indicate that the args were unbalanced + if print_depth or block_depth or comment_depth or inside_quotes: + raise AnsibleParserError(u"failed at splitting arguments, either an unbalanced jinja2 block or quotes: {0}".format(args)) + + return params diff --git a/lib/ansible/parsing/utils/__init__.py b/lib/ansible/parsing/utils/__init__.py new file mode 100644 index 0000000..ae8ccff --- /dev/null +++ b/lib/ansible/parsing/utils/__init__.py @@ -0,0 +1,20 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type diff --git a/lib/ansible/parsing/utils/addresses.py b/lib/ansible/parsing/utils/addresses.py new file mode 100644 index 0000000..0096af4 --- /dev/null +++ b/lib/ansible/parsing/utils/addresses.py @@ -0,0 +1,216 @@ +# Copyright 2015 Abhijit Menon-Sen <ams@2ndQuadrant.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import re +from ansible.errors import AnsibleParserError, AnsibleError + +# Components that match a numeric or alphanumeric begin:end or begin:end:step +# range expression inside square brackets. + +numeric_range = r''' + \[ + (?:[0-9]+:[0-9]+) # numeric begin:end + (?::[0-9]+)? # numeric :step (optional) + \] +''' + +hexadecimal_range = r''' + \[ + (?:[0-9a-f]+:[0-9a-f]+) # hexadecimal begin:end + (?::[0-9]+)? # numeric :step (optional) + \] +''' + +alphanumeric_range = r''' + \[ + (?: + [a-z]:[a-z]| # one-char alphabetic range + [0-9]+:[0-9]+ # ...or a numeric one + ) + (?::[0-9]+)? # numeric :step (optional) + \] +''' + +# Components that match a 16-bit portion of an IPv6 address in hexadecimal +# notation (0..ffff) or an 8-bit portion of an IPv4 address in decimal notation +# (0..255) or an [x:y(:z)] numeric range. + +ipv6_component = r''' + (?: + [0-9a-f]{{1,4}}| # 0..ffff + {range} # or a numeric range + ) +'''.format(range=hexadecimal_range) + +ipv4_component = r''' + (?: + [01]?[0-9]{{1,2}}| # 0..199 + 2[0-4][0-9]| # 200..249 + 25[0-5]| # 250..255 + {range} # or a numeric range + ) +'''.format(range=numeric_range) + +# A hostname label, e.g. 'foo' in 'foo.example.com'. Consists of alphanumeric +# characters plus dashes (and underscores) or valid ranges. The label may not +# start or end with a hyphen or an underscore. This is interpolated into the +# hostname pattern below. We don't try to enforce the 63-char length limit. + +label = r''' + (?:[\w]|{range}) # Starts with an alphanumeric or a range + (?:[\w_-]|{range})* # Then zero or more of the same or [_-] + (?<![_-]) # ...as long as it didn't end with [_-] +'''.format(range=alphanumeric_range) + +patterns = { + # This matches a square-bracketed expression with a port specification. What + # is inside the square brackets is validated later. + + 'bracketed_hostport': re.compile( + r'''^ + \[(.+)\] # [host identifier] + :([0-9]+) # :port number + $ + ''', re.X + ), + + # This matches a bare IPv4 address or hostname (or host pattern including + # [x:y(:z)] ranges) with a port specification. + + 'hostport': re.compile( + r'''^ + ((?: # We want to match: + [^:\[\]] # (a non-range character + | # ...or... + \[[^\]]*\] # a complete bracketed expression) + )*) # repeated as many times as possible + :([0-9]+) # followed by a port number + $ + ''', re.X + ), + + # This matches an IPv4 address, but also permits range expressions. + + 'ipv4': re.compile( + r'''^ + (?:{i4}\.){{3}}{i4} # Three parts followed by dots plus one + $ + '''.format(i4=ipv4_component), re.X | re.I + ), + + # This matches an IPv6 address, but also permits range expressions. + # + # This expression looks complex, but it really only spells out the various + # combinations in which the basic unit of an IPv6 address (0..ffff) can be + # written, from :: to 1:2:3:4:5:6:7:8, plus the IPv4-in-IPv6 variants such + # as ::ffff:192.0.2.3. + # + # Note that we can't just use ipaddress.ip_address() because we also have to + # accept ranges in place of each component. + + 'ipv6': re.compile( + r'''^ + (?:{0}:){{7}}{0}| # uncompressed: 1:2:3:4:5:6:7:8 + (?:{0}:){{1,6}}:| # compressed variants, which are all + (?:{0}:)(?::{0}){{1,6}}| # a::b for various lengths of a,b + (?:{0}:){{2}}(?::{0}){{1,5}}| + (?:{0}:){{3}}(?::{0}){{1,4}}| + (?:{0}:){{4}}(?::{0}){{1,3}}| + (?:{0}:){{5}}(?::{0}){{1,2}}| + (?:{0}:){{6}}(?::{0})| # ...all with 2 <= a+b <= 7 + :(?::{0}){{1,6}}| # ::ffff(:ffff...) + {0}?::| # ffff::, :: + # ipv4-in-ipv6 variants + (?:0:){{6}}(?:{0}\.){{3}}{0}| + ::(?:ffff:)?(?:{0}\.){{3}}{0}| + (?:0:){{5}}ffff:(?:{0}\.){{3}}{0} + $ + '''.format(ipv6_component), re.X | re.I + ), + + # This matches a hostname or host pattern including [x:y(:z)] ranges. + # + # We roughly follow DNS rules here, but also allow ranges (and underscores). + # In the past, no systematic rules were enforced about inventory hostnames, + # but the parsing context (e.g. shlex.split(), fnmatch.fnmatch()) excluded + # various metacharacters anyway. + # + # We don't enforce DNS length restrictions here (63 characters per label, + # 253 characters total) or make any attempt to process IDNs. + + 'hostname': re.compile( + r'''^ + {label} # We must have at least one label + (?:\.{label})* # Followed by zero or more .labels + $ + '''.format(label=label), re.X | re.I | re.UNICODE + ), + +} + + +def parse_address(address, allow_ranges=False): + """ + Takes a string and returns a (host, port) tuple. If the host is None, then + the string could not be parsed as a host identifier with an optional port + specification. If the port is None, then no port was specified. + + The host identifier may be a hostname (qualified or not), an IPv4 address, + or an IPv6 address. If allow_ranges is True, then any of those may contain + [x:y] range specifications, e.g. foo[1:3] or foo[0:5]-bar[x-z]. + + The port number is an optional :NN suffix on an IPv4 address or host name, + or a mandatory :NN suffix on any square-bracketed expression: IPv6 address, + IPv4 address, or host name. (This means the only way to specify a port for + an IPv6 address is to enclose it in square brackets.) + """ + + # First, we extract the port number if one is specified. + + port = None + for matching in ['bracketed_hostport', 'hostport']: + m = patterns[matching].match(address) + if m: + (address, port) = m.groups() + port = int(port) + continue + + # What we're left with now must be an IPv4 or IPv6 address, possibly with + # numeric ranges, or a hostname with alphanumeric ranges. + + host = None + for matching in ['ipv4', 'ipv6', 'hostname']: + m = patterns[matching].match(address) + if m: + host = address + continue + + # If it isn't any of the above, we don't understand it. + if not host: + raise AnsibleError("Not a valid network hostname: %s" % address) + + # If we get to this point, we know that any included ranges are valid. + # If the caller is prepared to handle them, all is well. + # Otherwise we treat it as a parse failure. + if not allow_ranges and '[' in host: + raise AnsibleParserError("Detected range in host but was asked to ignore ranges") + + return (host, port) diff --git a/lib/ansible/parsing/utils/jsonify.py b/lib/ansible/parsing/utils/jsonify.py new file mode 100644 index 0000000..19ebc56 --- /dev/null +++ b/lib/ansible/parsing/utils/jsonify.py @@ -0,0 +1,38 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import json + + +def jsonify(result, format=False): + ''' format JSON output (uncompressed or uncompressed) ''' + + if result is None: + return "{}" + + indent = None + if format: + indent = 4 + + try: + return json.dumps(result, sort_keys=True, indent=indent, ensure_ascii=False) + except UnicodeDecodeError: + return json.dumps(result, sort_keys=True, indent=indent) diff --git a/lib/ansible/parsing/utils/yaml.py b/lib/ansible/parsing/utils/yaml.py new file mode 100644 index 0000000..91e37f9 --- /dev/null +++ b/lib/ansible/parsing/utils/yaml.py @@ -0,0 +1,84 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# Copyright: (c) 2017, Ansible Project +# Copyright: (c) 2018, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import json + +from yaml import YAMLError + +from ansible.errors import AnsibleParserError +from ansible.errors.yaml_strings import YAML_SYNTAX_ERROR +from ansible.module_utils._text import to_native +from ansible.parsing.yaml.loader import AnsibleLoader +from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject +from ansible.parsing.ajson import AnsibleJSONDecoder + + +__all__ = ('from_yaml',) + + +def _handle_error(json_exc, yaml_exc, file_name, show_content): + ''' + Optionally constructs an object (AnsibleBaseYAMLObject) to encapsulate the + file name/position where a YAML exception occurred, and raises an AnsibleParserError + to display the syntax exception information. + ''' + + # if the YAML exception contains a problem mark, use it to construct + # an object the error class can use to display the faulty line + err_obj = None + if hasattr(yaml_exc, 'problem_mark'): + err_obj = AnsibleBaseYAMLObject() + err_obj.ansible_pos = (file_name, yaml_exc.problem_mark.line + 1, yaml_exc.problem_mark.column + 1) + + n_yaml_syntax_error = YAML_SYNTAX_ERROR % to_native(getattr(yaml_exc, 'problem', u'')) + n_err_msg = 'We were unable to read either as JSON nor YAML, these are the errors we got from each:\n' \ + 'JSON: %s\n\n%s' % (to_native(json_exc), n_yaml_syntax_error) + + raise AnsibleParserError(n_err_msg, obj=err_obj, show_content=show_content, orig_exc=yaml_exc) + + +def _safe_load(stream, file_name=None, vault_secrets=None): + ''' Implements yaml.safe_load(), except using our custom loader class. ''' + + loader = AnsibleLoader(stream, file_name, vault_secrets) + try: + return loader.get_single_data() + finally: + try: + loader.dispose() + except AttributeError: + pass # older versions of yaml don't have dispose function, ignore + + +def from_yaml(data, file_name='<string>', show_content=True, vault_secrets=None, json_only=False): + ''' + Creates a python datastructure from the given data, which can be either + a JSON or YAML string. + ''' + new_data = None + + try: + # in case we have to deal with vaults + AnsibleJSONDecoder.set_secrets(vault_secrets) + + # we first try to load this data as JSON. + # Fixes issues with extra vars json strings not being parsed correctly by the yaml parser + new_data = json.loads(data, cls=AnsibleJSONDecoder) + except Exception as json_exc: + + if json_only: + raise AnsibleParserError(to_native(json_exc), orig_exc=json_exc) + + # must not be JSON, let the rest try + try: + new_data = _safe_load(data, file_name=file_name, vault_secrets=vault_secrets) + except YAMLError as yaml_exc: + _handle_error(json_exc, yaml_exc, file_name, show_content) + + return new_data diff --git a/lib/ansible/parsing/vault/__init__.py b/lib/ansible/parsing/vault/__init__.py new file mode 100644 index 0000000..8ac22d4 --- /dev/null +++ b/lib/ansible/parsing/vault/__init__.py @@ -0,0 +1,1289 @@ +# (c) 2014, James Tanner <tanner.jc@gmail.com> +# (c) 2016, Adrian Likins <alikins@redhat.com> +# (c) 2016 Toshio Kuratomi <tkuratomi@ansible.com> +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import errno +import fcntl +import os +import random +import shlex +import shutil +import subprocess +import sys +import tempfile +import warnings + +from binascii import hexlify +from binascii import unhexlify +from binascii import Error as BinasciiError + +HAS_CRYPTOGRAPHY = False +CRYPTOGRAPHY_BACKEND = None +try: + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + from cryptography.exceptions import InvalidSignature + from cryptography.hazmat.backends import default_backend + from cryptography.hazmat.primitives import hashes, padding + from cryptography.hazmat.primitives.hmac import HMAC + from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC + from cryptography.hazmat.primitives.ciphers import ( + Cipher as C_Cipher, algorithms, modes + ) + CRYPTOGRAPHY_BACKEND = default_backend() + HAS_CRYPTOGRAPHY = True +except ImportError: + pass + +from ansible.errors import AnsibleError, AnsibleAssertionError +from ansible import constants as C +from ansible.module_utils.six import binary_type +from ansible.module_utils._text import to_bytes, to_text, to_native +from ansible.utils.display import Display +from ansible.utils.path import makedirs_safe, unfrackpath + +display = Display() + + +b_HEADER = b'$ANSIBLE_VAULT' +CIPHER_WHITELIST = frozenset((u'AES256',)) +CIPHER_WRITE_WHITELIST = frozenset((u'AES256',)) +# See also CIPHER_MAPPING at the bottom of the file which maps cipher strings +# (used in VaultFile header) to a cipher class + +NEED_CRYPTO_LIBRARY = "ansible-vault requires the cryptography library in order to function" + + +class AnsibleVaultError(AnsibleError): + pass + + +class AnsibleVaultPasswordError(AnsibleVaultError): + pass + + +class AnsibleVaultFormatError(AnsibleError): + pass + + +def is_encrypted(data): + """ Test if this is vault encrypted data blob + + :arg data: a byte or text string to test whether it is recognized as vault + encrypted data + :returns: True if it is recognized. Otherwise, False. + """ + try: + # Make sure we have a byte string and that it only contains ascii + # bytes. + b_data = to_bytes(to_text(data, encoding='ascii', errors='strict', nonstring='strict'), encoding='ascii', errors='strict') + except (UnicodeError, TypeError): + # The vault format is pure ascii so if we failed to encode to bytes + # via ascii we know that this is not vault data. + # Similarly, if it's not a string, it's not vault data + return False + + if b_data.startswith(b_HEADER): + return True + return False + + +def is_encrypted_file(file_obj, start_pos=0, count=-1): + """Test if the contents of a file obj are a vault encrypted data blob. + + :arg file_obj: A file object that will be read from. + :kwarg start_pos: A byte offset in the file to start reading the header + from. Defaults to 0, the beginning of the file. + :kwarg count: Read up to this number of bytes from the file to determine + if it looks like encrypted vault data. The default is -1, read to the + end of file. + :returns: True if the file looks like a vault file. Otherwise, False. + """ + # read the header and reset the file stream to where it started + current_position = file_obj.tell() + try: + file_obj.seek(start_pos) + return is_encrypted(file_obj.read(count)) + + finally: + file_obj.seek(current_position) + + +def _parse_vaulttext_envelope(b_vaulttext_envelope, default_vault_id=None): + + b_tmpdata = b_vaulttext_envelope.splitlines() + b_tmpheader = b_tmpdata[0].strip().split(b';') + + b_version = b_tmpheader[1].strip() + cipher_name = to_text(b_tmpheader[2].strip()) + vault_id = default_vault_id + + # Only attempt to find vault_id if the vault file is version 1.2 or newer + # if self.b_version == b'1.2': + if len(b_tmpheader) >= 4: + vault_id = to_text(b_tmpheader[3].strip()) + + b_ciphertext = b''.join(b_tmpdata[1:]) + + return b_ciphertext, b_version, cipher_name, vault_id + + +def parse_vaulttext_envelope(b_vaulttext_envelope, default_vault_id=None, filename=None): + """Parse the vaulttext envelope + + When data is saved, it has a header prepended and is formatted into 80 + character lines. This method extracts the information from the header + and then removes the header and the inserted newlines. The string returned + is suitable for processing by the Cipher classes. + + :arg b_vaulttext: byte str containing the data from a save file + :kwarg default_vault_id: The vault_id name to use if the vaulttext does not provide one. + :kwarg filename: The filename that the data came from. This is only + used to make better error messages in case the data cannot be + decrypted. This is optional. + :returns: A tuple of byte str of the vaulttext suitable to pass to parse_vaultext, + a byte str of the vault format version, + the name of the cipher used, and the vault_id. + :raises: AnsibleVaultFormatError: if the vaulttext_envelope format is invalid + """ + # used by decrypt + default_vault_id = default_vault_id or C.DEFAULT_VAULT_IDENTITY + + try: + return _parse_vaulttext_envelope(b_vaulttext_envelope, default_vault_id) + except Exception as exc: + msg = "Vault envelope format error" + if filename: + msg += ' in %s' % (filename) + msg += ': %s' % exc + raise AnsibleVaultFormatError(msg) + + +def format_vaulttext_envelope(b_ciphertext, cipher_name, version=None, vault_id=None): + """ Add header and format to 80 columns + + :arg b_ciphertext: the encrypted and hexlified data as a byte string + :arg cipher_name: unicode cipher name (for ex, u'AES256') + :arg version: unicode vault version (for ex, '1.2'). Optional ('1.1' is default) + :arg vault_id: unicode vault identifier. If provided, the version will be bumped to 1.2. + :returns: a byte str that should be dumped into a file. It's + formatted to 80 char columns and has the header prepended + """ + + if not cipher_name: + raise AnsibleError("the cipher must be set before adding a header") + + version = version or '1.1' + + # If we specify a vault_id, use format version 1.2. For no vault_id, stick to 1.1 + if vault_id and vault_id != u'default': + version = '1.2' + + b_version = to_bytes(version, 'utf-8', errors='strict') + b_vault_id = to_bytes(vault_id, 'utf-8', errors='strict') + b_cipher_name = to_bytes(cipher_name, 'utf-8', errors='strict') + + header_parts = [b_HEADER, + b_version, + b_cipher_name] + + if b_version == b'1.2' and b_vault_id: + header_parts.append(b_vault_id) + + header = b';'.join(header_parts) + + b_vaulttext = [header] + b_vaulttext += [b_ciphertext[i:i + 80] for i in range(0, len(b_ciphertext), 80)] + b_vaulttext += [b''] + b_vaulttext = b'\n'.join(b_vaulttext) + + return b_vaulttext + + +def _unhexlify(b_data): + try: + return unhexlify(b_data) + except (BinasciiError, TypeError) as exc: + raise AnsibleVaultFormatError('Vault format unhexlify error: %s' % exc) + + +def _parse_vaulttext(b_vaulttext): + b_vaulttext = _unhexlify(b_vaulttext) + b_salt, b_crypted_hmac, b_ciphertext = b_vaulttext.split(b"\n", 2) + b_salt = _unhexlify(b_salt) + b_ciphertext = _unhexlify(b_ciphertext) + + return b_ciphertext, b_salt, b_crypted_hmac + + +def parse_vaulttext(b_vaulttext): + """Parse the vaulttext + + :arg b_vaulttext: byte str containing the vaulttext (ciphertext, salt, crypted_hmac) + :returns: A tuple of byte str of the ciphertext suitable for passing to a + Cipher class's decrypt() function, a byte str of the salt, + and a byte str of the crypted_hmac + :raises: AnsibleVaultFormatError: if the vaulttext format is invalid + """ + # SPLIT SALT, DIGEST, AND DATA + try: + return _parse_vaulttext(b_vaulttext) + except AnsibleVaultFormatError: + raise + except Exception as exc: + msg = "Vault vaulttext format error: %s" % exc + raise AnsibleVaultFormatError(msg) + + +def verify_secret_is_not_empty(secret, msg=None): + '''Check the secret against minimal requirements. + + Raises: AnsibleVaultPasswordError if the password does not meet requirements. + + Currently, only requirement is that the password is not None or an empty string. + ''' + msg = msg or 'Invalid vault password was provided' + if not secret: + raise AnsibleVaultPasswordError(msg) + + +class VaultSecret: + '''Opaque/abstract objects for a single vault secret. ie, a password or a key.''' + + def __init__(self, _bytes=None): + # FIXME: ? that seems wrong... Unset etc? + self._bytes = _bytes + + @property + def bytes(self): + '''The secret as a bytestring. + + Sub classes that store text types will need to override to encode the text to bytes. + ''' + return self._bytes + + def load(self): + return self._bytes + + +class PromptVaultSecret(VaultSecret): + default_prompt_formats = ["Vault password (%s): "] + + def __init__(self, _bytes=None, vault_id=None, prompt_formats=None): + super(PromptVaultSecret, self).__init__(_bytes=_bytes) + self.vault_id = vault_id + + if prompt_formats is None: + self.prompt_formats = self.default_prompt_formats + else: + self.prompt_formats = prompt_formats + + @property + def bytes(self): + return self._bytes + + def load(self): + self._bytes = self.ask_vault_passwords() + + def ask_vault_passwords(self): + b_vault_passwords = [] + + for prompt_format in self.prompt_formats: + prompt = prompt_format % {'vault_id': self.vault_id} + try: + vault_pass = display.prompt(prompt, private=True) + except EOFError: + raise AnsibleVaultError('EOFError (ctrl-d) on prompt for (%s)' % self.vault_id) + + verify_secret_is_not_empty(vault_pass) + + b_vault_pass = to_bytes(vault_pass, errors='strict', nonstring='simplerepr').strip() + b_vault_passwords.append(b_vault_pass) + + # Make sure the passwords match by comparing them all to the first password + for b_vault_password in b_vault_passwords: + self.confirm(b_vault_passwords[0], b_vault_password) + + if b_vault_passwords: + return b_vault_passwords[0] + + return None + + def confirm(self, b_vault_pass_1, b_vault_pass_2): + # enforce no newline chars at the end of passwords + + if b_vault_pass_1 != b_vault_pass_2: + # FIXME: more specific exception + raise AnsibleError("Passwords do not match") + + +def script_is_client(filename): + '''Determine if a vault secret script is a client script that can be given --vault-id args''' + + # if password script is 'something-client' or 'something-client.[sh|py|rb|etc]' + # script_name can still have '.' or could be entire filename if there is no ext + script_name, dummy = os.path.splitext(filename) + + # TODO: for now, this is entirely based on filename + if script_name.endswith('-client'): + return True + + return False + + +def get_file_vault_secret(filename=None, vault_id=None, encoding=None, loader=None): + ''' Get secret from file content or execute file and get secret from stdout ''' + + # we unfrack but not follow the full path/context to possible vault script + # so when the script uses 'adjacent' file for configuration or similar + # it still works (as inventory scripts often also do). + # while files from --vault-password-file are already unfracked, other sources are not + this_path = unfrackpath(filename, follow=False) + if not os.path.exists(this_path): + raise AnsibleError("The vault password file %s was not found" % this_path) + + # it is a script? + if loader.is_executable(this_path): + + if script_is_client(filename): + # this is special script type that handles vault ids + display.vvvv(u'The vault password file %s is a client script.' % to_text(this_path)) + # TODO: pass vault_id_name to script via cli + return ClientScriptVaultSecret(filename=this_path, vault_id=vault_id, encoding=encoding, loader=loader) + + # just a plain vault password script. No args, returns a byte array + return ScriptVaultSecret(filename=this_path, encoding=encoding, loader=loader) + + return FileVaultSecret(filename=this_path, encoding=encoding, loader=loader) + + +# TODO: mv these classes to a separate file so we don't pollute vault with 'subprocess' etc +class FileVaultSecret(VaultSecret): + def __init__(self, filename=None, encoding=None, loader=None): + super(FileVaultSecret, self).__init__() + self.filename = filename + self.loader = loader + + self.encoding = encoding or 'utf8' + + # We could load from file here, but that is eventually a pain to test + self._bytes = None + self._text = None + + @property + def bytes(self): + if self._bytes: + return self._bytes + if self._text: + return self._text.encode(self.encoding) + return None + + def load(self): + self._bytes = self._read_file(self.filename) + + def _read_file(self, filename): + """ + Read a vault password from a file or if executable, execute the script and + retrieve password from STDOUT + """ + + # TODO: replace with use of self.loader + try: + with open(filename, "rb") as f: + vault_pass = f.read().strip() + except (OSError, IOError) as e: + raise AnsibleError("Could not read vault password file %s: %s" % (filename, e)) + + b_vault_data, dummy = self.loader._decrypt_if_vault_data(vault_pass, filename) + + vault_pass = b_vault_data.strip(b'\r\n') + + verify_secret_is_not_empty(vault_pass, + msg='Invalid vault password was provided from file (%s)' % filename) + + return vault_pass + + def __repr__(self): + if self.filename: + return "%s(filename='%s')" % (self.__class__.__name__, self.filename) + return "%s()" % (self.__class__.__name__) + + +class ScriptVaultSecret(FileVaultSecret): + def _read_file(self, filename): + if not self.loader.is_executable(filename): + raise AnsibleVaultError("The vault password script %s was not executable" % filename) + + command = self._build_command() + + stdout, stderr, p = self._run(command) + + self._check_results(stdout, stderr, p) + + vault_pass = stdout.strip(b'\r\n') + + empty_password_msg = 'Invalid vault password was provided from script (%s)' % filename + verify_secret_is_not_empty(vault_pass, msg=empty_password_msg) + + return vault_pass + + def _run(self, command): + try: + # STDERR not captured to make it easier for users to prompt for input in their scripts + p = subprocess.Popen(command, stdout=subprocess.PIPE) + except OSError as e: + msg_format = "Problem running vault password script %s (%s)." \ + " If this is not a script, remove the executable bit from the file." + msg = msg_format % (self.filename, e) + + raise AnsibleError(msg) + + stdout, stderr = p.communicate() + return stdout, stderr, p + + def _check_results(self, stdout, stderr, popen): + if popen.returncode != 0: + raise AnsibleError("Vault password script %s returned non-zero (%s): %s" % + (self.filename, popen.returncode, stderr)) + + def _build_command(self): + return [self.filename] + + +class ClientScriptVaultSecret(ScriptVaultSecret): + VAULT_ID_UNKNOWN_RC = 2 + + def __init__(self, filename=None, encoding=None, loader=None, vault_id=None): + super(ClientScriptVaultSecret, self).__init__(filename=filename, + encoding=encoding, + loader=loader) + self._vault_id = vault_id + display.vvvv(u'Executing vault password client script: %s --vault-id %s' % (to_text(filename), to_text(vault_id))) + + def _run(self, command): + try: + p = subprocess.Popen(command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + except OSError as e: + msg_format = "Problem running vault password client script %s (%s)." \ + " If this is not a script, remove the executable bit from the file." + msg = msg_format % (self.filename, e) + + raise AnsibleError(msg) + + stdout, stderr = p.communicate() + return stdout, stderr, p + + def _check_results(self, stdout, stderr, popen): + if popen.returncode == self.VAULT_ID_UNKNOWN_RC: + raise AnsibleError('Vault password client script %s did not find a secret for vault-id=%s: %s' % + (self.filename, self._vault_id, stderr)) + + if popen.returncode != 0: + raise AnsibleError("Vault password client script %s returned non-zero (%s) when getting secret for vault-id=%s: %s" % + (self.filename, popen.returncode, self._vault_id, stderr)) + + def _build_command(self): + command = [self.filename] + if self._vault_id: + command.extend(['--vault-id', self._vault_id]) + + return command + + def __repr__(self): + if self.filename: + return "%s(filename='%s', vault_id='%s')" % \ + (self.__class__.__name__, self.filename, self._vault_id) + return "%s()" % (self.__class__.__name__) + + +def match_secrets(secrets, target_vault_ids): + '''Find all VaultSecret objects that are mapped to any of the target_vault_ids in secrets''' + if not secrets: + return [] + + matches = [(vault_id, secret) for vault_id, secret in secrets if vault_id in target_vault_ids] + return matches + + +def match_best_secret(secrets, target_vault_ids): + '''Find the best secret from secrets that matches target_vault_ids + + Since secrets should be ordered so the early secrets are 'better' than later ones, this + just finds all the matches, then returns the first secret''' + matches = match_secrets(secrets, target_vault_ids) + if matches: + return matches[0] + # raise exception? + return None + + +def match_encrypt_vault_id_secret(secrets, encrypt_vault_id=None): + # See if the --encrypt-vault-id matches a vault-id + display.vvvv(u'encrypt_vault_id=%s' % to_text(encrypt_vault_id)) + + if encrypt_vault_id is None: + raise AnsibleError('match_encrypt_vault_id_secret requires a non None encrypt_vault_id') + + encrypt_vault_id_matchers = [encrypt_vault_id] + encrypt_secret = match_best_secret(secrets, encrypt_vault_id_matchers) + + # return the best match for --encrypt-vault-id + if encrypt_secret: + return encrypt_secret + + # If we specified a encrypt_vault_id and we couldn't find it, dont + # fallback to using the first/best secret + raise AnsibleVaultError('Did not find a match for --encrypt-vault-id=%s in the known vault-ids %s' % (encrypt_vault_id, + [_v for _v, _vs in secrets])) + + +def match_encrypt_secret(secrets, encrypt_vault_id=None): + '''Find the best/first/only secret in secrets to use for encrypting''' + + display.vvvv(u'encrypt_vault_id=%s' % to_text(encrypt_vault_id)) + # See if the --encrypt-vault-id matches a vault-id + if encrypt_vault_id: + return match_encrypt_vault_id_secret(secrets, + encrypt_vault_id=encrypt_vault_id) + + # Find the best/first secret from secrets since we didnt specify otherwise + # ie, consider all of the available secrets as matches + _vault_id_matchers = [_vault_id for _vault_id, dummy in secrets] + best_secret = match_best_secret(secrets, _vault_id_matchers) + + # can be empty list sans any tuple + return best_secret + + +class VaultLib: + def __init__(self, secrets=None): + self.secrets = secrets or [] + self.cipher_name = None + self.b_version = b'1.2' + + @staticmethod + def is_encrypted(vaulttext): + return is_encrypted(vaulttext) + + def encrypt(self, plaintext, secret=None, vault_id=None, salt=None): + """Vault encrypt a piece of data. + + :arg plaintext: a text or byte string to encrypt. + :returns: a utf-8 encoded byte str of encrypted data. The string + contains a header identifying this as vault encrypted data and + formatted to newline terminated lines of 80 characters. This is + suitable for dumping as is to a vault file. + + If the string passed in is a text string, it will be encoded to UTF-8 + before encryption. + """ + + if secret is None: + if self.secrets: + dummy, secret = match_encrypt_secret(self.secrets) + else: + raise AnsibleVaultError("A vault password must be specified to encrypt data") + + b_plaintext = to_bytes(plaintext, errors='surrogate_or_strict') + + if is_encrypted(b_plaintext): + raise AnsibleError("input is already encrypted") + + if not self.cipher_name or self.cipher_name not in CIPHER_WRITE_WHITELIST: + self.cipher_name = u"AES256" + + try: + this_cipher = CIPHER_MAPPING[self.cipher_name]() + except KeyError: + raise AnsibleError(u"{0} cipher could not be found".format(self.cipher_name)) + + # encrypt data + if vault_id: + display.vvvvv(u'Encrypting with vault_id "%s" and vault secret %s' % (to_text(vault_id), to_text(secret))) + else: + display.vvvvv(u'Encrypting without a vault_id using vault secret %s' % to_text(secret)) + + b_ciphertext = this_cipher.encrypt(b_plaintext, secret, salt) + + # format the data for output to the file + b_vaulttext = format_vaulttext_envelope(b_ciphertext, + self.cipher_name, + vault_id=vault_id) + return b_vaulttext + + def decrypt(self, vaulttext, filename=None, obj=None): + '''Decrypt a piece of vault encrypted data. + + :arg vaulttext: a string to decrypt. Since vault encrypted data is an + ascii text format this can be either a byte str or unicode string. + :kwarg filename: a filename that the data came from. This is only + used to make better error messages in case the data cannot be + decrypted. + :returns: a byte string containing the decrypted data and the vault-id that was used + + ''' + plaintext, vault_id, vault_secret = self.decrypt_and_get_vault_id(vaulttext, filename=filename, obj=obj) + return plaintext + + def decrypt_and_get_vault_id(self, vaulttext, filename=None, obj=None): + """Decrypt a piece of vault encrypted data. + + :arg vaulttext: a string to decrypt. Since vault encrypted data is an + ascii text format this can be either a byte str or unicode string. + :kwarg filename: a filename that the data came from. This is only + used to make better error messages in case the data cannot be + decrypted. + :returns: a byte string containing the decrypted data and the vault-id vault-secret that was used + + """ + b_vaulttext = to_bytes(vaulttext, errors='strict', encoding='utf-8') + + if self.secrets is None: + raise AnsibleVaultError("A vault password must be specified to decrypt data") + + if not is_encrypted(b_vaulttext): + msg = "input is not vault encrypted data. " + if filename: + msg += "%s is not a vault encrypted file" % to_native(filename) + raise AnsibleError(msg) + + b_vaulttext, dummy, cipher_name, vault_id = parse_vaulttext_envelope(b_vaulttext, filename=filename) + + # create the cipher object, note that the cipher used for decrypt can + # be different than the cipher used for encrypt + if cipher_name in CIPHER_WHITELIST: + this_cipher = CIPHER_MAPPING[cipher_name]() + else: + raise AnsibleError("{0} cipher could not be found".format(cipher_name)) + + b_plaintext = None + + if not self.secrets: + raise AnsibleVaultError('Attempting to decrypt but no vault secrets found') + + # WARNING: Currently, the vault id is not required to match the vault id in the vault blob to + # decrypt a vault properly. The vault id in the vault blob is not part of the encrypted + # or signed vault payload. There is no cryptographic checking/verification/validation of the + # vault blobs vault id. It can be tampered with and changed. The vault id is just a nick + # name to use to pick the best secret and provide some ux/ui info. + + # iterate over all the applicable secrets (all of them by default) until one works... + # if we specify a vault_id, only the corresponding vault secret is checked and + # we check it first. + + vault_id_matchers = [] + vault_id_used = None + vault_secret_used = None + + if vault_id: + display.vvvvv(u'Found a vault_id (%s) in the vaulttext' % to_text(vault_id)) + vault_id_matchers.append(vault_id) + _matches = match_secrets(self.secrets, vault_id_matchers) + if _matches: + display.vvvvv(u'We have a secret associated with vault id (%s), will try to use to decrypt %s' % (to_text(vault_id), to_text(filename))) + else: + display.vvvvv(u'Found a vault_id (%s) in the vault text, but we do not have a associated secret (--vault-id)' % to_text(vault_id)) + + # Not adding the other secrets to vault_secret_ids enforces a match between the vault_id from the vault_text and + # the known vault secrets. + if not C.DEFAULT_VAULT_ID_MATCH: + # Add all of the known vault_ids as candidates for decrypting a vault. + vault_id_matchers.extend([_vault_id for _vault_id, _dummy in self.secrets if _vault_id != vault_id]) + + matched_secrets = match_secrets(self.secrets, vault_id_matchers) + + # for vault_secret_id in vault_secret_ids: + for vault_secret_id, vault_secret in matched_secrets: + display.vvvvv(u'Trying to use vault secret=(%s) id=%s to decrypt %s' % (to_text(vault_secret), to_text(vault_secret_id), to_text(filename))) + + try: + # secret = self.secrets[vault_secret_id] + display.vvvv(u'Trying secret %s for vault_id=%s' % (to_text(vault_secret), to_text(vault_secret_id))) + b_plaintext = this_cipher.decrypt(b_vaulttext, vault_secret) + if b_plaintext is not None: + vault_id_used = vault_secret_id + vault_secret_used = vault_secret + file_slug = '' + if filename: + file_slug = ' of "%s"' % filename + display.vvvvv( + u'Decrypt%s successful with secret=%s and vault_id=%s' % (to_text(file_slug), to_text(vault_secret), to_text(vault_secret_id)) + ) + break + except AnsibleVaultFormatError as exc: + exc.obj = obj + msg = u"There was a vault format error" + if filename: + msg += u' in %s' % (to_text(filename)) + msg += u': %s' % to_text(exc) + display.warning(msg, formatted=True) + raise + except AnsibleError as e: + display.vvvv(u'Tried to use the vault secret (%s) to decrypt (%s) but it failed. Error: %s' % + (to_text(vault_secret_id), to_text(filename), e)) + continue + else: + msg = "Decryption failed (no vault secrets were found that could decrypt)" + if filename: + msg += " on %s" % to_native(filename) + raise AnsibleVaultError(msg) + + if b_plaintext is None: + msg = "Decryption failed" + if filename: + msg += " on %s" % to_native(filename) + raise AnsibleError(msg) + + return b_plaintext, vault_id_used, vault_secret_used + + +class VaultEditor: + + def __init__(self, vault=None): + # TODO: it may be more useful to just make VaultSecrets and index of VaultLib objects... + self.vault = vault or VaultLib() + + # TODO: mv shred file stuff to it's own class + def _shred_file_custom(self, tmp_path): + """"Destroy a file, when shred (core-utils) is not available + + Unix `shred' destroys files "so that they can be recovered only with great difficulty with + specialised hardware, if at all". It is based on the method from the paper + "Secure Deletion of Data from Magnetic and Solid-State Memory", + Proceedings of the Sixth USENIX Security Symposium (San Jose, California, July 22-25, 1996). + + We do not go to that length to re-implement shred in Python; instead, overwriting with a block + of random data should suffice. + + See https://github.com/ansible/ansible/pull/13700 . + """ + + file_len = os.path.getsize(tmp_path) + + if file_len > 0: # avoid work when file was empty + max_chunk_len = min(1024 * 1024 * 2, file_len) + + passes = 3 + with open(tmp_path, "wb") as fh: + for _ in range(passes): + fh.seek(0, 0) + # get a random chunk of data, each pass with other length + chunk_len = random.randint(max_chunk_len // 2, max_chunk_len) + data = os.urandom(chunk_len) + + for _ in range(0, file_len // chunk_len): + fh.write(data) + fh.write(data[:file_len % chunk_len]) + + # FIXME remove this assert once we have unittests to check its accuracy + if fh.tell() != file_len: + raise AnsibleAssertionError() + + os.fsync(fh) + + def _shred_file(self, tmp_path): + """Securely destroy a decrypted file + + Note standard limitations of GNU shred apply (For flash, overwriting would have no effect + due to wear leveling; for other storage systems, the async kernel->filesystem->disk calls never + guarantee data hits the disk; etc). Furthermore, if your tmp dirs is on tmpfs (ramdisks), + it is a non-issue. + + Nevertheless, some form of overwriting the data (instead of just removing the fs index entry) is + a good idea. If shred is not available (e.g. on windows, or no core-utils installed), fall back on + a custom shredding method. + """ + + if not os.path.isfile(tmp_path): + # file is already gone + return + + try: + r = subprocess.call(['shred', tmp_path]) + except (OSError, ValueError): + # shred is not available on this system, or some other error occurred. + # ValueError caught because macOS El Capitan is raising an + # exception big enough to hit a limit in python2-2.7.11 and below. + # Symptom is ValueError: insecure pickle when shred is not + # installed there. + r = 1 + + if r != 0: + # we could not successfully execute unix shred; therefore, do custom shred. + self._shred_file_custom(tmp_path) + + os.remove(tmp_path) + + def _edit_file_helper(self, filename, secret, existing_data=None, force_save=False, vault_id=None): + + # Create a tempfile + root, ext = os.path.splitext(os.path.realpath(filename)) + fd, tmp_path = tempfile.mkstemp(suffix=ext, dir=C.DEFAULT_LOCAL_TMP) + + cmd = self._editor_shell_command(tmp_path) + try: + if existing_data: + self.write_data(existing_data, fd, shred=False) + except Exception: + # if an error happens, destroy the decrypted file + self._shred_file(tmp_path) + raise + finally: + os.close(fd) + + try: + # drop the user into an editor on the tmp file + subprocess.call(cmd) + except Exception as e: + # if an error happens, destroy the decrypted file + self._shred_file(tmp_path) + raise AnsibleError('Unable to execute the command "%s": %s' % (' '.join(cmd), to_native(e))) + + b_tmpdata = self.read_data(tmp_path) + + # Do nothing if the content has not changed + if force_save or existing_data != b_tmpdata: + + # encrypt new data and write out to tmp + # An existing vaultfile will always be UTF-8, + # so decode to unicode here + b_ciphertext = self.vault.encrypt(b_tmpdata, secret, vault_id=vault_id) + self.write_data(b_ciphertext, tmp_path) + + # shuffle tmp file into place + self.shuffle_files(tmp_path, filename) + display.vvvvv(u'Saved edited file "%s" encrypted using %s and vault id "%s"' % (to_text(filename), to_text(secret), to_text(vault_id))) + + # always shred temp, jic + self._shred_file(tmp_path) + + def _real_path(self, filename): + # '-' is special to VaultEditor, dont expand it. + if filename == '-': + return filename + + real_path = os.path.realpath(filename) + return real_path + + def encrypt_bytes(self, b_plaintext, secret, vault_id=None): + + b_ciphertext = self.vault.encrypt(b_plaintext, secret, vault_id=vault_id) + + return b_ciphertext + + def encrypt_file(self, filename, secret, vault_id=None, output_file=None): + + # A file to be encrypted into a vaultfile could be any encoding + # so treat the contents as a byte string. + + # follow the symlink + filename = self._real_path(filename) + + b_plaintext = self.read_data(filename) + b_ciphertext = self.vault.encrypt(b_plaintext, secret, vault_id=vault_id) + self.write_data(b_ciphertext, output_file or filename) + + def decrypt_file(self, filename, output_file=None): + + # follow the symlink + filename = self._real_path(filename) + + ciphertext = self.read_data(filename) + + try: + plaintext = self.vault.decrypt(ciphertext, filename=filename) + except AnsibleError as e: + raise AnsibleError("%s for %s" % (to_native(e), to_native(filename))) + self.write_data(plaintext, output_file or filename, shred=False) + + def create_file(self, filename, secret, vault_id=None): + """ create a new encrypted file """ + + dirname = os.path.dirname(filename) + if dirname and not os.path.exists(dirname): + display.warning(u"%s does not exist, creating..." % to_text(dirname)) + makedirs_safe(dirname) + + # FIXME: If we can raise an error here, we can probably just make it + # behave like edit instead. + if os.path.isfile(filename): + raise AnsibleError("%s exists, please use 'edit' instead" % filename) + + self._edit_file_helper(filename, secret, vault_id=vault_id) + + def edit_file(self, filename): + vault_id_used = None + vault_secret_used = None + # follow the symlink + filename = self._real_path(filename) + + b_vaulttext = self.read_data(filename) + + # vault or yaml files are always utf8 + vaulttext = to_text(b_vaulttext) + + try: + # vaulttext gets converted back to bytes, but alas + # TODO: return the vault_id that worked? + plaintext, vault_id_used, vault_secret_used = self.vault.decrypt_and_get_vault_id(vaulttext) + except AnsibleError as e: + raise AnsibleError("%s for %s" % (to_native(e), to_native(filename))) + + # Figure out the vault id from the file, to select the right secret to re-encrypt it + # (duplicates parts of decrypt, but alas...) + dummy, dummy, cipher_name, vault_id = parse_vaulttext_envelope(b_vaulttext, filename=filename) + + # vault id here may not be the vault id actually used for decrypting + # as when the edited file has no vault-id but is decrypted by non-default id in secrets + # (vault_id=default, while a different vault-id decrypted) + + # we want to get rid of files encrypted with the AES cipher + force_save = (cipher_name not in CIPHER_WRITE_WHITELIST) + + # Keep the same vault-id (and version) as in the header + self._edit_file_helper(filename, vault_secret_used, existing_data=plaintext, force_save=force_save, vault_id=vault_id) + + def plaintext(self, filename): + + b_vaulttext = self.read_data(filename) + vaulttext = to_text(b_vaulttext) + + try: + plaintext = self.vault.decrypt(vaulttext, filename=filename) + return plaintext + except AnsibleError as e: + raise AnsibleVaultError("%s for %s" % (to_native(e), to_native(filename))) + + # FIXME/TODO: make this use VaultSecret + def rekey_file(self, filename, new_vault_secret, new_vault_id=None): + + # follow the symlink + filename = self._real_path(filename) + + prev = os.stat(filename) + b_vaulttext = self.read_data(filename) + vaulttext = to_text(b_vaulttext) + + display.vvvvv(u'Rekeying file "%s" to with new vault-id "%s" and vault secret %s' % + (to_text(filename), to_text(new_vault_id), to_text(new_vault_secret))) + try: + plaintext, vault_id_used, _dummy = self.vault.decrypt_and_get_vault_id(vaulttext) + except AnsibleError as e: + raise AnsibleError("%s for %s" % (to_native(e), to_native(filename))) + + # This is more or less an assert, see #18247 + if new_vault_secret is None: + raise AnsibleError('The value for the new_password to rekey %s with is not valid' % filename) + + # FIXME: VaultContext...? could rekey to a different vault_id in the same VaultSecrets + + # Need a new VaultLib because the new vault data can be a different + # vault lib format or cipher (for ex, when we migrate 1.0 style vault data to + # 1.1 style data we change the version and the cipher). This is where a VaultContext might help + + # the new vault will only be used for encrypting, so it doesn't need the vault secrets + # (we will pass one in directly to encrypt) + new_vault = VaultLib(secrets={}) + b_new_vaulttext = new_vault.encrypt(plaintext, new_vault_secret, vault_id=new_vault_id) + + self.write_data(b_new_vaulttext, filename) + + # preserve permissions + os.chmod(filename, prev.st_mode) + os.chown(filename, prev.st_uid, prev.st_gid) + + display.vvvvv(u'Rekeyed file "%s" (decrypted with vault id "%s") was encrypted with new vault-id "%s" and vault secret %s' % + (to_text(filename), to_text(vault_id_used), to_text(new_vault_id), to_text(new_vault_secret))) + + def read_data(self, filename): + + try: + if filename == '-': + data = sys.stdin.buffer.read() + else: + with open(filename, "rb") as fh: + data = fh.read() + except Exception as e: + msg = to_native(e) + if not msg: + msg = repr(e) + raise AnsibleError('Unable to read source file (%s): %s' % (to_native(filename), msg)) + + return data + + def write_data(self, data, thefile, shred=True, mode=0o600): + # TODO: add docstrings for arg types since this code is picky about that + """Write the data bytes to given path + + This is used to write a byte string to a file or stdout. It is used for + writing the results of vault encryption or decryption. It is used for + saving the ciphertext after encryption and it is also used for saving the + plaintext after decrypting a vault. The type of the 'data' arg should be bytes, + since in the plaintext case, the original contents can be of any text encoding + or arbitrary binary data. + + When used to write the result of vault encryption, the val of the 'data' arg + should be a utf-8 encoded byte string and not a text typ and not a text type.. + + When used to write the result of vault decryption, the val of the 'data' arg + should be a byte string and not a text type. + + :arg data: the byte string (bytes) data + :arg thefile: file descriptor or filename to save 'data' to. + :arg shred: if shred==True, make sure that the original data is first shredded so that is cannot be recovered. + :returns: None + """ + # FIXME: do we need this now? data_bytes should always be a utf-8 byte string + b_file_data = to_bytes(data, errors='strict') + + # check if we have a file descriptor instead of a path + is_fd = False + try: + is_fd = (isinstance(thefile, int) and fcntl.fcntl(thefile, fcntl.F_GETFD) != -1) + except Exception: + pass + + if is_fd: + # if passed descriptor, use that to ensure secure access, otherwise it is a string. + # assumes the fd is securely opened by caller (mkstemp) + os.ftruncate(thefile, 0) + os.write(thefile, b_file_data) + elif thefile == '-': + # get a ref to either sys.stdout.buffer for py3 or plain old sys.stdout for py2 + # We need sys.stdout.buffer on py3 so we can write bytes to it since the plaintext + # of the vaulted object could be anything/binary/etc + output = getattr(sys.stdout, 'buffer', sys.stdout) + output.write(b_file_data) + else: + # file names are insecure and prone to race conditions, so remove and create securely + if os.path.isfile(thefile): + if shred: + self._shred_file(thefile) + else: + os.remove(thefile) + + # when setting new umask, we get previous as return + current_umask = os.umask(0o077) + try: + try: + # create file with secure permissions + fd = os.open(thefile, os.O_CREAT | os.O_EXCL | os.O_RDWR | os.O_TRUNC, mode) + except OSError as ose: + # Want to catch FileExistsError, which doesn't exist in Python 2, so catch OSError + # and compare the error number to get equivalent behavior in Python 2/3 + if ose.errno == errno.EEXIST: + raise AnsibleError('Vault file got recreated while we were operating on it: %s' % to_native(ose)) + + raise AnsibleError('Problem creating temporary vault file: %s' % to_native(ose)) + + try: + # now write to the file and ensure ours is only data in it + os.ftruncate(fd, 0) + os.write(fd, b_file_data) + except OSError as e: + raise AnsibleError('Unable to write to temporary vault file: %s' % to_native(e)) + finally: + # Make sure the file descriptor is always closed and reset umask + os.close(fd) + finally: + os.umask(current_umask) + + def shuffle_files(self, src, dest): + prev = None + # overwrite dest with src + if os.path.isfile(dest): + prev = os.stat(dest) + # old file 'dest' was encrypted, no need to _shred_file + os.remove(dest) + shutil.move(src, dest) + + # reset permissions if needed + if prev is not None: + # TODO: selinux, ACLs, xattr? + os.chmod(dest, prev.st_mode) + os.chown(dest, prev.st_uid, prev.st_gid) + + def _editor_shell_command(self, filename): + env_editor = os.environ.get('EDITOR', 'vi') + editor = shlex.split(env_editor) + editor.append(filename) + + return editor + + +######################################## +# CIPHERS # +######################################## + +class VaultAES256: + + """ + Vault implementation using AES-CTR with an HMAC-SHA256 authentication code. + Keys are derived using PBKDF2 + """ + + # http://www.daemonology.net/blog/2009-06-11-cryptographic-right-answers.html + + # Note: strings in this class should be byte strings by default. + + def __init__(self): + if not HAS_CRYPTOGRAPHY: + raise AnsibleError(NEED_CRYPTO_LIBRARY) + + @staticmethod + def _create_key_cryptography(b_password, b_salt, key_length, iv_length): + kdf = PBKDF2HMAC( + algorithm=hashes.SHA256(), + length=2 * key_length + iv_length, + salt=b_salt, + iterations=10000, + backend=CRYPTOGRAPHY_BACKEND) + b_derivedkey = kdf.derive(b_password) + + return b_derivedkey + + @classmethod + def _gen_key_initctr(cls, b_password, b_salt): + # 16 for AES 128, 32 for AES256 + key_length = 32 + + if HAS_CRYPTOGRAPHY: + # AES is a 128-bit block cipher, so IVs and counter nonces are 16 bytes + iv_length = algorithms.AES.block_size // 8 + + b_derivedkey = cls._create_key_cryptography(b_password, b_salt, key_length, iv_length) + b_iv = b_derivedkey[(key_length * 2):(key_length * 2) + iv_length] + else: + raise AnsibleError(NEED_CRYPTO_LIBRARY + '(Detected in initctr)') + + b_key1 = b_derivedkey[:key_length] + b_key2 = b_derivedkey[key_length:(key_length * 2)] + + return b_key1, b_key2, b_iv + + @staticmethod + def _encrypt_cryptography(b_plaintext, b_key1, b_key2, b_iv): + cipher = C_Cipher(algorithms.AES(b_key1), modes.CTR(b_iv), CRYPTOGRAPHY_BACKEND) + encryptor = cipher.encryptor() + padder = padding.PKCS7(algorithms.AES.block_size).padder() + b_ciphertext = encryptor.update(padder.update(b_plaintext) + padder.finalize()) + b_ciphertext += encryptor.finalize() + + # COMBINE SALT, DIGEST AND DATA + hmac = HMAC(b_key2, hashes.SHA256(), CRYPTOGRAPHY_BACKEND) + hmac.update(b_ciphertext) + b_hmac = hmac.finalize() + + return to_bytes(hexlify(b_hmac), errors='surrogate_or_strict'), hexlify(b_ciphertext) + + @classmethod + def encrypt(cls, b_plaintext, secret, salt=None): + + if secret is None: + raise AnsibleVaultError('The secret passed to encrypt() was None') + + if salt is None: + b_salt = os.urandom(32) + elif not salt: + raise AnsibleVaultError('Empty or invalid salt passed to encrypt()') + else: + b_salt = to_bytes(salt) + + b_password = secret.bytes + b_key1, b_key2, b_iv = cls._gen_key_initctr(b_password, b_salt) + + if HAS_CRYPTOGRAPHY: + b_hmac, b_ciphertext = cls._encrypt_cryptography(b_plaintext, b_key1, b_key2, b_iv) + else: + raise AnsibleError(NEED_CRYPTO_LIBRARY + '(Detected in encrypt)') + + b_vaulttext = b'\n'.join([hexlify(b_salt), b_hmac, b_ciphertext]) + # Unnecessary but getting rid of it is a backwards incompatible vault + # format change + b_vaulttext = hexlify(b_vaulttext) + return b_vaulttext + + @classmethod + def _decrypt_cryptography(cls, b_ciphertext, b_crypted_hmac, b_key1, b_key2, b_iv): + # b_key1, b_key2, b_iv = self._gen_key_initctr(b_password, b_salt) + # EXIT EARLY IF DIGEST DOESN'T MATCH + hmac = HMAC(b_key2, hashes.SHA256(), CRYPTOGRAPHY_BACKEND) + hmac.update(b_ciphertext) + try: + hmac.verify(_unhexlify(b_crypted_hmac)) + except InvalidSignature as e: + raise AnsibleVaultError('HMAC verification failed: %s' % e) + + cipher = C_Cipher(algorithms.AES(b_key1), modes.CTR(b_iv), CRYPTOGRAPHY_BACKEND) + decryptor = cipher.decryptor() + unpadder = padding.PKCS7(128).unpadder() + b_plaintext = unpadder.update( + decryptor.update(b_ciphertext) + decryptor.finalize() + ) + unpadder.finalize() + + return b_plaintext + + @staticmethod + def _is_equal(b_a, b_b): + """ + Comparing 2 byte arrays in constant time to avoid timing attacks. + + It would be nice if there were a library for this but hey. + """ + if not (isinstance(b_a, binary_type) and isinstance(b_b, binary_type)): + raise TypeError('_is_equal can only be used to compare two byte strings') + + # http://codahale.com/a-lesson-in-timing-attacks/ + if len(b_a) != len(b_b): + return False + + result = 0 + for b_x, b_y in zip(b_a, b_b): + result |= b_x ^ b_y + return result == 0 + + @classmethod + def decrypt(cls, b_vaulttext, secret): + + b_ciphertext, b_salt, b_crypted_hmac = parse_vaulttext(b_vaulttext) + + # TODO: would be nice if a VaultSecret could be passed directly to _decrypt_* + # (move _gen_key_initctr() to a AES256 VaultSecret or VaultContext impl?) + # though, likely needs to be python cryptography specific impl that basically + # creates a Cipher() with b_key1, a Mode.CTR() with b_iv, and a HMAC() with sign key b_key2 + b_password = secret.bytes + + b_key1, b_key2, b_iv = cls._gen_key_initctr(b_password, b_salt) + + if HAS_CRYPTOGRAPHY: + b_plaintext = cls._decrypt_cryptography(b_ciphertext, b_crypted_hmac, b_key1, b_key2, b_iv) + else: + raise AnsibleError(NEED_CRYPTO_LIBRARY + '(Detected in decrypt)') + + return b_plaintext + + +# Keys could be made bytes later if the code that gets the data is more +# naturally byte-oriented +CIPHER_MAPPING = { + u'AES256': VaultAES256, +} diff --git a/lib/ansible/parsing/yaml/__init__.py b/lib/ansible/parsing/yaml/__init__.py new file mode 100644 index 0000000..ae8ccff --- /dev/null +++ b/lib/ansible/parsing/yaml/__init__.py @@ -0,0 +1,20 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type diff --git a/lib/ansible/parsing/yaml/constructor.py b/lib/ansible/parsing/yaml/constructor.py new file mode 100644 index 0000000..4b79578 --- /dev/null +++ b/lib/ansible/parsing/yaml/constructor.py @@ -0,0 +1,178 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from yaml.constructor import SafeConstructor, ConstructorError +from yaml.nodes import MappingNode + +from ansible import constants as C +from ansible.module_utils._text import to_bytes, to_native +from ansible.parsing.yaml.objects import AnsibleMapping, AnsibleSequence, AnsibleUnicode, AnsibleVaultEncryptedUnicode +from ansible.parsing.vault import VaultLib +from ansible.utils.display import Display +from ansible.utils.unsafe_proxy import wrap_var + +display = Display() + + +class AnsibleConstructor(SafeConstructor): + def __init__(self, file_name=None, vault_secrets=None): + self._ansible_file_name = file_name + super(AnsibleConstructor, self).__init__() + self._vaults = {} + self.vault_secrets = vault_secrets or [] + self._vaults['default'] = VaultLib(secrets=self.vault_secrets) + + def construct_yaml_map(self, node): + data = AnsibleMapping() + yield data + value = self.construct_mapping(node) + data.update(value) + data.ansible_pos = self._node_position_info(node) + + def construct_mapping(self, node, deep=False): + # Most of this is from yaml.constructor.SafeConstructor. We replicate + # it here so that we can warn users when they have duplicate dict keys + # (pyyaml silently allows overwriting keys) + if not isinstance(node, MappingNode): + raise ConstructorError(None, None, + "expected a mapping node, but found %s" % node.id, + node.start_mark) + self.flatten_mapping(node) + mapping = AnsibleMapping() + + # Add our extra information to the returned value + mapping.ansible_pos = self._node_position_info(node) + + for key_node, value_node in node.value: + key = self.construct_object(key_node, deep=deep) + try: + hash(key) + except TypeError as exc: + raise ConstructorError("while constructing a mapping", node.start_mark, + "found unacceptable key (%s)" % exc, key_node.start_mark) + + if key in mapping: + msg = (u'While constructing a mapping from {1}, line {2}, column {3}, found a duplicate dict key ({0}).' + u' Using last defined value only.'.format(key, *mapping.ansible_pos)) + if C.DUPLICATE_YAML_DICT_KEY == 'warn': + display.warning(msg) + elif C.DUPLICATE_YAML_DICT_KEY == 'error': + raise ConstructorError(context=None, context_mark=None, + problem=to_native(msg), + problem_mark=node.start_mark, + note=None) + else: + # when 'ignore' + display.debug(msg) + + value = self.construct_object(value_node, deep=deep) + mapping[key] = value + + return mapping + + def construct_yaml_str(self, node): + # Override the default string handling function + # to always return unicode objects + value = self.construct_scalar(node) + ret = AnsibleUnicode(value) + + ret.ansible_pos = self._node_position_info(node) + + return ret + + def construct_vault_encrypted_unicode(self, node): + value = self.construct_scalar(node) + b_ciphertext_data = to_bytes(value) + # could pass in a key id here to choose the vault to associate with + # TODO/FIXME: plugin vault selector + vault = self._vaults['default'] + if vault.secrets is None: + raise ConstructorError(context=None, context_mark=None, + problem="found !vault but no vault password provided", + problem_mark=node.start_mark, + note=None) + ret = AnsibleVaultEncryptedUnicode(b_ciphertext_data) + ret.vault = vault + ret.ansible_pos = self._node_position_info(node) + return ret + + def construct_yaml_seq(self, node): + data = AnsibleSequence() + yield data + data.extend(self.construct_sequence(node)) + data.ansible_pos = self._node_position_info(node) + + def construct_yaml_unsafe(self, node): + try: + constructor = getattr(node, 'id', 'object') + if constructor is not None: + constructor = getattr(self, 'construct_%s' % constructor) + except AttributeError: + constructor = self.construct_object + + value = constructor(node) + + return wrap_var(value) + + def _node_position_info(self, node): + # the line number where the previous token has ended (plus empty lines) + # Add one so that the first line is line 1 rather than line 0 + column = node.start_mark.column + 1 + line = node.start_mark.line + 1 + + # in some cases, we may have pre-read the data and then + # passed it to the load() call for YAML, in which case we + # want to override the default datasource (which would be + # '<string>') to the actual filename we read in + datasource = self._ansible_file_name or node.start_mark.name + + return (datasource, line, column) + + +AnsibleConstructor.add_constructor( + u'tag:yaml.org,2002:map', + AnsibleConstructor.construct_yaml_map) + +AnsibleConstructor.add_constructor( + u'tag:yaml.org,2002:python/dict', + AnsibleConstructor.construct_yaml_map) + +AnsibleConstructor.add_constructor( + u'tag:yaml.org,2002:str', + AnsibleConstructor.construct_yaml_str) + +AnsibleConstructor.add_constructor( + u'tag:yaml.org,2002:python/unicode', + AnsibleConstructor.construct_yaml_str) + +AnsibleConstructor.add_constructor( + u'tag:yaml.org,2002:seq', + AnsibleConstructor.construct_yaml_seq) + +AnsibleConstructor.add_constructor( + u'!unsafe', + AnsibleConstructor.construct_yaml_unsafe) + +AnsibleConstructor.add_constructor( + u'!vault', + AnsibleConstructor.construct_vault_encrypted_unicode) + +AnsibleConstructor.add_constructor(u'!vault-encrypted', AnsibleConstructor.construct_vault_encrypted_unicode) diff --git a/lib/ansible/parsing/yaml/dumper.py b/lib/ansible/parsing/yaml/dumper.py new file mode 100644 index 0000000..8701bb8 --- /dev/null +++ b/lib/ansible/parsing/yaml/dumper.py @@ -0,0 +1,122 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import yaml + +from ansible.module_utils.six import text_type, binary_type +from ansible.module_utils.common.yaml import SafeDumper +from ansible.parsing.yaml.objects import AnsibleUnicode, AnsibleSequence, AnsibleMapping, AnsibleVaultEncryptedUnicode +from ansible.utils.unsafe_proxy import AnsibleUnsafeText, AnsibleUnsafeBytes, NativeJinjaUnsafeText, NativeJinjaText +from ansible.template import AnsibleUndefined +from ansible.vars.hostvars import HostVars, HostVarsVars +from ansible.vars.manager import VarsWithSources + + +class AnsibleDumper(SafeDumper): + ''' + A simple stub class that allows us to add representers + for our overridden object types. + ''' + + +def represent_hostvars(self, data): + return self.represent_dict(dict(data)) + + +# Note: only want to represent the encrypted data +def represent_vault_encrypted_unicode(self, data): + return self.represent_scalar(u'!vault', data._ciphertext.decode(), style='|') + + +def represent_unicode(self, data): + return yaml.representer.SafeRepresenter.represent_str(self, text_type(data)) + + +def represent_binary(self, data): + return yaml.representer.SafeRepresenter.represent_binary(self, binary_type(data)) + + +def represent_undefined(self, data): + # Here bool will ensure _fail_with_undefined_error happens + # if the value is Undefined. + # This happens because Jinja sets __bool__ on StrictUndefined + return bool(data) + + +AnsibleDumper.add_representer( + AnsibleUnicode, + represent_unicode, +) + +AnsibleDumper.add_representer( + AnsibleUnsafeText, + represent_unicode, +) + +AnsibleDumper.add_representer( + AnsibleUnsafeBytes, + represent_binary, +) + +AnsibleDumper.add_representer( + HostVars, + represent_hostvars, +) + +AnsibleDumper.add_representer( + HostVarsVars, + represent_hostvars, +) + +AnsibleDumper.add_representer( + VarsWithSources, + represent_hostvars, +) + +AnsibleDumper.add_representer( + AnsibleSequence, + yaml.representer.SafeRepresenter.represent_list, +) + +AnsibleDumper.add_representer( + AnsibleMapping, + yaml.representer.SafeRepresenter.represent_dict, +) + +AnsibleDumper.add_representer( + AnsibleVaultEncryptedUnicode, + represent_vault_encrypted_unicode, +) + +AnsibleDumper.add_representer( + AnsibleUndefined, + represent_undefined, +) + +AnsibleDumper.add_representer( + NativeJinjaUnsafeText, + represent_unicode, +) + +AnsibleDumper.add_representer( + NativeJinjaText, + represent_unicode, +) diff --git a/lib/ansible/parsing/yaml/loader.py b/lib/ansible/parsing/yaml/loader.py new file mode 100644 index 0000000..15bde79 --- /dev/null +++ b/lib/ansible/parsing/yaml/loader.py @@ -0,0 +1,45 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from yaml.resolver import Resolver + +from ansible.parsing.yaml.constructor import AnsibleConstructor +from ansible.module_utils.common.yaml import HAS_LIBYAML, Parser + +if HAS_LIBYAML: + class AnsibleLoader(Parser, AnsibleConstructor, Resolver): # type: ignore[misc] # pylint: disable=inconsistent-mro + def __init__(self, stream, file_name=None, vault_secrets=None): + Parser.__init__(self, stream) + AnsibleConstructor.__init__(self, file_name=file_name, vault_secrets=vault_secrets) + Resolver.__init__(self) +else: + from yaml.composer import Composer + from yaml.reader import Reader + from yaml.scanner import Scanner + + class AnsibleLoader(Reader, Scanner, Parser, Composer, AnsibleConstructor, Resolver): # type: ignore[misc,no-redef] # pylint: disable=inconsistent-mro + def __init__(self, stream, file_name=None, vault_secrets=None): + Reader.__init__(self, stream) + Scanner.__init__(self) + Parser.__init__(self) + Composer.__init__(self) + AnsibleConstructor.__init__(self, file_name=file_name, vault_secrets=vault_secrets) + Resolver.__init__(self) diff --git a/lib/ansible/parsing/yaml/objects.py b/lib/ansible/parsing/yaml/objects.py new file mode 100644 index 0000000..a2e2a66 --- /dev/null +++ b/lib/ansible/parsing/yaml/objects.py @@ -0,0 +1,365 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import string +import sys as _sys + +from collections.abc import Sequence + +import sys +import yaml + +from ansible.module_utils.six import text_type +from ansible.module_utils._text import to_bytes, to_text, to_native + + +class AnsibleBaseYAMLObject(object): + ''' + the base class used to sub-class python built-in objects + so that we can add attributes to them during yaml parsing + + ''' + _data_source = None + _line_number = 0 + _column_number = 0 + + def _get_ansible_position(self): + return (self._data_source, self._line_number, self._column_number) + + def _set_ansible_position(self, obj): + try: + (src, line, col) = obj + except (TypeError, ValueError): + raise AssertionError( + 'ansible_pos can only be set with a tuple/list ' + 'of three values: source, line number, column number' + ) + self._data_source = src + self._line_number = line + self._column_number = col + + ansible_pos = property(_get_ansible_position, _set_ansible_position) + + +class AnsibleMapping(AnsibleBaseYAMLObject, dict): + ''' sub class for dictionaries ''' + pass + + +class AnsibleUnicode(AnsibleBaseYAMLObject, text_type): + ''' sub class for unicode objects ''' + pass + + +class AnsibleSequence(AnsibleBaseYAMLObject, list): + ''' sub class for lists ''' + pass + + +class AnsibleVaultEncryptedUnicode(Sequence, AnsibleBaseYAMLObject): + '''Unicode like object that is not evaluated (decrypted) until it needs to be''' + __UNSAFE__ = True + __ENCRYPTED__ = True + yaml_tag = u'!vault' + + @classmethod + def from_plaintext(cls, seq, vault, secret): + if not vault: + raise vault.AnsibleVaultError('Error creating AnsibleVaultEncryptedUnicode, invalid vault (%s) provided' % vault) + + ciphertext = vault.encrypt(seq, secret) + avu = cls(ciphertext) + avu.vault = vault + return avu + + def __init__(self, ciphertext): + '''A AnsibleUnicode with a Vault attribute that can decrypt it. + + ciphertext is a byte string (str on PY2, bytestring on PY3). + + The .data attribute is a property that returns the decrypted plaintext + of the ciphertext as a PY2 unicode or PY3 string object. + ''' + super(AnsibleVaultEncryptedUnicode, self).__init__() + + # after construction, calling code has to set the .vault attribute to a vaultlib object + self.vault = None + self._ciphertext = to_bytes(ciphertext) + + @property + def data(self): + if not self.vault: + return to_text(self._ciphertext) + return to_text(self.vault.decrypt(self._ciphertext, obj=self)) + + @data.setter + def data(self, value): + self._ciphertext = to_bytes(value) + + def is_encrypted(self): + return self.vault and self.vault.is_encrypted(self._ciphertext) + + def __eq__(self, other): + if self.vault: + return other == self.data + return False + + def __ne__(self, other): + if self.vault: + return other != self.data + return True + + def __reversed__(self): + # This gets inerhited from ``collections.Sequence`` which returns a generator + # make this act more like the string implementation + return to_text(self[::-1], errors='surrogate_or_strict') + + def __str__(self): + return to_native(self.data, errors='surrogate_or_strict') + + def __unicode__(self): + return to_text(self.data, errors='surrogate_or_strict') + + def encode(self, encoding=None, errors=None): + return to_bytes(self.data, encoding=encoding, errors=errors) + + # Methods below are a copy from ``collections.UserString`` + # Some are copied as is, where others are modified to not + # auto wrap with ``self.__class__`` + def __repr__(self): + return repr(self.data) + + def __int__(self, base=10): + return int(self.data, base=base) + + def __float__(self): + return float(self.data) + + def __complex__(self): + return complex(self.data) + + def __hash__(self): + return hash(self.data) + + # This breaks vault, do not define it, we cannot satisfy this + # def __getnewargs__(self): + # return (self.data[:],) + + def __lt__(self, string): + if isinstance(string, AnsibleVaultEncryptedUnicode): + return self.data < string.data + return self.data < string + + def __le__(self, string): + if isinstance(string, AnsibleVaultEncryptedUnicode): + return self.data <= string.data + return self.data <= string + + def __gt__(self, string): + if isinstance(string, AnsibleVaultEncryptedUnicode): + return self.data > string.data + return self.data > string + + def __ge__(self, string): + if isinstance(string, AnsibleVaultEncryptedUnicode): + return self.data >= string.data + return self.data >= string + + def __contains__(self, char): + if isinstance(char, AnsibleVaultEncryptedUnicode): + char = char.data + return char in self.data + + def __len__(self): + return len(self.data) + + def __getitem__(self, index): + return self.data[index] + + def __getslice__(self, start, end): + start = max(start, 0) + end = max(end, 0) + return self.data[start:end] + + def __add__(self, other): + if isinstance(other, AnsibleVaultEncryptedUnicode): + return self.data + other.data + elif isinstance(other, text_type): + return self.data + other + return self.data + to_text(other) + + def __radd__(self, other): + if isinstance(other, text_type): + return other + self.data + return to_text(other) + self.data + + def __mul__(self, n): + return self.data * n + + __rmul__ = __mul__ + + def __mod__(self, args): + return self.data % args + + def __rmod__(self, template): + return to_text(template) % self + + # the following methods are defined in alphabetical order: + def capitalize(self): + return self.data.capitalize() + + def casefold(self): + return self.data.casefold() + + def center(self, width, *args): + return self.data.center(width, *args) + + def count(self, sub, start=0, end=_sys.maxsize): + if isinstance(sub, AnsibleVaultEncryptedUnicode): + sub = sub.data + return self.data.count(sub, start, end) + + def endswith(self, suffix, start=0, end=_sys.maxsize): + return self.data.endswith(suffix, start, end) + + def expandtabs(self, tabsize=8): + return self.data.expandtabs(tabsize) + + def find(self, sub, start=0, end=_sys.maxsize): + if isinstance(sub, AnsibleVaultEncryptedUnicode): + sub = sub.data + return self.data.find(sub, start, end) + + def format(self, *args, **kwds): + return self.data.format(*args, **kwds) + + def format_map(self, mapping): + return self.data.format_map(mapping) + + def index(self, sub, start=0, end=_sys.maxsize): + return self.data.index(sub, start, end) + + def isalpha(self): + return self.data.isalpha() + + def isalnum(self): + return self.data.isalnum() + + def isascii(self): + return self.data.isascii() + + def isdecimal(self): + return self.data.isdecimal() + + def isdigit(self): + return self.data.isdigit() + + def isidentifier(self): + return self.data.isidentifier() + + def islower(self): + return self.data.islower() + + def isnumeric(self): + return self.data.isnumeric() + + def isprintable(self): + return self.data.isprintable() + + def isspace(self): + return self.data.isspace() + + def istitle(self): + return self.data.istitle() + + def isupper(self): + return self.data.isupper() + + def join(self, seq): + return self.data.join(seq) + + def ljust(self, width, *args): + return self.data.ljust(width, *args) + + def lower(self): + return self.data.lower() + + def lstrip(self, chars=None): + return self.data.lstrip(chars) + + maketrans = str.maketrans + + def partition(self, sep): + return self.data.partition(sep) + + def replace(self, old, new, maxsplit=-1): + if isinstance(old, AnsibleVaultEncryptedUnicode): + old = old.data + if isinstance(new, AnsibleVaultEncryptedUnicode): + new = new.data + return self.data.replace(old, new, maxsplit) + + def rfind(self, sub, start=0, end=_sys.maxsize): + if isinstance(sub, AnsibleVaultEncryptedUnicode): + sub = sub.data + return self.data.rfind(sub, start, end) + + def rindex(self, sub, start=0, end=_sys.maxsize): + return self.data.rindex(sub, start, end) + + def rjust(self, width, *args): + return self.data.rjust(width, *args) + + def rpartition(self, sep): + return self.data.rpartition(sep) + + def rstrip(self, chars=None): + return self.data.rstrip(chars) + + def split(self, sep=None, maxsplit=-1): + return self.data.split(sep, maxsplit) + + def rsplit(self, sep=None, maxsplit=-1): + return self.data.rsplit(sep, maxsplit) + + def splitlines(self, keepends=False): + return self.data.splitlines(keepends) + + def startswith(self, prefix, start=0, end=_sys.maxsize): + return self.data.startswith(prefix, start, end) + + def strip(self, chars=None): + return self.data.strip(chars) + + def swapcase(self): + return self.data.swapcase() + + def title(self): + return self.data.title() + + def translate(self, *args): + return self.data.translate(*args) + + def upper(self): + return self.data.upper() + + def zfill(self, width): + return self.data.zfill(width) |