diff options
Diffstat (limited to 'test/support/windows-integration/plugins/action/win_copy.py')
| -rw-r--r-- | test/support/windows-integration/plugins/action/win_copy.py | 522 |
1 files changed, 522 insertions, 0 deletions
diff --git a/test/support/windows-integration/plugins/action/win_copy.py b/test/support/windows-integration/plugins/action/win_copy.py new file mode 100644 index 0000000..adb918b --- /dev/null +++ b/test/support/windows-integration/plugins/action/win_copy.py @@ -0,0 +1,522 @@ +# This file is part of Ansible + +# Copyright (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import base64 +import json +import os +import os.path +import shutil +import tempfile +import traceback +import zipfile + +from ansible import constants as C +from ansible.errors import AnsibleError, AnsibleFileNotFound +from ansible.module_utils._text import to_bytes, to_native, to_text +from ansible.module_utils.parsing.convert_bool import boolean +from ansible.plugins.action import ActionBase +from ansible.utils.hashing import checksum + + +def _walk_dirs(topdir, loader, decrypt=True, base_path=None, local_follow=False, trailing_slash_detector=None, checksum_check=False): + """ + Walk a filesystem tree returning enough information to copy the files. + This is similar to the _walk_dirs function in ``copy.py`` but returns + a dict instead of a tuple for each entry and includes the checksum of + a local file if wanted. + + :arg topdir: The directory that the filesystem tree is rooted at + :arg loader: The self._loader object from ActionBase + :kwarg decrypt: Whether to decrypt a file encrypted with ansible-vault + :kwarg base_path: The initial directory structure to strip off of the + files for the destination directory. If this is None (the default), + the base_path is set to ``top_dir``. + :kwarg local_follow: Whether to follow symlinks on the source. When set + to False, no symlinks are dereferenced. When set to True (the + default), the code will dereference most symlinks. However, symlinks + can still be present if needed to break a circular link. + :kwarg trailing_slash_detector: Function to determine if a path has + a trailing directory separator. Only needed when dealing with paths on + a remote machine (in which case, pass in a function that is aware of the + directory separator conventions on the remote machine). + :kawrg whether to get the checksum of the local file and add to the dict + :returns: dictionary of dictionaries. All of the path elements in the structure are text string. + This separates all the files, directories, and symlinks along with + import information about each:: + + { + 'files'; [{ + src: '/absolute/path/to/copy/from', + dest: 'relative/path/to/copy/to', + checksum: 'b54ba7f5621240d403f06815f7246006ef8c7d43' + }, ...], + 'directories'; [{ + src: '/absolute/path/to/copy/from', + dest: 'relative/path/to/copy/to' + }, ...], + 'symlinks'; [{ + src: '/symlink/target/path', + dest: 'relative/path/to/copy/to' + }, ...], + + } + + The ``symlinks`` field is only populated if ``local_follow`` is set to False + *or* a circular symlink cannot be dereferenced. The ``checksum`` entry is set + to None if checksum_check=False. + + """ + # Convert the path segments into byte strings + + r_files = {'files': [], 'directories': [], 'symlinks': []} + + def _recurse(topdir, rel_offset, parent_dirs, rel_base=u'', checksum_check=False): + """ + This is a closure (function utilizing variables from it's parent + function's scope) so that we only need one copy of all the containers. + Note that this function uses side effects (See the Variables used from + outer scope). + + :arg topdir: The directory we are walking for files + :arg rel_offset: Integer defining how many characters to strip off of + the beginning of a path + :arg parent_dirs: Directories that we're copying that this directory is in. + :kwarg rel_base: String to prepend to the path after ``rel_offset`` is + applied to form the relative path. + + Variables used from the outer scope + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + :r_files: Dictionary of files in the hierarchy. See the return value + for :func:`walk` for the structure of this dictionary. + :local_follow: Read-only inside of :func:`_recurse`. Whether to follow symlinks + """ + for base_path, sub_folders, files in os.walk(topdir): + for filename in files: + filepath = os.path.join(base_path, filename) + dest_filepath = os.path.join(rel_base, filepath[rel_offset:]) + + if os.path.islink(filepath): + # Dereference the symlnk + real_file = loader.get_real_file(os.path.realpath(filepath), decrypt=decrypt) + if local_follow and os.path.isfile(real_file): + # Add the file pointed to by the symlink + r_files['files'].append( + { + "src": real_file, + "dest": dest_filepath, + "checksum": _get_local_checksum(checksum_check, real_file) + } + ) + else: + # Mark this file as a symlink to copy + r_files['symlinks'].append({"src": os.readlink(filepath), "dest": dest_filepath}) + else: + # Just a normal file + real_file = loader.get_real_file(filepath, decrypt=decrypt) + r_files['files'].append( + { + "src": real_file, + "dest": dest_filepath, + "checksum": _get_local_checksum(checksum_check, real_file) + } + ) + + for dirname in sub_folders: + dirpath = os.path.join(base_path, dirname) + dest_dirpath = os.path.join(rel_base, dirpath[rel_offset:]) + real_dir = os.path.realpath(dirpath) + dir_stats = os.stat(real_dir) + + if os.path.islink(dirpath): + if local_follow: + if (dir_stats.st_dev, dir_stats.st_ino) in parent_dirs: + # Just insert the symlink if the target directory + # exists inside of the copy already + r_files['symlinks'].append({"src": os.readlink(dirpath), "dest": dest_dirpath}) + else: + # Walk the dirpath to find all parent directories. + new_parents = set() + parent_dir_list = os.path.dirname(dirpath).split(os.path.sep) + for parent in range(len(parent_dir_list), 0, -1): + parent_stat = os.stat(u'/'.join(parent_dir_list[:parent])) + if (parent_stat.st_dev, parent_stat.st_ino) in parent_dirs: + # Reached the point at which the directory + # tree is already known. Don't add any + # more or we might go to an ancestor that + # isn't being copied. + break + new_parents.add((parent_stat.st_dev, parent_stat.st_ino)) + + if (dir_stats.st_dev, dir_stats.st_ino) in new_parents: + # This was a a circular symlink. So add it as + # a symlink + r_files['symlinks'].append({"src": os.readlink(dirpath), "dest": dest_dirpath}) + else: + # Walk the directory pointed to by the symlink + r_files['directories'].append({"src": real_dir, "dest": dest_dirpath}) + offset = len(real_dir) + 1 + _recurse(real_dir, offset, parent_dirs.union(new_parents), + rel_base=dest_dirpath, + checksum_check=checksum_check) + else: + # Add the symlink to the destination + r_files['symlinks'].append({"src": os.readlink(dirpath), "dest": dest_dirpath}) + else: + # Just a normal directory + r_files['directories'].append({"src": dirpath, "dest": dest_dirpath}) + + # Check if the source ends with a "/" so that we know which directory + # level to work at (similar to rsync) + source_trailing_slash = False + if trailing_slash_detector: + source_trailing_slash = trailing_slash_detector(topdir) + else: + source_trailing_slash = topdir.endswith(os.path.sep) + + # Calculate the offset needed to strip the base_path to make relative + # paths + if base_path is None: + base_path = topdir + if not source_trailing_slash: + base_path = os.path.dirname(base_path) + if topdir.startswith(base_path): + offset = len(base_path) + + # Make sure we're making the new paths relative + if trailing_slash_detector and not trailing_slash_detector(base_path): + offset += 1 + elif not base_path.endswith(os.path.sep): + offset += 1 + + if os.path.islink(topdir) and not local_follow: + r_files['symlinks'] = {"src": os.readlink(topdir), "dest": os.path.basename(topdir)} + return r_files + + dir_stats = os.stat(topdir) + parents = frozenset(((dir_stats.st_dev, dir_stats.st_ino),)) + # Actually walk the directory hierarchy + _recurse(topdir, offset, parents, checksum_check=checksum_check) + + return r_files + + +def _get_local_checksum(get_checksum, local_path): + if get_checksum: + return checksum(local_path) + else: + return None + + +class ActionModule(ActionBase): + + WIN_PATH_SEPARATOR = "\\" + + def _create_content_tempfile(self, content): + ''' Create a tempfile containing defined content ''' + fd, content_tempfile = tempfile.mkstemp(dir=C.DEFAULT_LOCAL_TMP) + f = os.fdopen(fd, 'wb') + content = to_bytes(content) + try: + f.write(content) + except Exception as err: + os.remove(content_tempfile) + raise Exception(err) + finally: + f.close() + return content_tempfile + + def _create_zip_tempfile(self, files, directories): + tmpdir = tempfile.mkdtemp(dir=C.DEFAULT_LOCAL_TMP) + zip_file_path = os.path.join(tmpdir, "win_copy.zip") + zip_file = zipfile.ZipFile(zip_file_path, "w", zipfile.ZIP_STORED, True) + + # encoding the file/dir name with base64 so Windows can unzip a unicode + # filename and get the right name, Windows doesn't handle unicode names + # very well + for directory in directories: + directory_path = to_bytes(directory['src'], errors='surrogate_or_strict') + archive_path = to_bytes(directory['dest'], errors='surrogate_or_strict') + + encoded_path = to_text(base64.b64encode(archive_path), errors='surrogate_or_strict') + zip_file.write(directory_path, encoded_path, zipfile.ZIP_DEFLATED) + + for file in files: + file_path = to_bytes(file['src'], errors='surrogate_or_strict') + archive_path = to_bytes(file['dest'], errors='surrogate_or_strict') + + encoded_path = to_text(base64.b64encode(archive_path), errors='surrogate_or_strict') + zip_file.write(file_path, encoded_path, zipfile.ZIP_DEFLATED) + + return zip_file_path + + def _remove_tempfile_if_content_defined(self, content, content_tempfile): + if content is not None: + os.remove(content_tempfile) + + def _copy_single_file(self, local_file, dest, source_rel, task_vars, tmp, backup): + if self._play_context.check_mode: + module_return = dict(changed=True) + return module_return + + # copy the file across to the server + tmp_src = self._connection._shell.join_path(tmp, 'source') + self._transfer_file(local_file, tmp_src) + + copy_args = self._task.args.copy() + copy_args.update( + dict( + dest=dest, + src=tmp_src, + _original_basename=source_rel, + _copy_mode="single", + backup=backup, + ) + ) + copy_args.pop('content', None) + + copy_result = self._execute_module(module_name="copy", + module_args=copy_args, + task_vars=task_vars) + + return copy_result + + def _copy_zip_file(self, dest, files, directories, task_vars, tmp, backup): + # create local zip file containing all the files and directories that + # need to be copied to the server + if self._play_context.check_mode: + module_return = dict(changed=True) + return module_return + + try: + zip_file = self._create_zip_tempfile(files, directories) + except Exception as e: + module_return = dict( + changed=False, + failed=True, + msg="failed to create tmp zip file: %s" % to_text(e), + exception=traceback.format_exc() + ) + return module_return + + zip_path = self._loader.get_real_file(zip_file) + + # send zip file to remote, file must end in .zip so + # Com Shell.Application works + tmp_src = self._connection._shell.join_path(tmp, 'source.zip') + self._transfer_file(zip_path, tmp_src) + + # run the explode operation of win_copy on remote + copy_args = self._task.args.copy() + copy_args.update( + dict( + src=tmp_src, + dest=dest, + _copy_mode="explode", + backup=backup, + ) + ) + copy_args.pop('content', None) + module_return = self._execute_module(module_name='copy', + module_args=copy_args, + task_vars=task_vars) + shutil.rmtree(os.path.dirname(zip_path)) + return module_return + + def run(self, tmp=None, task_vars=None): + ''' handler for file transfer operations ''' + if task_vars is None: + task_vars = dict() + + result = super(ActionModule, self).run(tmp, task_vars) + del tmp # tmp no longer has any effect + + source = self._task.args.get('src', None) + content = self._task.args.get('content', None) + dest = self._task.args.get('dest', None) + remote_src = boolean(self._task.args.get('remote_src', False), strict=False) + local_follow = boolean(self._task.args.get('local_follow', False), strict=False) + force = boolean(self._task.args.get('force', True), strict=False) + decrypt = boolean(self._task.args.get('decrypt', True), strict=False) + backup = boolean(self._task.args.get('backup', False), strict=False) + + result['src'] = source + result['dest'] = dest + + result['failed'] = True + if (source is None and content is None) or dest is None: + result['msg'] = "src (or content) and dest are required" + elif source is not None and content is not None: + result['msg'] = "src and content are mutually exclusive" + elif content is not None and dest is not None and ( + dest.endswith(os.path.sep) or dest.endswith(self.WIN_PATH_SEPARATOR)): + result['msg'] = "dest must be a file if content is defined" + else: + del result['failed'] + + if result.get('failed'): + return result + + # If content is defined make a temp file and write the content into it + content_tempfile = None + if content is not None: + try: + # if content comes to us as a dict it should be decoded json. + # We need to encode it back into a string and write it out + if isinstance(content, dict) or isinstance(content, list): + content_tempfile = self._create_content_tempfile(json.dumps(content)) + else: + content_tempfile = self._create_content_tempfile(content) + source = content_tempfile + except Exception as err: + result['failed'] = True + result['msg'] = "could not write content tmp file: %s" % to_native(err) + return result + # all actions should occur on the remote server, run win_copy module + elif remote_src: + new_module_args = self._task.args.copy() + new_module_args.update( + dict( + _copy_mode="remote", + dest=dest, + src=source, + force=force, + backup=backup, + ) + ) + new_module_args.pop('content', None) + result.update(self._execute_module(module_args=new_module_args, task_vars=task_vars)) + return result + # find_needle returns a path that may not have a trailing slash on a + # directory so we need to find that out first and append at the end + else: + trailing_slash = source.endswith(os.path.sep) + try: + # find in expected paths + source = self._find_needle('files', source) + except AnsibleError as e: + result['failed'] = True + result['msg'] = to_text(e) + result['exception'] = traceback.format_exc() + return result + + if trailing_slash != source.endswith(os.path.sep): + if source[-1] == os.path.sep: + source = source[:-1] + else: + source = source + os.path.sep + + # A list of source file tuples (full_path, relative_path) which will try to copy to the destination + source_files = {'files': [], 'directories': [], 'symlinks': []} + + # If source is a directory populate our list else source is a file and translate it to a tuple. + if os.path.isdir(to_bytes(source, errors='surrogate_or_strict')): + result['operation'] = 'folder_copy' + + # Get a list of the files we want to replicate on the remote side + source_files = _walk_dirs(source, self._loader, decrypt=decrypt, local_follow=local_follow, + trailing_slash_detector=self._connection._shell.path_has_trailing_slash, + checksum_check=force) + + # If it's recursive copy, destination is always a dir, + # explicitly mark it so (note - win_copy module relies on this). + if not self._connection._shell.path_has_trailing_slash(dest): + dest = "%s%s" % (dest, self.WIN_PATH_SEPARATOR) + + check_dest = dest + # Source is a file, add details to source_files dict + else: + result['operation'] = 'file_copy' + + # If the local file does not exist, get_real_file() raises AnsibleFileNotFound + try: + source_full = self._loader.get_real_file(source, decrypt=decrypt) + except AnsibleFileNotFound as e: + result['failed'] = True + result['msg'] = "could not find src=%s, %s" % (source_full, to_text(e)) + return result + + original_basename = os.path.basename(source) + result['original_basename'] = original_basename + + # check if dest ends with / or \ and append source filename to dest + if self._connection._shell.path_has_trailing_slash(dest): + check_dest = dest + filename = original_basename + result['dest'] = self._connection._shell.join_path(dest, filename) + else: + # replace \\ with / so we can use os.path to get the filename or dirname + unix_path = dest.replace(self.WIN_PATH_SEPARATOR, os.path.sep) + filename = os.path.basename(unix_path) + check_dest = os.path.dirname(unix_path) + + file_checksum = _get_local_checksum(force, source_full) + source_files['files'].append( + dict( + src=source_full, + dest=filename, + checksum=file_checksum + ) + ) + result['checksum'] = file_checksum + result['size'] = os.path.getsize(to_bytes(source_full, errors='surrogate_or_strict')) + + # find out the files/directories/symlinks that we need to copy to the server + query_args = self._task.args.copy() + query_args.update( + dict( + _copy_mode="query", + dest=check_dest, + force=force, + files=source_files['files'], + directories=source_files['directories'], + symlinks=source_files['symlinks'], + ) + ) + # src is not required for query, will fail path validation is src has unix allowed chars + query_args.pop('src', None) + + query_args.pop('content', None) + query_return = self._execute_module(module_args=query_args, + task_vars=task_vars) + + if query_return.get('failed') is True: + result.update(query_return) + return result + + if len(query_return['files']) > 0 or len(query_return['directories']) > 0 and self._connection._shell.tmpdir is None: + self._connection._shell.tmpdir = self._make_tmp_path() + + if len(query_return['files']) == 1 and len(query_return['directories']) == 0: + # we only need to copy 1 file, don't mess around with zips + file_src = query_return['files'][0]['src'] + file_dest = query_return['files'][0]['dest'] + result.update(self._copy_single_file(file_src, dest, file_dest, + task_vars, self._connection._shell.tmpdir, backup)) + if result.get('failed') is True: + result['msg'] = "failed to copy file %s: %s" % (file_src, result['msg']) + result['changed'] = True + + elif len(query_return['files']) > 0 or len(query_return['directories']) > 0: + # either multiple files or directories need to be copied, compress + # to a zip and 'explode' the zip on the server + # TODO: handle symlinks + result.update(self._copy_zip_file(dest, source_files['files'], + source_files['directories'], + task_vars, self._connection._shell.tmpdir, backup)) + result['changed'] = True + else: + # no operations need to occur + result['failed'] = False + result['changed'] = False + + # remove the content tmp file and remote tmp file if it was created + self._remove_tempfile_if_content_defined(content, content_tempfile) + self._remove_tmp_path(self._connection._shell.tmpdir) + return result |