diff options
Diffstat (limited to 'test/integration/targets/old_style_cache_plugins')
11 files changed, 596 insertions, 0 deletions
diff --git a/test/integration/targets/old_style_cache_plugins/aliases b/test/integration/targets/old_style_cache_plugins/aliases new file mode 100644 index 0000000..3777383 --- /dev/null +++ b/test/integration/targets/old_style_cache_plugins/aliases @@ -0,0 +1,6 @@ +destructive +needs/root +shippable/posix/group5 +context/controller +skip/osx +skip/macos diff --git a/test/integration/targets/old_style_cache_plugins/cleanup.yml b/test/integration/targets/old_style_cache_plugins/cleanup.yml new file mode 100644 index 0000000..93f5cc5 --- /dev/null +++ b/test/integration/targets/old_style_cache_plugins/cleanup.yml @@ -0,0 +1,41 @@ +--- +- hosts: localhost + gather_facts: no + ignore_errors: yes + tasks: + - command: redis-cli keys + + - name: delete cache keys + command: redis-cli del {{ item }} + loop: + - ansible_facts_localhost + - ansible_inventory_localhost + - ansible_cache_keys + + - name: shutdown the server + command: redis-cli shutdown + + - name: cleanup set up files + file: + path: "{{ item }}" + state: absent + loop: + - redis-stable.tar.gz + + - name: remove executables + file: + state: absent + path: "/usr/local/bin/{{ item }}" + follow: no + become: yes + loop: + - redis-server + - redis-cli + + - name: clean the rest of the files + file: + path: "{{ item }}" + state: absent + loop: + - ./redis-stable.tar.gz + - ./redis-stable diff --git a/test/integration/targets/old_style_cache_plugins/inspect_cache.yml b/test/integration/targets/old_style_cache_plugins/inspect_cache.yml new file mode 100644 index 0000000..72810e1 --- /dev/null +++ b/test/integration/targets/old_style_cache_plugins/inspect_cache.yml @@ -0,0 +1,36 @@ +--- +- hosts: localhost + gather_facts: no + vars: + json_cache: "{{ cache.stdout | from_json }}" + tasks: + - command: redis-cli get ansible_facts_localhost + register: cache + tags: + - always + + - name: test that the cache only contains the set_fact var + assert: + that: + - "json_cache | length == 1" + - "json_cache.foo == ansible_facts.foo" + tags: + - set_fact + + - name: test that the cache contains gathered facts and the var + assert: + that: + - "json_cache | length > 1" + - "json_cache.foo == 'bar'" + - "json_cache.ansible_distribution is defined" + tags: + - additive_gather_facts + + - name: test that the cache contains only gathered facts + assert: + that: + - "json_cache | length > 1" + - "json_cache.foo is undefined" + - "json_cache.ansible_distribution is defined" + tags: + - gather_facts diff --git a/test/integration/targets/old_style_cache_plugins/inventory_config b/test/integration/targets/old_style_cache_plugins/inventory_config new file mode 100644 index 0000000..d87c2a9 --- /dev/null +++ b/test/integration/targets/old_style_cache_plugins/inventory_config @@ -0,0 +1 @@ +# inventory config file for consistent source diff --git a/test/integration/targets/old_style_cache_plugins/plugins/cache/configurable_redis.py b/test/integration/targets/old_style_cache_plugins/plugins/cache/configurable_redis.py new file mode 100644 index 0000000..44b6cf9 --- /dev/null +++ b/test/integration/targets/old_style_cache_plugins/plugins/cache/configurable_redis.py @@ -0,0 +1,147 @@ +# (c) 2014, Brian Coca, Josh Drake, et al +# (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' + cache: configurable_redis + short_description: Use Redis DB for cache + description: + - This cache uses JSON formatted, per host records saved in Redis. + version_added: "1.9" + requirements: + - redis>=2.4.5 (python lib) + options: + _uri: + description: + - A colon separated string of connection information for Redis. + required: True + env: + - name: ANSIBLE_CACHE_PLUGIN_CONNECTION + ini: + - key: fact_caching_connection + section: defaults + _prefix: + description: User defined prefix to use when creating the DB entries + default: ansible_facts + env: + - name: ANSIBLE_CACHE_PLUGIN_PREFIX + ini: + - key: fact_caching_prefix + section: defaults + _timeout: + default: 86400 + description: Expiration timeout for the cache plugin data + env: + - name: ANSIBLE_CACHE_PLUGIN_TIMEOUT + ini: + - key: fact_caching_timeout + section: defaults + type: integer +''' + +import time +import json + +from ansible import constants as C +from ansible.errors import AnsibleError +from ansible.parsing.ajson import AnsibleJSONEncoder, AnsibleJSONDecoder +from ansible.plugins.cache import BaseCacheModule +from ansible.utils.display import Display + +try: + from redis import StrictRedis, VERSION +except ImportError: + raise AnsibleError("The 'redis' python module (version 2.4.5 or newer) is required for the redis fact cache, 'pip install redis'") + +display = Display() + + +class CacheModule(BaseCacheModule): + """ + A caching module backed by redis. + Keys are maintained in a zset with their score being the timestamp + when they are inserted. This allows for the usage of 'zremrangebyscore' + to expire keys. This mechanism is used or a pattern matched 'scan' for + performance. + """ + def __init__(self, *args, **kwargs): + connection = [] + + super(CacheModule, self).__init__(*args, **kwargs) + if self.get_option('_uri'): + connection = self.get_option('_uri').split(':') + self._timeout = float(self.get_option('_timeout')) + self._prefix = self.get_option('_prefix') + + self._cache = {} + self._db = StrictRedis(*connection) + self._keys_set = 'ansible_cache_keys' + + def _make_key(self, key): + return self._prefix + key + + def get(self, key): + + if key not in self._cache: + value = self._db.get(self._make_key(key)) + # guard against the key not being removed from the zset; + # this could happen in cases where the timeout value is changed + # between invocations + if value is None: + self.delete(key) + raise KeyError + self._cache[key] = json.loads(value, cls=AnsibleJSONDecoder) + + return self._cache.get(key) + + def set(self, key, value): + + value2 = json.dumps(value, cls=AnsibleJSONEncoder, sort_keys=True, indent=4) + if self._timeout > 0: # a timeout of 0 is handled as meaning 'never expire' + self._db.setex(self._make_key(key), int(self._timeout), value2) + else: + self._db.set(self._make_key(key), value2) + + if VERSION[0] == 2: + self._db.zadd(self._keys_set, time.time(), key) + else: + self._db.zadd(self._keys_set, {key: time.time()}) + self._cache[key] = value + + def _expire_keys(self): + if self._timeout > 0: + expiry_age = time.time() - self._timeout + self._db.zremrangebyscore(self._keys_set, 0, expiry_age) + + def keys(self): + self._expire_keys() + return self._db.zrange(self._keys_set, 0, -1) + + def contains(self, key): + self._expire_keys() + return (self._db.zrank(self._keys_set, key) is not None) + + def delete(self, key): + if key in self._cache: + del self._cache[key] + self._db.delete(self._make_key(key)) + self._db.zrem(self._keys_set, key) + + def flush(self): + for key in self.keys(): + self.delete(key) + + def copy(self): + # TODO: there is probably a better way to do this in redis + ret = dict() + for key in self.keys(): + ret[key] = self.get(key) + return ret + + def __getstate__(self): + return dict() + + def __setstate__(self, data): + self.__init__() diff --git a/test/integration/targets/old_style_cache_plugins/plugins/cache/legacy_redis.py b/test/integration/targets/old_style_cache_plugins/plugins/cache/legacy_redis.py new file mode 100644 index 0000000..9879dec --- /dev/null +++ b/test/integration/targets/old_style_cache_plugins/plugins/cache/legacy_redis.py @@ -0,0 +1,141 @@ +# (c) 2014, Brian Coca, Josh Drake, et al +# (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' + cache: redis + short_description: Use Redis DB for cache + description: + - This cache uses JSON formatted, per host records saved in Redis. + version_added: "1.9" + requirements: + - redis>=2.4.5 (python lib) + options: + _uri: + description: + - A colon separated string of connection information for Redis. + required: True + env: + - name: ANSIBLE_CACHE_PLUGIN_CONNECTION + ini: + - key: fact_caching_connection + section: defaults + _prefix: + description: User defined prefix to use when creating the DB entries + env: + - name: ANSIBLE_CACHE_PLUGIN_PREFIX + ini: + - key: fact_caching_prefix + section: defaults + _timeout: + default: 86400 + description: Expiration timeout for the cache plugin data + env: + - name: ANSIBLE_CACHE_PLUGIN_TIMEOUT + ini: + - key: fact_caching_timeout + section: defaults + type: integer +''' + +import time +import json + +from ansible import constants as C +from ansible.errors import AnsibleError +from ansible.plugins.cache import BaseCacheModule + +try: + from redis import StrictRedis, VERSION +except ImportError: + raise AnsibleError("The 'redis' python module (version 2.4.5 or newer) is required for the redis fact cache, 'pip install redis'") + + +class CacheModule(BaseCacheModule): + """ + A caching module backed by redis. + Keys are maintained in a zset with their score being the timestamp + when they are inserted. This allows for the usage of 'zremrangebyscore' + to expire keys. This mechanism is used or a pattern matched 'scan' for + performance. + """ + def __init__(self, *args, **kwargs): + if C.CACHE_PLUGIN_CONNECTION: + connection = C.CACHE_PLUGIN_CONNECTION.split(':') + else: + connection = [] + + self._timeout = float(C.CACHE_PLUGIN_TIMEOUT) + self._prefix = C.CACHE_PLUGIN_PREFIX + self._cache = {} + self._db = StrictRedis(*connection) + self._keys_set = 'ansible_cache_keys' + + def _make_key(self, key): + return self._prefix + key + + def get(self, key): + + if key not in self._cache: + value = self._db.get(self._make_key(key)) + # guard against the key not being removed from the zset; + # this could happen in cases where the timeout value is changed + # between invocations + if value is None: + self.delete(key) + raise KeyError + self._cache[key] = json.loads(value) + + return self._cache.get(key) + + def set(self, key, value): + + value2 = json.dumps(value) + if self._timeout > 0: # a timeout of 0 is handled as meaning 'never expire' + self._db.setex(self._make_key(key), int(self._timeout), value2) + else: + self._db.set(self._make_key(key), value2) + + if VERSION[0] == 2: + self._db.zadd(self._keys_set, time.time(), key) + else: + self._db.zadd(self._keys_set, {key: time.time()}) + self._cache[key] = value + + def _expire_keys(self): + if self._timeout > 0: + expiry_age = time.time() - self._timeout + self._db.zremrangebyscore(self._keys_set, 0, expiry_age) + + def keys(self): + self._expire_keys() + return self._db.zrange(self._keys_set, 0, -1) + + def contains(self, key): + self._expire_keys() + return (self._db.zrank(self._keys_set, key) is not None) + + def delete(self, key): + if key in self._cache: + del self._cache[key] + self._db.delete(self._make_key(key)) + self._db.zrem(self._keys_set, key) + + def flush(self): + for key in self.keys(): + self.delete(key) + + def copy(self): + # TODO: there is probably a better way to do this in redis + ret = dict() + for key in self.keys(): + ret[key] = self.get(key) + return ret + + def __getstate__(self): + return dict() + + def __setstate__(self, data): + self.__init__() diff --git a/test/integration/targets/old_style_cache_plugins/plugins/inventory/test.py b/test/integration/targets/old_style_cache_plugins/plugins/inventory/test.py new file mode 100644 index 0000000..7e59195 --- /dev/null +++ b/test/integration/targets/old_style_cache_plugins/plugins/inventory/test.py @@ -0,0 +1,59 @@ +# Copyright (c) 2019 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' + name: test + plugin_type: inventory + short_description: test inventory source + extends_documentation_fragment: + - inventory_cache +''' + +from ansible.plugins.inventory import BaseInventoryPlugin, Cacheable + + +class InventoryModule(BaseInventoryPlugin, Cacheable): + + NAME = 'test' + + def populate(self, hosts): + for host in list(hosts.keys()): + self.inventory.add_host(host, group='all') + for hostvar, hostval in hosts[host].items(): + self.inventory.set_variable(host, hostvar, hostval) + + def get_hosts(self): + return {'host1': {'one': 'two'}, 'host2': {'three': 'four'}} + + def parse(self, inventory, loader, path, cache=True): + super(InventoryModule, self).parse(inventory, loader, path) + + self.load_cache_plugin() + + cache_key = self.get_cache_key(path) + + # cache may be True or False at this point to indicate if the inventory is being refreshed + # get the user's cache option + cache_setting = self.get_option('cache') + + attempt_to_read_cache = cache_setting and cache + cache_needs_update = cache_setting and not cache + + # attempt to read the cache if inventory isn't being refreshed and the user has caching enabled + if attempt_to_read_cache: + try: + results = self._cache[cache_key] + except KeyError: + # This occurs if the cache_key is not in the cache or if the cache_key expired, so the cache needs to be updated + cache_needs_update = True + + if cache_needs_update: + results = self.get_hosts() + + # set the cache + self._cache[cache_key] = results + + self.populate(results) diff --git a/test/integration/targets/old_style_cache_plugins/runme.sh b/test/integration/targets/old_style_cache_plugins/runme.sh new file mode 100755 index 0000000..ffa6723 --- /dev/null +++ b/test/integration/targets/old_style_cache_plugins/runme.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +set -eux + +source virtualenv.sh + +trap 'ansible-playbook cleanup.yml' EXIT + +export PATH="$PATH:/usr/local/bin" + +ansible-playbook setup_redis_cache.yml "$@" + +# Cache should start empty +redis-cli keys ansible_ +[ "$(redis-cli keys ansible_)" = "" ] + +export ANSIBLE_CACHE_PLUGINS=./plugins/cache +export ANSIBLE_CACHE_PLUGIN_CONNECTION=localhost:6379:0 +export ANSIBLE_CACHE_PLUGIN_PREFIX='ansible_facts_' + +# Test legacy cache plugins (that use ansible.constants) and +# new cache plugins that use config manager both work for facts. +for fact_cache in legacy_redis configurable_redis; do + + export ANSIBLE_CACHE_PLUGIN="$fact_cache" + + # test set_fact with cacheable: true + ansible-playbook test_fact_gathering.yml --tags set_fact "$@" + [ "$(redis-cli keys ansible_facts_localhost | wc -l)" -eq 1 ] + ansible-playbook inspect_cache.yml --tags set_fact "$@" + + # cache gathered facts in addition + ansible-playbook test_fact_gathering.yml --tags gather_facts "$@" + ansible-playbook inspect_cache.yml --tags additive_gather_facts "$@" + + # flush cache and only cache gathered facts + ansible-playbook test_fact_gathering.yml --flush-cache --tags gather_facts --tags flush "$@" + ansible-playbook inspect_cache.yml --tags gather_facts "$@" + + redis-cli del ansible_facts_localhost + unset ANSIBLE_CACHE_PLUGIN + +done + +# Legacy cache plugins need to be updated to use set_options/get_option to be compatible with inventory plugins. +# Inventory plugins load cache options with the config manager. +ansible-playbook test_inventory_cache.yml "$@" diff --git a/test/integration/targets/old_style_cache_plugins/setup_redis_cache.yml b/test/integration/targets/old_style_cache_plugins/setup_redis_cache.yml new file mode 100644 index 0000000..8aad37a --- /dev/null +++ b/test/integration/targets/old_style_cache_plugins/setup_redis_cache.yml @@ -0,0 +1,51 @@ +--- +- hosts: localhost + vars: + make: "{{ ( ansible_distribution != 'FreeBSD' ) | ternary('make', 'gmake') }}" + tasks: + - name: name ensure make is available + command: "which {{ make }}" + register: has_make + ignore_errors: yes + + - command: apk add --no-cache make + when: "has_make is failed and ansible_distribution == 'Alpine'" + become: yes + + - package: + name: "{{ make }}" + state: present + become: yes + when: "has_make is failed and ansible_distribution != 'Alpine'" + + - name: get the latest stable redis server release + get_url: + url: http://download.redis.io/redis-stable.tar.gz + dest: ./ + + - name: unzip download + unarchive: + src: redis-stable.tar.gz + dest: ./ + + - command: "{{ make }}" + args: + chdir: redis-stable + + - name: copy the executable into the path + copy: + src: "redis-stable/src/{{ item }}" + dest: /usr/local/bin/ + mode: 755 + become: yes + loop: + - redis-server + - redis-cli + + - name: start the redis server in the background + command: redis-server --daemonize yes + + - name: install dependency for the cache plugin + pip: + name: redis>2.4.5 + state: present diff --git a/test/integration/targets/old_style_cache_plugins/test_fact_gathering.yml b/test/integration/targets/old_style_cache_plugins/test_fact_gathering.yml new file mode 100644 index 0000000..2c77f0d --- /dev/null +++ b/test/integration/targets/old_style_cache_plugins/test_fact_gathering.yml @@ -0,0 +1,22 @@ +--- +- hosts: localhost + gather_facts: no + tags: + - flush + tasks: + - meta: clear_facts + +- hosts: localhost + gather_facts: yes + gather_subset: min + tags: + - gather_facts + +- hosts: localhost + gather_facts: no + tags: + - set_fact + tasks: + - set_fact: + foo: bar + cacheable: true diff --git a/test/integration/targets/old_style_cache_plugins/test_inventory_cache.yml b/test/integration/targets/old_style_cache_plugins/test_inventory_cache.yml new file mode 100644 index 0000000..83b7983 --- /dev/null +++ b/test/integration/targets/old_style_cache_plugins/test_inventory_cache.yml @@ -0,0 +1,45 @@ +- hosts: localhost + gather_facts: no + vars: + reset_color: '\x1b\[0m' + color: '\x1b\[[0-9];[0-9]{2}m' + base_environment: + ANSIBLE_INVENTORY_PLUGINS: ./plugins/inventory + ANSIBLE_INVENTORY_ENABLED: test + ANSIBLE_INVENTORY_CACHE: true + ANSIBLE_CACHE_PLUGINS: ./plugins/cache + ANSIBLE_CACHE_PLUGIN_CONNECTION: localhost:6379:0 + ANSIBLE_CACHE_PLUGIN_PREFIX: 'ansible_inventory_' + legacy_cache: + ANSIBLE_INVENTORY_CACHE_PLUGIN: legacy_redis + updated_cache: + ANSIBLE_INVENTORY_CACHE_PLUGIN: configurable_redis + tasks: + - name: legacy-style cache plugin should cause a warning + command: ansible-inventory -i inventory_config --graph + register: result + environment: "{{ base_environment | combine(legacy_cache) }}" + + - name: test warning message + assert: + that: + - expected_warning in warning + - "'No inventory was parsed, only implicit localhost is available' in warning" + vars: + warning: "{{ result.stderr | regex_replace(reset_color) | regex_replace(color) | regex_replace('\\n', ' ') }}" + expected_warning: "Cache options were provided but may not reconcile correctly unless set via set_options" + + - name: cache plugin updated to use config manager should work + command: ansible-inventory -i inventory_config --graph + register: result + environment: "{{ base_environment | combine(updated_cache) }}" + + - name: test warning message + assert: + that: + - unexpected_warning not in warning + - "'No inventory was parsed, only implicit localhost is available' not in warning" + - '"host1" in result.stdout' + vars: + warning: "{{ result.stderr | regex_replace(reset_color) | regex_replace(color) | regex_replace('\\n', ' ') }}" + unexpected_warning: "Cache options were provided but may not reconcile correctly unless set via set_options" |