diff options
Diffstat (limited to '')
| -rw-r--r-- | lib/ansible/inventory/__init__.py | 0 | ||||
| -rw-r--r-- | lib/ansible/inventory/data.py | 283 | ||||
| -rw-r--r-- | lib/ansible/inventory/group.py | 288 | ||||
| -rw-r--r-- | lib/ansible/inventory/helpers.py | 40 | ||||
| -rw-r--r-- | lib/ansible/inventory/host.py | 169 | ||||
| -rw-r--r-- | lib/ansible/inventory/manager.py | 752 |
6 files changed, 1532 insertions, 0 deletions
diff --git a/lib/ansible/inventory/__init__.py b/lib/ansible/inventory/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/lib/ansible/inventory/__init__.py diff --git a/lib/ansible/inventory/data.py b/lib/ansible/inventory/data.py new file mode 100644 index 0000000..15a6420 --- /dev/null +++ b/lib/ansible/inventory/data.py @@ -0,0 +1,283 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +############################################# +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import sys + +from ansible import constants as C +from ansible.errors import AnsibleError +from ansible.inventory.group import Group +from ansible.inventory.host import Host +from ansible.module_utils.six import string_types +from ansible.utils.display import Display +from ansible.utils.vars import combine_vars +from ansible.utils.path import basedir + +display = Display() + + +class InventoryData(object): + """ + Holds inventory data (host and group objects). + Using it's methods should guarantee expected relationships and data. + """ + + def __init__(self): + + self.groups = {} + self.hosts = {} + + # provides 'groups' magic var, host object has group_names + self._groups_dict_cache = {} + + # current localhost, implicit or explicit + self.localhost = None + + self.current_source = None + self.processed_sources = [] + + # Always create the 'all' and 'ungrouped' groups, + for group in ('all', 'ungrouped'): + self.add_group(group) + self.add_child('all', 'ungrouped') + + def serialize(self): + self._groups_dict_cache = None + data = { + 'groups': self.groups, + 'hosts': self.hosts, + 'local': self.localhost, + 'source': self.current_source, + 'processed_sources': self.processed_sources + } + return data + + def deserialize(self, data): + self._groups_dict_cache = {} + self.hosts = data.get('hosts') + self.groups = data.get('groups') + self.localhost = data.get('local') + self.current_source = data.get('source') + self.processed_sources = data.get('processed_sources') + + def _create_implicit_localhost(self, pattern): + + if self.localhost: + new_host = self.localhost + else: + new_host = Host(pattern) + + new_host.address = "127.0.0.1" + new_host.implicit = True + + # set localhost defaults + py_interp = sys.executable + if not py_interp: + # sys.executable is not set in some cornercases. see issue #13585 + py_interp = '/usr/bin/python' + display.warning('Unable to determine python interpreter from sys.executable. Using /usr/bin/python default. ' + 'You can correct this by setting ansible_python_interpreter for localhost') + new_host.set_variable("ansible_python_interpreter", py_interp) + new_host.set_variable("ansible_connection", 'local') + + self.localhost = new_host + + return new_host + + def reconcile_inventory(self): + ''' Ensure inventory basic rules, run after updates ''' + + display.debug('Reconcile groups and hosts in inventory.') + self.current_source = None + + group_names = set() + # set group vars from group_vars/ files and vars plugins + for g in self.groups: + group = self.groups[g] + group_names.add(group.name) + + # ensure all groups inherit from 'all' + if group.name != 'all' and not group.get_ancestors(): + self.add_child('all', group.name) + + host_names = set() + # get host vars from host_vars/ files and vars plugins + for host in self.hosts.values(): + host_names.add(host.name) + + mygroups = host.get_groups() + + if self.groups['ungrouped'] in mygroups: + # clear ungrouped of any incorrectly stored by parser + if set(mygroups).difference(set([self.groups['all'], self.groups['ungrouped']])): + self.groups['ungrouped'].remove_host(host) + + elif not host.implicit: + # add ungrouped hosts to ungrouped, except implicit + length = len(mygroups) + if length == 0 or (length == 1 and self.groups['all'] in mygroups): + self.add_child('ungrouped', host.name) + + # special case for implicit hosts + if host.implicit: + host.vars = combine_vars(self.groups['all'].get_vars(), host.vars) + + # warn if overloading identifier as both group and host + for conflict in group_names.intersection(host_names): + display.warning("Found both group and host with same name: %s" % conflict) + + self._groups_dict_cache = {} + + def get_host(self, hostname): + ''' fetch host object using name deal with implicit localhost ''' + + matching_host = self.hosts.get(hostname, None) + + # if host is not in hosts dict + if matching_host is None and hostname in C.LOCALHOST: + # might need to create implicit localhost + matching_host = self._create_implicit_localhost(hostname) + + return matching_host + + def add_group(self, group): + ''' adds a group to inventory if not there already, returns named actually used ''' + + if group: + if not isinstance(group, string_types): + raise AnsibleError("Invalid group name supplied, expected a string but got %s for %s" % (type(group), group)) + if group not in self.groups: + g = Group(group) + if g.name not in self.groups: + self.groups[g.name] = g + self._groups_dict_cache = {} + display.debug("Added group %s to inventory" % group) + group = g.name + else: + display.debug("group %s already in inventory" % group) + else: + raise AnsibleError("Invalid empty/false group name provided: %s" % group) + + return group + + def remove_group(self, group): + + if group in self.groups: + del self.groups[group] + display.debug("Removed group %s from inventory" % group) + self._groups_dict_cache = {} + + for host in self.hosts: + h = self.hosts[host] + h.remove_group(group) + + def add_host(self, host, group=None, port=None): + ''' adds a host to inventory and possibly a group if not there already ''' + + if host: + if not isinstance(host, string_types): + raise AnsibleError("Invalid host name supplied, expected a string but got %s for %s" % (type(host), host)) + + # TODO: add to_safe_host_name + g = None + if group: + if group in self.groups: + g = self.groups[group] + else: + raise AnsibleError("Could not find group %s in inventory" % group) + + if host not in self.hosts: + h = Host(host, port) + self.hosts[host] = h + if self.current_source: # set to 'first source' in which host was encountered + self.set_variable(host, 'inventory_file', self.current_source) + self.set_variable(host, 'inventory_dir', basedir(self.current_source)) + else: + self.set_variable(host, 'inventory_file', None) + self.set_variable(host, 'inventory_dir', None) + display.debug("Added host %s to inventory" % (host)) + + # set default localhost from inventory to avoid creating an implicit one. Last localhost defined 'wins'. + if host in C.LOCALHOST: + if self.localhost is None: + self.localhost = self.hosts[host] + display.vvvv("Set default localhost to %s" % h) + else: + display.warning("A duplicate localhost-like entry was found (%s). First found localhost was %s" % (h, self.localhost.name)) + else: + h = self.hosts[host] + + if g: + g.add_host(h) + self._groups_dict_cache = {} + display.debug("Added host %s to group %s" % (host, group)) + else: + raise AnsibleError("Invalid empty host name provided: %s" % host) + + return host + + def remove_host(self, host): + + if host.name in self.hosts: + del self.hosts[host.name] + + for group in self.groups: + g = self.groups[group] + g.remove_host(host) + + def set_variable(self, entity, varname, value): + ''' sets a variable for an inventory object ''' + + if entity in self.groups: + inv_object = self.groups[entity] + elif entity in self.hosts: + inv_object = self.hosts[entity] + else: + raise AnsibleError("Could not identify group or host named %s" % entity) + + inv_object.set_variable(varname, value) + display.debug('set %s for %s' % (varname, entity)) + + def add_child(self, group, child): + ''' Add host or group to group ''' + added = False + if group in self.groups: + g = self.groups[group] + if child in self.groups: + added = g.add_child_group(self.groups[child]) + elif child in self.hosts: + added = g.add_host(self.hosts[child]) + else: + raise AnsibleError("%s is not a known host nor group" % child) + self._groups_dict_cache = {} + display.debug('Group %s now contains %s' % (group, child)) + else: + raise AnsibleError("%s is not a known group" % group) + return added + + def get_groups_dict(self): + """ + We merge a 'magic' var 'groups' with group name keys and hostname list values into every host variable set. Cache for speed. + """ + if not self._groups_dict_cache: + for (group_name, group) in self.groups.items(): + self._groups_dict_cache[group_name] = [h.name for h in group.get_hosts()] + + return self._groups_dict_cache diff --git a/lib/ansible/inventory/group.py b/lib/ansible/inventory/group.py new file mode 100644 index 0000000..c7af685 --- /dev/null +++ b/lib/ansible/inventory/group.py @@ -0,0 +1,288 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from collections.abc import Mapping, MutableMapping +from itertools import chain + +from ansible import constants as C +from ansible.errors import AnsibleError +from ansible.module_utils._text import to_native, to_text +from ansible.utils.display import Display +from ansible.utils.vars import combine_vars + +display = Display() + + +def to_safe_group_name(name, replacer="_", force=False, silent=False): + # Converts 'bad' characters in a string to underscores (or provided replacer) so they can be used as Ansible hosts or groups + + warn = '' + if name: # when deserializing we might not have name yet + invalid_chars = C.INVALID_VARIABLE_NAMES.findall(name) + if invalid_chars: + msg = 'invalid character(s) "%s" in group name (%s)' % (to_text(set(invalid_chars)), to_text(name)) + if C.TRANSFORM_INVALID_GROUP_CHARS not in ('never', 'ignore') or force: + name = C.INVALID_VARIABLE_NAMES.sub(replacer, name) + if not (silent or C.TRANSFORM_INVALID_GROUP_CHARS == 'silently'): + display.vvvv('Replacing ' + msg) + warn = 'Invalid characters were found in group names and automatically replaced, use -vvvv to see details' + else: + if C.TRANSFORM_INVALID_GROUP_CHARS == 'never': + display.vvvv('Not replacing %s' % msg) + warn = 'Invalid characters were found in group names but not replaced, use -vvvv to see details' + + if warn: + display.warning(warn) + + return name + + +class Group: + ''' a group of ansible hosts ''' + + # __slots__ = [ 'name', 'hosts', 'vars', 'child_groups', 'parent_groups', 'depth', '_hosts_cache' ] + + def __init__(self, name=None): + + self.depth = 0 + self.name = to_safe_group_name(name) + self.hosts = [] + self._hosts = None + self.vars = {} + self.child_groups = [] + self.parent_groups = [] + self._hosts_cache = None + self.priority = 1 + + def __repr__(self): + return self.get_name() + + def __str__(self): + return self.get_name() + + def __getstate__(self): + return self.serialize() + + def __setstate__(self, data): + return self.deserialize(data) + + def serialize(self): + parent_groups = [] + for parent in self.parent_groups: + parent_groups.append(parent.serialize()) + + self._hosts = None + + result = dict( + name=self.name, + vars=self.vars.copy(), + parent_groups=parent_groups, + depth=self.depth, + hosts=self.hosts, + ) + + return result + + def deserialize(self, data): + self.__init__() # used by __setstate__ to deserialize in place # pylint: disable=unnecessary-dunder-call + self.name = data.get('name') + self.vars = data.get('vars', dict()) + self.depth = data.get('depth', 0) + self.hosts = data.get('hosts', []) + self._hosts = None + + parent_groups = data.get('parent_groups', []) + for parent_data in parent_groups: + g = Group() + g.deserialize(parent_data) + self.parent_groups.append(g) + + def _walk_relationship(self, rel, include_self=False, preserve_ordering=False): + ''' + Given `rel` that is an iterable property of Group, + consitituting a directed acyclic graph among all groups, + Returns a set of all groups in full tree + A B C + | / | / + | / | / + D -> E + | / vertical connections + | / are directed upward + F + Called on F, returns set of (A, B, C, D, E) + ''' + seen = set([]) + unprocessed = set(getattr(self, rel)) + if include_self: + unprocessed.add(self) + if preserve_ordering: + ordered = [self] if include_self else [] + ordered.extend(getattr(self, rel)) + + while unprocessed: + seen.update(unprocessed) + new_unprocessed = set([]) + + for new_item in chain.from_iterable(getattr(g, rel) for g in unprocessed): + new_unprocessed.add(new_item) + if preserve_ordering: + if new_item not in seen: + ordered.append(new_item) + + new_unprocessed.difference_update(seen) + unprocessed = new_unprocessed + + if preserve_ordering: + return ordered + return seen + + def get_ancestors(self): + return self._walk_relationship('parent_groups') + + def get_descendants(self, **kwargs): + return self._walk_relationship('child_groups', **kwargs) + + @property + def host_names(self): + if self._hosts is None: + self._hosts = set(self.hosts) + return self._hosts + + def get_name(self): + return self.name + + def add_child_group(self, group): + added = False + if self == group: + raise Exception("can't add group to itself") + + # don't add if it's already there + if group not in self.child_groups: + + # prepare list of group's new ancestors this edge creates + start_ancestors = group.get_ancestors() + new_ancestors = self.get_ancestors() + if group in new_ancestors: + raise AnsibleError("Adding group '%s' as child to '%s' creates a recursive dependency loop." % (to_native(group.name), to_native(self.name))) + new_ancestors.add(self) + new_ancestors.difference_update(start_ancestors) + + added = True + self.child_groups.append(group) + + # update the depth of the child + group.depth = max([self.depth + 1, group.depth]) + + # update the depth of the grandchildren + group._check_children_depth() + + # now add self to child's parent_groups list, but only if there + # isn't already a group with the same name + if self.name not in [g.name for g in group.parent_groups]: + group.parent_groups.append(self) + for h in group.get_hosts(): + h.populate_ancestors(additions=new_ancestors) + + self.clear_hosts_cache() + return added + + def _check_children_depth(self): + + depth = self.depth + start_depth = self.depth # self.depth could change over loop + seen = set([]) + unprocessed = set(self.child_groups) + + while unprocessed: + seen.update(unprocessed) + depth += 1 + to_process = unprocessed.copy() + unprocessed = set([]) + for g in to_process: + if g.depth < depth: + g.depth = depth + unprocessed.update(g.child_groups) + if depth - start_depth > len(seen): + raise AnsibleError("The group named '%s' has a recursive dependency loop." % to_native(self.name)) + + def add_host(self, host): + added = False + if host.name not in self.host_names: + self.hosts.append(host) + self._hosts.add(host.name) + host.add_group(self) + self.clear_hosts_cache() + added = True + return added + + def remove_host(self, host): + removed = False + if host.name in self.host_names: + self.hosts.remove(host) + self._hosts.remove(host.name) + host.remove_group(self) + self.clear_hosts_cache() + removed = True + return removed + + def set_variable(self, key, value): + + if key == 'ansible_group_priority': + self.set_priority(int(value)) + else: + if key in self.vars and isinstance(self.vars[key], MutableMapping) and isinstance(value, Mapping): + self.vars = combine_vars(self.vars, {key: value}) + else: + self.vars[key] = value + + def clear_hosts_cache(self): + + self._hosts_cache = None + for g in self.get_ancestors(): + g._hosts_cache = None + + def get_hosts(self): + + if self._hosts_cache is None: + self._hosts_cache = self._get_hosts() + return self._hosts_cache + + def _get_hosts(self): + + hosts = [] + seen = {} + for kid in self.get_descendants(include_self=True, preserve_ordering=True): + kid_hosts = kid.hosts + for kk in kid_hosts: + if kk not in seen: + seen[kk] = 1 + if self.name == 'all' and kk.implicit: + continue + hosts.append(kk) + return hosts + + def get_vars(self): + return self.vars.copy() + + def set_priority(self, priority): + try: + self.priority = int(priority) + except TypeError: + # FIXME: warn about invalid priority + pass diff --git a/lib/ansible/inventory/helpers.py b/lib/ansible/inventory/helpers.py new file mode 100644 index 0000000..39c7221 --- /dev/null +++ b/lib/ansible/inventory/helpers.py @@ -0,0 +1,40 @@ +# (c) 2017, Ansible by RedHat Inc, +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +############################################# +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible.utils.vars import combine_vars + + +def sort_groups(groups): + return sorted(groups, key=lambda g: (g.depth, g.priority, g.name)) + + +def get_group_vars(groups): + """ + Combine all the group vars from a list of inventory groups. + + :param groups: list of ansible.inventory.group.Group objects + :rtype: dict + """ + results = {} + for group in sort_groups(groups): + results = combine_vars(results, group.get_vars()) + + return results diff --git a/lib/ansible/inventory/host.py b/lib/ansible/inventory/host.py new file mode 100644 index 0000000..18569ce --- /dev/null +++ b/lib/ansible/inventory/host.py @@ -0,0 +1,169 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from collections.abc import Mapping, MutableMapping + +from ansible.inventory.group import Group +from ansible.parsing.utils.addresses import patterns +from ansible.utils.vars import combine_vars, get_unique_id + + +__all__ = ['Host'] + + +class Host: + ''' a single ansible host ''' + + # __slots__ = [ 'name', 'vars', 'groups' ] + + def __getstate__(self): + return self.serialize() + + def __setstate__(self, data): + return self.deserialize(data) + + def __eq__(self, other): + if not isinstance(other, Host): + return False + return self._uuid == other._uuid + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash(self.name) + + def __str__(self): + return self.get_name() + + def __repr__(self): + return self.get_name() + + def serialize(self): + groups = [] + for group in self.groups: + groups.append(group.serialize()) + + return dict( + name=self.name, + vars=self.vars.copy(), + address=self.address, + uuid=self._uuid, + groups=groups, + implicit=self.implicit, + ) + + def deserialize(self, data): + self.__init__(gen_uuid=False) # used by __setstate__ to deserialize in place # pylint: disable=unnecessary-dunder-call + + self.name = data.get('name') + self.vars = data.get('vars', dict()) + self.address = data.get('address', '') + self._uuid = data.get('uuid', None) + self.implicit = data.get('implicit', False) + + groups = data.get('groups', []) + for group_data in groups: + g = Group() + g.deserialize(group_data) + self.groups.append(g) + + def __init__(self, name=None, port=None, gen_uuid=True): + + self.vars = {} + self.groups = [] + self._uuid = None + + self.name = name + self.address = name + + if port: + self.set_variable('ansible_port', int(port)) + + if gen_uuid: + self._uuid = get_unique_id() + self.implicit = False + + def get_name(self): + return self.name + + def populate_ancestors(self, additions=None): + # populate ancestors + if additions is None: + for group in self.groups: + self.add_group(group) + else: + for group in additions: + if group not in self.groups: + self.groups.append(group) + + def add_group(self, group): + added = False + # populate ancestors first + for oldg in group.get_ancestors(): + if oldg not in self.groups: + self.groups.append(oldg) + + # actually add group + if group not in self.groups: + self.groups.append(group) + added = True + return added + + def remove_group(self, group): + removed = False + if group in self.groups: + self.groups.remove(group) + removed = True + + # remove exclusive ancestors, xcept all! + for oldg in group.get_ancestors(): + if oldg.name != 'all': + for childg in self.groups: + if oldg in childg.get_ancestors(): + break + else: + self.remove_group(oldg) + return removed + + def set_variable(self, key, value): + if key in self.vars and isinstance(self.vars[key], MutableMapping) and isinstance(value, Mapping): + self.vars = combine_vars(self.vars, {key: value}) + else: + self.vars[key] = value + + def get_groups(self): + return self.groups + + def get_magic_vars(self): + results = {} + results['inventory_hostname'] = self.name + if patterns['ipv4'].match(self.name) or patterns['ipv6'].match(self.name): + results['inventory_hostname_short'] = self.name + else: + results['inventory_hostname_short'] = self.name.split('.')[0] + + results['group_names'] = sorted([g.name for g in self.get_groups() if g.name != 'all']) + + return results + + def get_vars(self): + return combine_vars(self.vars, self.get_magic_vars()) diff --git a/lib/ansible/inventory/manager.py b/lib/ansible/inventory/manager.py new file mode 100644 index 0000000..400bc6b --- /dev/null +++ b/lib/ansible/inventory/manager.py @@ -0,0 +1,752 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +############################################# +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import fnmatch +import os +import sys +import re +import itertools +import traceback + +from operator import attrgetter +from random import shuffle + +from ansible import constants as C +from ansible.errors import AnsibleError, AnsibleOptionsError, AnsibleParserError +from ansible.inventory.data import InventoryData +from ansible.module_utils.six import string_types +from ansible.module_utils._text import to_bytes, to_text +from ansible.parsing.utils.addresses import parse_address +from ansible.plugins.loader import inventory_loader +from ansible.utils.helpers import deduplicate_list +from ansible.utils.path import unfrackpath +from ansible.utils.display import Display +from ansible.utils.vars import combine_vars +from ansible.vars.plugins import get_vars_from_inventory_sources + +display = Display() + +IGNORED_ALWAYS = [br"^\.", b"^host_vars$", b"^group_vars$", b"^vars_plugins$"] +IGNORED_PATTERNS = [to_bytes(x) for x in C.INVENTORY_IGNORE_PATTERNS] +IGNORED_EXTS = [b'%s$' % to_bytes(re.escape(x)) for x in C.INVENTORY_IGNORE_EXTS] + +IGNORED = re.compile(b'|'.join(IGNORED_ALWAYS + IGNORED_PATTERNS + IGNORED_EXTS)) + +PATTERN_WITH_SUBSCRIPT = re.compile( + r'''^ + (.+) # A pattern expression ending with... + \[(?: # A [subscript] expression comprising: + (-?[0-9]+)| # A single positive or negative number + ([0-9]+)([:-]) # Or an x:y or x: range. + ([0-9]*) + )\] + $ + ''', re.X +) + + +def order_patterns(patterns): + ''' takes a list of patterns and reorders them by modifier to apply them consistently ''' + + # FIXME: this goes away if we apply patterns incrementally or by groups + pattern_regular = [] + pattern_intersection = [] + pattern_exclude = [] + for p in patterns: + if not p: + continue + + if p[0] == "!": + pattern_exclude.append(p) + elif p[0] == "&": + pattern_intersection.append(p) + else: + pattern_regular.append(p) + + # if no regular pattern was given, hence only exclude and/or intersection + # make that magically work + if pattern_regular == []: + pattern_regular = ['all'] + + # when applying the host selectors, run those without the "&" or "!" + # first, then the &s, then the !s. + return pattern_regular + pattern_intersection + pattern_exclude + + +def split_host_pattern(pattern): + """ + Takes a string containing host patterns separated by commas (or a list + thereof) and returns a list of single patterns (which may not contain + commas). Whitespace is ignored. + + Also accepts ':' as a separator for backwards compatibility, but it is + not recommended due to the conflict with IPv6 addresses and host ranges. + + Example: 'a,b[1], c[2:3] , d' -> ['a', 'b[1]', 'c[2:3]', 'd'] + """ + + if isinstance(pattern, list): + results = (split_host_pattern(p) for p in pattern) + # flatten the results + return list(itertools.chain.from_iterable(results)) + elif not isinstance(pattern, string_types): + pattern = to_text(pattern, errors='surrogate_or_strict') + + # If it's got commas in it, we'll treat it as a straightforward + # comma-separated list of patterns. + if u',' in pattern: + patterns = pattern.split(u',') + + # If it doesn't, it could still be a single pattern. This accounts for + # non-separator uses of colons: IPv6 addresses and [x:y] host ranges. + else: + try: + (base, port) = parse_address(pattern, allow_ranges=True) + patterns = [pattern] + except Exception: + # The only other case we accept is a ':'-separated list of patterns. + # This mishandles IPv6 addresses, and is retained only for backwards + # compatibility. + patterns = re.findall( + to_text(r'''(?: # We want to match something comprising: + [^\s:\[\]] # (anything other than whitespace or ':[]' + | # ...or... + \[[^\]]*\] # a single complete bracketed expression) + )+ # occurring once or more + '''), pattern, re.X + ) + + return [p.strip() for p in patterns if p.strip()] + + +class InventoryManager(object): + ''' Creates and manages inventory ''' + + def __init__(self, loader, sources=None, parse=True, cache=True): + + # base objects + self._loader = loader + self._inventory = InventoryData() + + # a list of host(names) to contain current inquiries to + self._restriction = None + self._subset = None + + # caches + self._hosts_patterns_cache = {} # resolved full patterns + self._pattern_cache = {} # resolved individual patterns + + # the inventory dirs, files, script paths or lists of hosts + if sources is None: + self._sources = [] + elif isinstance(sources, string_types): + self._sources = [sources] + else: + self._sources = sources + + # get to work! + if parse: + self.parse_sources(cache=cache) + + self._cached_dynamic_hosts = [] + self._cached_dynamic_grouping = [] + + @property + def localhost(self): + return self._inventory.get_host('localhost') + + @property + def groups(self): + return self._inventory.groups + + @property + def hosts(self): + return self._inventory.hosts + + def add_host(self, host, group=None, port=None): + return self._inventory.add_host(host, group, port) + + def add_group(self, group): + return self._inventory.add_group(group) + + def get_groups_dict(self): + return self._inventory.get_groups_dict() + + def reconcile_inventory(self): + self.clear_caches() + return self._inventory.reconcile_inventory() + + def get_host(self, hostname): + return self._inventory.get_host(hostname) + + def _fetch_inventory_plugins(self): + ''' sets up loaded inventory plugins for usage ''' + + display.vvvv('setting up inventory plugins') + + plugins = [] + for name in C.INVENTORY_ENABLED: + plugin = inventory_loader.get(name) + if plugin: + plugins.append(plugin) + else: + display.warning('Failed to load inventory plugin, skipping %s' % name) + + if not plugins: + raise AnsibleError("No inventory plugins available to generate inventory, make sure you have at least one enabled.") + + return plugins + + def parse_sources(self, cache=False): + ''' iterate over inventory sources and parse each one to populate it''' + + parsed = False + # allow for multiple inventory parsing + for source in self._sources: + + if source: + if ',' not in source: + source = unfrackpath(source, follow=False) + parse = self.parse_source(source, cache=cache) + if parse and not parsed: + parsed = True + + if parsed: + # do post processing + self._inventory.reconcile_inventory() + else: + if C.INVENTORY_UNPARSED_IS_FAILED: + raise AnsibleError("No inventory was parsed, please check your configuration and options.") + elif C.INVENTORY_UNPARSED_WARNING: + display.warning("No inventory was parsed, only implicit localhost is available") + + for group in self.groups.values(): + group.vars = combine_vars(group.vars, get_vars_from_inventory_sources(self._loader, self._sources, [group], 'inventory')) + for host in self.hosts.values(): + host.vars = combine_vars(host.vars, get_vars_from_inventory_sources(self._loader, self._sources, [host], 'inventory')) + + def parse_source(self, source, cache=False): + ''' Generate or update inventory for the source provided ''' + + parsed = False + failures = [] + display.debug(u'Examining possible inventory source: %s' % source) + + # use binary for path functions + b_source = to_bytes(source) + + # process directories as a collection of inventories + if os.path.isdir(b_source): + display.debug(u'Searching for inventory files in directory: %s' % source) + for i in sorted(os.listdir(b_source)): + + display.debug(u'Considering %s' % i) + # Skip hidden files and stuff we explicitly ignore + if IGNORED.search(i): + continue + + # recursively deal with directory entries + fullpath = to_text(os.path.join(b_source, i), errors='surrogate_or_strict') + parsed_this_one = self.parse_source(fullpath, cache=cache) + display.debug(u'parsed %s as %s' % (fullpath, parsed_this_one)) + if not parsed: + parsed = parsed_this_one + else: + # left with strings or files, let plugins figure it out + + # set so new hosts can use for inventory_file/dir vars + self._inventory.current_source = source + + # try source with each plugin + for plugin in self._fetch_inventory_plugins(): + + plugin_name = to_text(getattr(plugin, '_load_name', getattr(plugin, '_original_path', ''))) + display.debug(u'Attempting to use plugin %s (%s)' % (plugin_name, plugin._original_path)) + + # initialize and figure out if plugin wants to attempt parsing this file + try: + plugin_wants = bool(plugin.verify_file(source)) + except Exception: + plugin_wants = False + + if plugin_wants: + try: + # FIXME in case plugin fails 1/2 way we have partial inventory + plugin.parse(self._inventory, self._loader, source, cache=cache) + try: + plugin.update_cache_if_changed() + except AttributeError: + # some plugins might not implement caching + pass + parsed = True + display.vvv('Parsed %s inventory source with %s plugin' % (source, plugin_name)) + break + except AnsibleParserError as e: + display.debug('%s was not parsable by %s' % (source, plugin_name)) + tb = ''.join(traceback.format_tb(sys.exc_info()[2])) + failures.append({'src': source, 'plugin': plugin_name, 'exc': e, 'tb': tb}) + except Exception as e: + display.debug('%s failed while attempting to parse %s' % (plugin_name, source)) + tb = ''.join(traceback.format_tb(sys.exc_info()[2])) + failures.append({'src': source, 'plugin': plugin_name, 'exc': AnsibleError(e), 'tb': tb}) + else: + display.vvv("%s declined parsing %s as it did not pass its verify_file() method" % (plugin_name, source)) + + if parsed: + self._inventory.processed_sources.append(self._inventory.current_source) + else: + # only warn/error if NOT using the default or using it and the file is present + # TODO: handle 'non file' inventory and detect vs hardcode default + if source != '/etc/ansible/hosts' or os.path.exists(source): + + if failures: + # only if no plugin processed files should we show errors. + for fail in failures: + display.warning(u'\n* Failed to parse %s with %s plugin: %s' % (to_text(fail['src']), fail['plugin'], to_text(fail['exc']))) + if 'tb' in fail: + display.vvv(to_text(fail['tb'])) + + # final error/warning on inventory source failure + if C.INVENTORY_ANY_UNPARSED_IS_FAILED: + raise AnsibleError(u'Completely failed to parse inventory source %s' % (source)) + else: + display.warning("Unable to parse %s as an inventory source" % source) + + # clear up, jic + self._inventory.current_source = None + + return parsed + + def clear_caches(self): + ''' clear all caches ''' + self._hosts_patterns_cache = {} + self._pattern_cache = {} + + def refresh_inventory(self): + ''' recalculate inventory ''' + + self.clear_caches() + self._inventory = InventoryData() + self.parse_sources(cache=False) + for host in self._cached_dynamic_hosts: + self.add_dynamic_host(host, {'refresh': True}) + for host, result in self._cached_dynamic_grouping: + result['refresh'] = True + self.add_dynamic_group(host, result) + + def _match_list(self, items, pattern_str): + # compile patterns + try: + if not pattern_str[0] == '~': + pattern = re.compile(fnmatch.translate(pattern_str)) + else: + pattern = re.compile(pattern_str[1:]) + except Exception: + raise AnsibleError('Invalid host list pattern: %s' % pattern_str) + + # apply patterns + results = [] + for item in items: + if pattern.match(item): + results.append(item) + return results + + def get_hosts(self, pattern="all", ignore_limits=False, ignore_restrictions=False, order=None): + """ + Takes a pattern or list of patterns and returns a list of matching + inventory host names, taking into account any active restrictions + or applied subsets + """ + + hosts = [] + + # Check if pattern already computed + if isinstance(pattern, list): + pattern_list = pattern[:] + else: + pattern_list = [pattern] + + if pattern_list: + if not ignore_limits and self._subset: + pattern_list.extend(self._subset) + + if not ignore_restrictions and self._restriction: + pattern_list.extend(self._restriction) + + # This is only used as a hash key in the self._hosts_patterns_cache dict + # a tuple is faster than stringifying + pattern_hash = tuple(pattern_list) + + if pattern_hash not in self._hosts_patterns_cache: + + patterns = split_host_pattern(pattern) + hosts = self._evaluate_patterns(patterns) + + # mainly useful for hostvars[host] access + if not ignore_limits and self._subset: + # exclude hosts not in a subset, if defined + subset_uuids = set(s._uuid for s in self._evaluate_patterns(self._subset)) + hosts = [h for h in hosts if h._uuid in subset_uuids] + + if not ignore_restrictions and self._restriction: + # exclude hosts mentioned in any restriction (ex: failed hosts) + hosts = [h for h in hosts if h.name in self._restriction] + + self._hosts_patterns_cache[pattern_hash] = deduplicate_list(hosts) + + # sort hosts list if needed (should only happen when called from strategy) + if order in ['sorted', 'reverse_sorted']: + hosts = sorted(self._hosts_patterns_cache[pattern_hash][:], key=attrgetter('name'), reverse=(order == 'reverse_sorted')) + elif order == 'reverse_inventory': + hosts = self._hosts_patterns_cache[pattern_hash][::-1] + else: + hosts = self._hosts_patterns_cache[pattern_hash][:] + if order == 'shuffle': + shuffle(hosts) + elif order not in [None, 'inventory']: + raise AnsibleOptionsError("Invalid 'order' specified for inventory hosts: %s" % order) + + return hosts + + def _evaluate_patterns(self, patterns): + """ + Takes a list of patterns and returns a list of matching host names, + taking into account any negative and intersection patterns. + """ + + patterns = order_patterns(patterns) + hosts = [] + + for p in patterns: + # avoid resolving a pattern that is a plain host + if p in self._inventory.hosts: + hosts.append(self._inventory.get_host(p)) + else: + that = self._match_one_pattern(p) + if p[0] == "!": + that = set(that) + hosts = [h for h in hosts if h not in that] + elif p[0] == "&": + that = set(that) + hosts = [h for h in hosts if h in that] + else: + existing_hosts = set(y.name for y in hosts) + hosts.extend([h for h in that if h.name not in existing_hosts]) + return hosts + + def _match_one_pattern(self, pattern): + """ + Takes a single pattern and returns a list of matching host names. + Ignores intersection (&) and exclusion (!) specifiers. + + The pattern may be: + + 1. A regex starting with ~, e.g. '~[abc]*' + 2. A shell glob pattern with ?/*/[chars]/[!chars], e.g. 'foo*' + 3. An ordinary word that matches itself only, e.g. 'foo' + + The pattern is matched using the following rules: + + 1. If it's 'all', it matches all hosts in all groups. + 2. Otherwise, for each known group name: + (a) if it matches the group name, the results include all hosts + in the group or any of its children. + (b) otherwise, if it matches any hosts in the group, the results + include the matching hosts. + + This means that 'foo*' may match one or more groups (thus including all + hosts therein) but also hosts in other groups. + + The built-in groups 'all' and 'ungrouped' are special. No pattern can + match these group names (though 'all' behaves as though it matches, as + described above). The word 'ungrouped' can match a host of that name, + and patterns like 'ungr*' and 'al*' can match either hosts or groups + other than all and ungrouped. + + If the pattern matches one or more group names according to these rules, + it may have an optional range suffix to select a subset of the results. + This is allowed only if the pattern is not a regex, i.e. '~foo[1]' does + not work (the [1] is interpreted as part of the regex), but 'foo*[1]' + would work if 'foo*' matched the name of one or more groups. + + Duplicate matches are always eliminated from the results. + """ + + if pattern[0] in ("&", "!"): + pattern = pattern[1:] + + if pattern not in self._pattern_cache: + (expr, slice) = self._split_subscript(pattern) + hosts = self._enumerate_matches(expr) + try: + hosts = self._apply_subscript(hosts, slice) + except IndexError: + raise AnsibleError("No hosts matched the subscripted pattern '%s'" % pattern) + self._pattern_cache[pattern] = hosts + + return self._pattern_cache[pattern] + + def _split_subscript(self, pattern): + """ + Takes a pattern, checks if it has a subscript, and returns the pattern + without the subscript and a (start,end) tuple representing the given + subscript (or None if there is no subscript). + + Validates that the subscript is in the right syntax, but doesn't make + sure the actual indices make sense in context. + """ + + # Do not parse regexes for enumeration info + if pattern[0] == '~': + return (pattern, None) + + # We want a pattern followed by an integer or range subscript. + # (We can't be more restrictive about the expression because the + # fnmatch semantics permit [\[:\]] to occur.) + + subscript = None + m = PATTERN_WITH_SUBSCRIPT.match(pattern) + if m: + (pattern, idx, start, sep, end) = m.groups() + if idx: + subscript = (int(idx), None) + else: + if not end: + end = -1 + subscript = (int(start), int(end)) + if sep == '-': + display.warning("Use [x:y] inclusive subscripts instead of [x-y] which has been removed") + + return (pattern, subscript) + + def _apply_subscript(self, hosts, subscript): + """ + Takes a list of hosts and a (start,end) tuple and returns the subset of + hosts based on the subscript (which may be None to return all hosts). + """ + + if not hosts or not subscript: + return hosts + + (start, end) = subscript + + if end: + if end == -1: + end = len(hosts) - 1 + return hosts[start:end + 1] + else: + return [hosts[start]] + + def _enumerate_matches(self, pattern): + """ + Returns a list of host names matching the given pattern according to the + rules explained above in _match_one_pattern. + """ + + results = [] + # check if pattern matches group + matching_groups = self._match_list(self._inventory.groups, pattern) + if matching_groups: + for groupname in matching_groups: + results.extend(self._inventory.groups[groupname].get_hosts()) + + # check hosts if no groups matched or it is a regex/glob pattern + if not matching_groups or pattern[0] == '~' or any(special in pattern for special in ('.', '?', '*', '[')): + # pattern might match host + matching_hosts = self._match_list(self._inventory.hosts, pattern) + if matching_hosts: + for hostname in matching_hosts: + results.append(self._inventory.hosts[hostname]) + + if not results and pattern in C.LOCALHOST: + # get_host autocreates implicit when needed + implicit = self._inventory.get_host(pattern) + if implicit: + results.append(implicit) + + # Display warning if specified host pattern did not match any groups or hosts + if not results and not matching_groups and pattern != 'all': + msg = "Could not match supplied host pattern, ignoring: %s" % pattern + display.debug(msg) + if C.HOST_PATTERN_MISMATCH == 'warning': + display.warning(msg) + elif C.HOST_PATTERN_MISMATCH == 'error': + raise AnsibleError(msg) + # no need to write 'ignore' state + + return results + + def list_hosts(self, pattern="all"): + """ return a list of hostnames for a pattern """ + # FIXME: cache? + result = self.get_hosts(pattern) + + # allow implicit localhost if pattern matches and no other results + if len(result) == 0 and pattern in C.LOCALHOST: + result = [pattern] + + return result + + def list_groups(self): + # FIXME: cache? + return sorted(self._inventory.groups.keys()) + + def restrict_to_hosts(self, restriction): + """ + Restrict list operations to the hosts given in restriction. This is used + to batch serial operations in main playbook code, don't use this for other + reasons. + """ + if restriction is None: + return + elif not isinstance(restriction, list): + restriction = [restriction] + self._restriction = set(to_text(h.name) for h in restriction) + + def subset(self, subset_pattern): + """ + Limits inventory results to a subset of inventory that matches a given + pattern, such as to select a given geographic of numeric slice amongst + a previous 'hosts' selection that only select roles, or vice versa. + Corresponds to --limit parameter to ansible-playbook + """ + if subset_pattern is None: + self._subset = None + else: + subset_patterns = split_host_pattern(subset_pattern) + results = [] + # allow Unix style @filename data + for x in subset_patterns: + if not x: + continue + + if x[0] == "@": + b_limit_file = to_bytes(x[1:]) + if not os.path.exists(b_limit_file): + raise AnsibleError(u'Unable to find limit file %s' % b_limit_file) + if not os.path.isfile(b_limit_file): + raise AnsibleError(u'Limit starting with "@" must be a file, not a directory: %s' % b_limit_file) + with open(b_limit_file) as fd: + results.extend([to_text(l.strip()) for l in fd.read().split("\n")]) + else: + results.append(to_text(x)) + self._subset = results + + def remove_restriction(self): + """ Do not restrict list operations """ + self._restriction = None + + def clear_pattern_cache(self): + self._pattern_cache = {} + + def add_dynamic_host(self, host_info, result_item): + ''' + Helper function to add a new host to inventory based on a task result. + ''' + + changed = False + if not result_item.get('refresh'): + self._cached_dynamic_hosts.append(host_info) + + if host_info: + host_name = host_info.get('host_name') + + # Check if host in inventory, add if not + if host_name not in self.hosts: + self.add_host(host_name, 'all') + changed = True + new_host = self.hosts.get(host_name) + + # Set/update the vars for this host + new_host_vars = new_host.get_vars() + new_host_combined_vars = combine_vars(new_host_vars, host_info.get('host_vars', dict())) + if new_host_vars != new_host_combined_vars: + new_host.vars = new_host_combined_vars + changed = True + + new_groups = host_info.get('groups', []) + for group_name in new_groups: + if group_name not in self.groups: + group_name = self._inventory.add_group(group_name) + changed = True + new_group = self.groups[group_name] + if new_group.add_host(self.hosts[host_name]): + changed = True + + # reconcile inventory, ensures inventory rules are followed + if changed: + self.reconcile_inventory() + + result_item['changed'] = changed + + def add_dynamic_group(self, host, result_item): + ''' + Helper function to add a group (if it does not exist), and to assign the + specified host to that group. + ''' + + changed = False + + if not result_item.get('refresh'): + self._cached_dynamic_grouping.append((host, result_item)) + + # the host here is from the executor side, which means it was a + # serialized/cloned copy and we'll need to look up the proper + # host object from the master inventory + real_host = self.hosts.get(host.name) + if real_host is None: + if host.name == self.localhost.name: + real_host = self.localhost + elif not result_item.get('refresh'): + raise AnsibleError('%s cannot be matched in inventory' % host.name) + else: + # host was removed from inventory during refresh, we should not process + return + + group_name = result_item.get('add_group') + parent_group_names = result_item.get('parent_groups', []) + + if group_name not in self.groups: + group_name = self.add_group(group_name) + + for name in parent_group_names: + if name not in self.groups: + # create the new group and add it to inventory + self.add_group(name) + changed = True + + group = self._inventory.groups[group_name] + for parent_group_name in parent_group_names: + parent_group = self.groups[parent_group_name] + new = parent_group.add_child_group(group) + if new and not changed: + changed = True + + if real_host not in group.get_hosts(): + changed = group.add_host(real_host) + + if group not in real_host.get_groups(): + changed = real_host.add_group(group) + + if changed: + self.reconcile_inventory() + + result_item['changed'] = changed |