# Copyright (c) 2022, Dell Inc. or its subsidiaries. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See the LICENSE file for details. # # This file is part of NVMe STorage Appliance Services (nvme-stas). # # Authors: Martin Belanger # '''nvme-stas configuration module''' import re import os import sys import logging import functools import configparser from urllib.parse import urlparse from staslib import defs, iputil, nbft, singleton, timeparse __TOKEN_RE = re.compile(r'\s*;\s*') __OPTION_RE = re.compile(r'\s*=\s*') class InvalidOption(Exception): '''Exception raised when an invalid option value is detected''' def _parse_controller(controller): '''@brief Parse a "controller" entry. Controller entries are strings composed of several configuration parameters delimited by semi-colons. Each configuration parameter is specified as a "key=value" pair. @return A dictionary of key-value pairs. ''' options = dict() tokens = __TOKEN_RE.split(controller) for token in tokens: if token: try: option, val = __OPTION_RE.split(token) options[option.strip()] = val.strip() except ValueError: pass return options def _parse_single_val(text): if isinstance(text, str): return text if not isinstance(text, list) or len(text) == 0: return None return text[-1] def _parse_list(text): return text if isinstance(text, list) else [text] def _to_int(text): try: return int(_parse_single_val(text)) except (ValueError, TypeError): raise InvalidOption # pylint: disable=raise-missing-from def _to_bool(text, positive='true'): return _parse_single_val(text).lower() == positive def _to_ncc(text): value = _to_int(text) if value == 1: # 1 is invalid. A minimum of 2 is required (with the exception of 0, which is valid). value = 2 return value def _to_ip_family(text): return tuple((4 if text == 'ipv4' else 6 for text in _parse_single_val(text).split('+'))) # ****************************************************************************** class OrderedMultisetDict(dict): '''This class is used to change the behavior of configparser.ConfigParser and allow multiple configuration parameters with the same key. The result is a list of values, where values are sorted by the order they appear in the file. ''' def __setitem__(self, key, value): if key in self and isinstance(value, list): self[key].extend(value) else: super().__setitem__(key, value) def __getitem__(self, key): value = super().__getitem__(key) if isinstance(value, str): return value.split('\n') return value class SvcConf(metaclass=singleton.Singleton): # pylint: disable=too-many-public-methods '''Read and cache configuration file.''' OPTION_CHECKER = { 'Global': { 'tron': { 'convert': _to_bool, 'default': False, 'txt-chk': lambda text: _parse_single_val(text).lower() in ('false', 'true'), }, 'kato': { 'convert': _to_int, }, 'pleo': { 'convert': functools.partial(_to_bool, positive='enabled'), 'default': True, 'txt-chk': lambda text: _parse_single_val(text).lower() in ('disabled', 'enabled'), }, 'ip-family': { 'convert': _to_ip_family, 'default': (4, 6), 'txt-chk': lambda text: _parse_single_val(text) in ('ipv4', 'ipv6', 'ipv4+ipv6', 'ipv6+ipv4'), }, 'queue-size': { 'convert': _to_int, 'rng-chk': lambda value: None if value in range(16, 1025) else range(16, 1025), }, 'hdr-digest': { 'convert': _to_bool, 'default': False, 'txt-chk': lambda text: _parse_single_val(text).lower() in ('false', 'true'), }, 'data-digest': { 'convert': _to_bool, 'default': False, 'txt-chk': lambda text: _parse_single_val(text).lower() in ('false', 'true'), }, 'ignore-iface': { 'convert': _to_bool, 'default': False, 'txt-chk': lambda text: _parse_single_val(text).lower() in ('false', 'true'), }, 'nr-io-queues': { 'convert': _to_int, }, 'ctrl-loss-tmo': { 'convert': _to_int, }, 'disable-sqflow': { 'convert': _to_bool, 'txt-chk': lambda text: _parse_single_val(text).lower() in ('false', 'true'), }, 'nr-poll-queues': { 'convert': _to_int, }, 'nr-write-queues': { 'convert': _to_int, }, 'reconnect-delay': { 'convert': _to_int, }, ### BEGIN: LEGACY SECTION TO BE REMOVED ### 'persistent-connections': { 'convert': _to_bool, 'default': False, 'txt-chk': lambda text: _parse_single_val(text).lower() in ('false', 'true'), }, ### END: LEGACY SECTION TO BE REMOVED ### }, 'Service Discovery': { 'zeroconf': { 'convert': functools.partial(_to_bool, positive='enabled'), 'default': True, 'txt-chk': lambda text: _parse_single_val(text).lower() in ('disabled', 'enabled'), }, }, 'Discovery controller connection management': { 'persistent-connections': { 'convert': _to_bool, 'default': True, 'txt-chk': lambda text: _parse_single_val(text).lower() in ('false', 'true'), }, 'zeroconf-connections-persistence': { 'convert': lambda text: timeparse.timeparse(_parse_single_val(text)), 'default': timeparse.timeparse('72hours'), }, }, 'I/O controller connection management': { 'disconnect-scope': { 'convert': _parse_single_val, 'default': 'only-stas-connections', 'txt-chk': lambda text: _parse_single_val(text) in ('only-stas-connections', 'all-connections-matching-disconnect-trtypes', 'no-disconnect'), }, 'disconnect-trtypes': { # Use set() to eliminate potential duplicates 'convert': lambda text: set(_parse_single_val(text).split('+')), 'default': [ 'tcp', ], 'lst-chk': ('tcp', 'rdma', 'fc'), }, 'connect-attempts-on-ncc': { 'convert': _to_ncc, 'default': 0, }, }, 'Controllers': { 'controller': { 'convert': _parse_list, 'default': [], }, 'exclude': { 'convert': _parse_list, 'default': [], }, ### BEGIN: LEGACY SECTION TO BE REMOVED ### 'blacklist': { 'convert': _parse_list, 'default': [], }, ### END: LEGACY SECTION TO BE REMOVED ### }, } def __init__(self, default_conf=None, conf_file='/dev/null'): self._config = None self._defaults = default_conf if default_conf else {} if self._defaults is not None and len(self._defaults) != 0: self._valid_conf = {} for section, option in self._defaults: self._valid_conf.setdefault(section, set()).add(option) else: self._valid_conf = None self._conf_file = conf_file self.reload() def reload(self): '''@brief Reload the configuration file.''' self._config = self._read_conf_file() @property def conf_file(self): '''Return the configuration file name''' return self._conf_file def set_conf_file(self, fname): '''Set the configuration file name and reload config''' self._conf_file = fname self.reload() def get_option(self, section, option, ignore_default=False): # pylint: disable=too-many-locals '''Retrieve @option from @section, convert raw text to appropriate object type, and validate.''' try: checker = self.OPTION_CHECKER[section][option] except KeyError: logging.error('Requesting invalid section=%s and/or option=%s', section, option) raise default = checker.get('default', None) try: text = self._config.get(section=section, option=option) except (configparser.NoSectionError, configparser.NoOptionError, KeyError): return None if ignore_default else self._defaults.get((section, option), default) return self._check(text, section, option, default) tron = property(functools.partial(get_option, section='Global', option='tron')) kato = property(functools.partial(get_option, section='Global', option='kato')) ip_family = property(functools.partial(get_option, section='Global', option='ip-family')) queue_size = property(functools.partial(get_option, section='Global', option='queue-size')) hdr_digest = property(functools.partial(get_option, section='Global', option='hdr-digest')) data_digest = property(functools.partial(get_option, section='Global', option='data-digest')) ignore_iface = property(functools.partial(get_option, section='Global', option='ignore-iface')) pleo_enabled = property(functools.partial(get_option, section='Global', option='pleo')) nr_io_queues = property(functools.partial(get_option, section='Global', option='nr-io-queues')) ctrl_loss_tmo = property(functools.partial(get_option, section='Global', option='ctrl-loss-tmo')) disable_sqflow = property(functools.partial(get_option, section='Global', option='disable-sqflow')) nr_poll_queues = property(functools.partial(get_option, section='Global', option='nr-poll-queues')) nr_write_queues = property(functools.partial(get_option, section='Global', option='nr-write-queues')) reconnect_delay = property(functools.partial(get_option, section='Global', option='reconnect-delay')) zeroconf_enabled = property(functools.partial(get_option, section='Service Discovery', option='zeroconf')) zeroconf_persistence_sec = property( functools.partial( get_option, section='Discovery controller connection management', option='zeroconf-connections-persistence' ) ) disconnect_scope = property( functools.partial(get_option, section='I/O controller connection management', option='disconnect-scope') ) disconnect_trtypes = property( functools.partial(get_option, section='I/O controller connection management', option='disconnect-trtypes') ) connect_attempts_on_ncc = property( functools.partial(get_option, section='I/O controller connection management', option='connect-attempts-on-ncc') ) @property def stypes(self): '''@brief Get the DNS-SD/mDNS service types.''' return ['_nvme-disc._tcp', '_nvme-disc._udp'] if self.zeroconf_enabled else list() @property def persistent_connections(self): '''@brief return the "persistent-connections" config parameter''' section = 'Discovery controller connection management' option = 'persistent-connections' value = self.get_option(section, option, ignore_default=True) legacy = self.get_option('Global', option, ignore_default=True) if value is None and legacy is None: return self._defaults.get((section, option), True) return value or legacy def get_controllers(self): '''@brief Return the list of controllers in the config file. Each controller is in the form of a dictionary as follows. Note that some of the keys are optional. { 'transport': [TRANSPORT], 'traddr': [TRADDR], 'trsvcid': [TRSVCID], 'host-traddr': [TRADDR], 'host-iface': [IFACE], 'subsysnqn': [NQN], 'dhchap-ctrl-secret': [KEY], 'hdr-digest': [BOOL] 'data-digest': [BOOL] 'nr-io-queues': [NUMBER] 'nr-write-queues': [NUMBER] 'nr-poll-queues': [NUMBER] 'queue-size': [SIZE] 'kato': [KATO] 'reconnect-delay': [SECONDS] 'ctrl-loss-tmo': [SECONDS] 'disable-sqflow': [BOOL] } ''' controller_list = self.get_option('Controllers', 'controller') cids = [_parse_controller(controller) for controller in controller_list] for cid in cids: try: # replace 'nqn' key by 'subsysnqn', if present. cid['subsysnqn'] = cid.pop('nqn') except KeyError: pass # Verify values of the options used to overload the matching [Global] options for option in cid: if option in self.OPTION_CHECKER['Global']: value = self._check(cid[option], 'Global', option, None) if value is not None: cid[option] = value return cids def get_excluded(self): '''@brief Return the list of excluded controllers in the config file. Each excluded controller is in the form of a dictionary as follows. All the keys are optional. { 'transport': [TRANSPORT], 'traddr': [TRADDR], 'trsvcid': [TRSVCID], 'host-iface': [IFACE], 'subsysnqn': [NQN], } ''' controller_list = self.get_option('Controllers', 'exclude') # 2022-09-20: Look for "blacklist". This is for backwards compatibility # with releases 1.0 to 1.1.x. This is to be phased out (i.e. remove by 2024) controller_list += self.get_option('Controllers', 'blacklist') excluded = [_parse_controller(controller) for controller in controller_list] for controller in excluded: controller.pop('host-traddr', None) # remove host-traddr try: # replace 'nqn' key by 'subsysnqn', if present. controller['subsysnqn'] = controller.pop('nqn') except KeyError: pass return excluded def _check(self, text, section, option, default): checker = self.OPTION_CHECKER[section][option] text_checker = checker.get('txt-chk', None) if text_checker is not None and not text_checker(text): logging.warning( 'File:%s [%s]: %s - Text check found invalid value "%s". Default will be used', self.conf_file, section, option, text, ) return self._defaults.get((section, option), default) converter = checker.get('convert', None) try: value = converter(text) except InvalidOption: logging.warning( 'File:%s [%s]: %s - Data converter found invalid value "%s". Default will be used', self.conf_file, section, option, text, ) return self._defaults.get((section, option), default) value_in_range = checker.get('rng-chk', None) if value_in_range is not None: expected_range = value_in_range(value) if expected_range is not None: logging.warning( 'File:%s [%s]: %s - "%s" is not within range %s..%s. Default will be used', self.conf_file, section, option, value, min(expected_range), max(expected_range), ) return self._defaults.get((section, option), default) list_checker = checker.get('lst-chk', None) if list_checker: values = set() for item in value: if item not in list_checker: logging.warning( 'File:%s [%s]: %s - List checker found invalid item "%s" will be ignored.', self.conf_file, section, option, item, ) else: values.add(item) if len(values) == 0: return self._defaults.get((section, option), default) value = list(values) return value def _read_conf_file(self): '''@brief Read the configuration file if the file exists.''' config = configparser.ConfigParser( default_section=None, allow_no_value=True, delimiters=('='), interpolation=None, strict=False, dict_type=OrderedMultisetDict, ) if self._conf_file and os.path.isfile(self._conf_file): config.read(self._conf_file) # Parse Configuration and validate. if self._valid_conf is not None: invalid_sections = set() for section in config.sections(): if section not in self._valid_conf: invalid_sections.add(section) else: invalid_options = set() for option in config.options(section): if option not in self._valid_conf.get(section, []): invalid_options.add(option) if len(invalid_options) != 0: logging.error( 'File:%s [%s] contains invalid options: %s', self.conf_file, section, invalid_options, ) if len(invalid_sections) != 0: logging.error( 'File:%s contains invalid sections: %s', self.conf_file, invalid_sections, ) return config # ****************************************************************************** class SysConf(metaclass=singleton.Singleton): '''Read and cache the host configuration file.''' def __init__(self, conf_file=defs.SYS_CONF_FILE): self._config = None self._conf_file = conf_file self.reload() def reload(self): '''@brief Reload the configuration file.''' self._config = self._read_conf_file() @property def conf_file(self): '''Return the configuration file name''' return self._conf_file def set_conf_file(self, fname): '''Set the configuration file name and reload config''' self._conf_file = fname self.reload() def as_dict(self): '''Return configuration as a dictionary''' return { 'hostnqn': self.hostnqn, 'hostid': self.hostid, 'hostkey': self.hostkey, 'symname': self.hostsymname, } @property def hostnqn(self): '''@brief return the host NQN @return: Host NQN @raise: Host NQN is mandatory. The program will terminate if a Host NQN cannot be determined. ''' try: value = self.__get_value('Host', 'nqn', defs.NVME_HOSTNQN) except FileNotFoundError as ex: sys.exit(f'Error reading mandatory Host NQN (see stasadm --help): {ex}') if value is not None and not value.startswith('nqn.'): sys.exit(f'Error Host NQN "{value}" should start with "nqn."') return value @property def hostid(self): '''@brief return the host ID @return: Host ID @raise: Host ID is mandatory. The program will terminate if a Host ID cannot be determined. ''' try: value = self.__get_value('Host', 'id', defs.NVME_HOSTID) except FileNotFoundError as ex: sys.exit(f'Error reading mandatory Host ID (see stasadm --help): {ex}') return value @property def hostkey(self): '''@brief return the host key @return: Host key @raise: Host key is optional, but mandatory if authorization will be performed. ''' try: value = self.__get_value('Host', 'key', defs.NVME_HOSTKEY) except FileNotFoundError as ex: logging.debug('Host key undefined: %s', ex) value = None return value @property def hostsymname(self): '''@brief return the host symbolic name (or None) @return: symbolic name or None ''' try: value = self.__get_value('Host', 'symname') except FileNotFoundError as ex: logging.warning('Error reading host symbolic name (will remain undefined): %s', ex) value = None return value def _read_conf_file(self): '''@brief Read the configuration file if the file exists.''' config = configparser.ConfigParser( default_section=None, allow_no_value=True, delimiters=('='), interpolation=None, strict=False ) if os.path.isfile(self._conf_file): config.read(self._conf_file) return config def __get_value(self, section, option, default_file=None): '''@brief A configuration file consists of sections, each led by a [section] header, followed by key/value entries separated by a equal sign (=). This method retrieves the value associated with the key @option from the section @section. If the value starts with the string "file://", then the value will be retrieved from that file. @param section: Configuration section @param option: The key to look for @param default_file: A file that contains the default value @return: On success, the value associated with the key. On failure, this method will return None is a default_file is not specified, or will raise an exception if a file is not found. @raise: This method will raise the FileNotFoundError exception if the value retrieved is a file that does not exist. ''' try: value = self._config.get(section=section, option=option) if not value.startswith('file://'): return value file = value[7:] except (configparser.NoSectionError, configparser.NoOptionError, KeyError): if default_file is None: return None file = default_file try: with open(file) as f: # pylint: disable=unspecified-encoding return f.readline().split()[0] except IndexError: return None # ****************************************************************************** class NvmeOptions(metaclass=singleton.Singleton): '''Object used to read and cache contents of file /dev/nvme-fabrics. Note that this file was not readable prior to Linux 5.16. ''' def __init__(self): # Supported options can be determined by looking at the kernel version # or by reading '/dev/nvme-fabrics'. The ability to read the options # from '/dev/nvme-fabrics' was only introduced in kernel 5.17, but may # have been backported to older kernels. In any case, if the kernel # version meets the minimum version for that option, then we don't # even need to read '/dev/nvme-fabrics'. self._supported_options = { 'discovery': defs.KERNEL_VERSION >= defs.KERNEL_TP8013_MIN_VERSION, 'host_iface': defs.KERNEL_VERSION >= defs.KERNEL_IFACE_MIN_VERSION, 'dhchap_secret': defs.KERNEL_VERSION >= defs.KERNEL_HOSTKEY_MIN_VERSION, 'dhchap_ctrl_secret': defs.KERNEL_VERSION >= defs.KERNEL_CTRLKEY_MIN_VERSION, } # If some of the options are False, we need to check wether they can be # read from '/dev/nvme-fabrics'. This method allows us to determine that # an older kernel actually supports a specific option because it was # backported to that kernel. if not all(self._supported_options.values()): # At least one option is False. try: with open('/dev/nvme-fabrics') as f: # pylint: disable=unspecified-encoding options = [option.split('=')[0].strip() for option in f.readline().rstrip('\n').split(',')] except PermissionError: # Must be root to read this file raise except (OSError, FileNotFoundError): logging.warning('Cannot determine which NVMe options the kernel supports') else: for option, supported in self._supported_options.items(): if not supported: self._supported_options[option] = option in options def __str__(self): return f'supported options: {self._supported_options}' def get(self): '''get the supported options as a dict''' return self._supported_options @property def discovery_supp(self): '''This option adds support for TP8013''' return self._supported_options['discovery'] @property def host_iface_supp(self): '''This option allows forcing connections to go over a specific interface regardless of the routing tables. ''' return self._supported_options['host_iface'] @property def dhchap_hostkey_supp(self): '''This option allows specifying the host DHCHAP key used for authentication.''' return self._supported_options['dhchap_secret'] @property def dhchap_ctrlkey_supp(self): '''This option allows specifying the controller DHCHAP key used for authentication.''' return self._supported_options['dhchap_ctrl_secret'] # ****************************************************************************** class NbftConf(metaclass=singleton.Singleton): '''Read and cache configuration file.''' def __init__(self, root_dir=defs.NBFT_SYSFS_PATH): self._disc_ctrls = [] self._subs_ctrls = [] nbft_files = nbft.get_nbft_files(root_dir) if len(nbft_files): logging.info('NBFT location(s): %s', list(nbft_files.keys())) for data in nbft_files.values(): hfis = data.get('hfi', []) discovery = data.get('discovery', []) subsystem = data.get('subsystem', []) self._disc_ctrls.extend(NbftConf.__nbft_disc_to_cids(discovery, hfis)) self._subs_ctrls.extend(NbftConf.__nbft_subs_to_cids(subsystem, hfis)) dcs = property(lambda self: self._disc_ctrls) iocs = property(lambda self: self._subs_ctrls) def get_controllers(self): '''Retrieve the list of controllers. Stafd only cares about discovery controllers. Stacd only cares about I/O controllers.''' # For now, only return DCs. There are still unanswered questions # regarding I/O controllers, e.g. what if multipathing has been # configured. return self.dcs if defs.PROG_NAME == 'stafd' else [] @staticmethod def __nbft_disc_to_cids(discovery, hfis): cids = [] for ctrl in discovery: cid = NbftConf.__uri2cid(ctrl['uri']) cid['subsysnqn'] = ctrl['nqn'] host_iface = NbftConf.__get_host_iface(ctrl.get('hfi_index'), hfis) if host_iface: cid['host-iface'] = host_iface cids.append(cid) return cids @staticmethod def __nbft_subs_to_cids(subsystem, hfis): cids = [] for ctrl in subsystem: cid = { 'transport': ctrl['trtype'], 'traddr': ctrl['traddr'], 'trsvcid': ctrl['trsvcid'], 'subsysnqn': ctrl['subsys_nqn'], 'hdr-digest': ctrl['pdu_header_digest_required'], 'data-digest': ctrl['data_digest_required'], } indexes = ctrl.get('hfi_indexes') if isinstance(indexes, list) and len(indexes) > 0: host_iface = NbftConf.__get_host_iface(indexes[0], hfis) if host_iface: cid['host-iface'] = host_iface cids.append(cid) return cids @staticmethod def __get_host_iface(indx, hfis): if indx is None or indx >= len(hfis): return None mac = hfis[indx].get('mac_addr') if mac is None: return None return iputil.mac2iface(mac) @staticmethod def __uri2cid(uri: str): '''Convert a URI of the form "nvme+tcp://100.71.103.50:8009/" to a Controller ID''' obj = urlparse(uri) return { 'transport': obj.scheme.split('+')[1], 'traddr': obj.hostname, 'trsvcid': str(obj.port), }