Adding upstream version 2.2.1.upstream/2.2.1

Signed-off-by: Benjamin Drung <bdrung@debian.org>

1 files changed, 703 insertions, 0 deletions

diff --git a/staslib/conf.py b/staslib/conf.py
new file mode 100644
index 0000000..a54da98
--- /dev/null
+++ b/staslib/conf.py
@@ -0,0 +1,703 @@
+# Copyright (c) 2022, Dell Inc. or its subsidiaries.  All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+# See the LICENSE file for details.
+#
+# This file is part of NVMe STorage Appliance Services (nvme-stas).
+#
+# Authors: Martin Belanger <Martin.Belanger@dell.com>
+#
+'''nvme-stas configuration module'''
+
+import re
+import os
+import sys
+import logging
+import functools
+import configparser
+from staslib import defs, singleton, timeparse
+
+__TOKEN_RE = re.compile(r'\s*;\s*')
+__OPTION_RE = re.compile(r'\s*=\s*')
+
+
+class InvalidOption(Exception):
+    '''Exception raised when an invalid option value is detected'''
+
+
+def _parse_controller(controller):
+    '''@brief Parse a "controller" entry. Controller entries are strings
+           composed of several configuration parameters delimited by
+           semi-colons. Each configuration parameter is specified as a
+           "key=value" pair.
+    @return A dictionary of key-value pairs.
+    '''
+    options = dict()
+    tokens = __TOKEN_RE.split(controller)
+    for token in tokens:
+        if token:
+            try:
+                option, val = __OPTION_RE.split(token)
+                options[option.strip()] = val.strip()
+            except ValueError:
+                pass
+
+    return options
+
+
+def _parse_single_val(text):
+    if isinstance(text, str):
+        return text
+    if not isinstance(text, list) or len(text) == 0:
+        return None
+
+    return text[-1]
+
+
+def _parse_list(text):
+    return text if isinstance(text, list) else [text]
+
+
+def _to_int(text):
+    try:
+        return int(_parse_single_val(text))
+    except (ValueError, TypeError):
+        raise InvalidOption  # pylint: disable=raise-missing-from
+
+
+def _to_bool(text, positive='true'):
+    return _parse_single_val(text).lower() == positive
+
+
+def _to_ncc(text):
+    value = _to_int(text)
+    if value == 1:  # 1 is invalid. A minimum of 2 is required (with the exception of 0, which is valid).
+        value = 2
+    return value
+
+
+def _to_ip_family(text):
+    return tuple((4 if text == 'ipv4' else 6 for text in _parse_single_val(text).split('+')))
+
+
+# ******************************************************************************
+class OrderedMultisetDict(dict):
+    '''This class is used to change the behavior of configparser.ConfigParser
+    and allow multiple configuration parameters with the same key. The
+    result is a list of values.
+    '''
+
+    def __setitem__(self, key, value):
+        if key in self and isinstance(value, list):
+            self[key].extend(value)
+        else:
+            super().__setitem__(key, value)
+
+    def __getitem__(self, key):
+        value = super().__getitem__(key)
+
+        if isinstance(value, str):
+            return value.split('\n')
+
+        return value
+
+
+class SvcConf(metaclass=singleton.Singleton):  # pylint: disable=too-many-public-methods
+    '''Read and cache configuration file.'''
+
+    OPTION_CHECKER = {
+        'Global': {
+            'tron': {
+                'convert': _to_bool,
+                'default': False,
+                'txt-chk': lambda text: _parse_single_val(text).lower() in ('false', 'true'),
+            },
+            'kato': {
+                'convert': _to_int,
+            },
+            'pleo': {
+                'convert': functools.partial(_to_bool, positive='enabled'),
+                'default': True,
+                'txt-chk': lambda text: _parse_single_val(text).lower() in ('disabled', 'enabled'),
+            },
+            'ip-family': {
+                'convert': _to_ip_family,
+                'default': (4, 6),
+                'txt-chk': lambda text: _parse_single_val(text) in ('ipv4', 'ipv6', 'ipv4+ipv6', 'ipv6+ipv4'),
+            },
+            'queue-size': {
+                'convert': _to_int,
+                'rng-chk': lambda value: None if value in range(16, 1025) else range(16, 1025),
+            },
+            'hdr-digest': {
+                'convert': _to_bool,
+                'default': False,
+                'txt-chk': lambda text: _parse_single_val(text).lower() in ('false', 'true'),
+            },
+            'data-digest': {
+                'convert': _to_bool,
+                'default': False,
+                'txt-chk': lambda text: _parse_single_val(text).lower() in ('false', 'true'),
+            },
+            'ignore-iface': {
+                'convert': _to_bool,
+                'default': False,
+                'txt-chk': lambda text: _parse_single_val(text).lower() in ('false', 'true'),
+            },
+            'nr-io-queues': {
+                'convert': _to_int,
+            },
+            'ctrl-loss-tmo': {
+                'convert': _to_int,
+            },
+            'disable-sqflow': {
+                'convert': _to_bool,
+                'txt-chk': lambda text: _parse_single_val(text).lower() in ('false', 'true'),
+            },
+            'nr-poll-queues': {
+                'convert': _to_int,
+            },
+            'nr-write-queues': {
+                'convert': _to_int,
+            },
+            'reconnect-delay': {
+                'convert': _to_int,
+            },
+            ### BEGIN: LEGACY SECTION TO BE REMOVED ###
+            'persistent-connections': {
+                'convert': _to_bool,
+                'default': False,
+                'txt-chk': lambda text: _parse_single_val(text).lower() in ('false', 'true'),
+            },
+            ### END: LEGACY SECTION TO BE REMOVED ###
+        },
+        'Service Discovery': {
+            'zeroconf': {
+                'convert': functools.partial(_to_bool, positive='enabled'),
+                'default': True,
+                'txt-chk': lambda text: _parse_single_val(text).lower() in ('disabled', 'enabled'),
+            },
+        },
+        'Discovery controller connection management': {
+            'persistent-connections': {
+                'convert': _to_bool,
+                'default': True,
+                'txt-chk': lambda text: _parse_single_val(text).lower() in ('false', 'true'),
+            },
+            'zeroconf-connections-persistence': {
+                'convert': lambda text: timeparse.timeparse(_parse_single_val(text)),
+                'default': timeparse.timeparse('72hours'),
+            },
+        },
+        'I/O controller connection management': {
+            'disconnect-scope': {
+                'convert': _parse_single_val,
+                'default': 'only-stas-connections',
+                'txt-chk': lambda text: _parse_single_val(text)
+                in ('only-stas-connections', 'all-connections-matching-disconnect-trtypes', 'no-disconnect'),
+            },
+            'disconnect-trtypes': {
+                # Use set() to eliminate potential duplicates
+                'convert': lambda text: set(_parse_single_val(text).split('+')),
+                'default': [
+                    'tcp',
+                ],
+                'lst-chk': ('tcp', 'rdma', 'fc'),
+            },
+            'connect-attempts-on-ncc': {
+                'convert': _to_ncc,
+                'default': 0,
+            },
+        },
+        'Controllers': {
+            'controller': {
+                'convert': _parse_list,
+                'default': [],
+            },
+            'exclude': {
+                'convert': _parse_list,
+                'default': [],
+            },
+            ### BEGIN: LEGACY SECTION TO BE REMOVED ###
+            'blacklist': {
+                'convert': _parse_list,
+                'default': [],
+            },
+            ### END: LEGACY SECTION TO BE REMOVED ###
+        },
+    }
+
+    def __init__(self, default_conf=None, conf_file='/dev/null'):
+        self._config = None
+        self._defaults = default_conf if default_conf else {}
+
+        if self._defaults is not None and len(self._defaults) != 0:
+            self._valid_conf = {}
+            for section, option in self._defaults:
+                self._valid_conf.setdefault(section, set()).add(option)
+        else:
+            self._valid_conf = None
+
+        self._conf_file = conf_file
+        self.reload()
+
+    def reload(self):
+        '''@brief Reload the configuration file.'''
+        self._config = self._read_conf_file()
+
+    @property
+    def conf_file(self):
+        '''Return the configuration file name'''
+        return self._conf_file
+
+    def set_conf_file(self, fname):
+        '''Set the configuration file name and reload config'''
+        self._conf_file = fname
+        self.reload()
+
+    def get_option(self, section, option, ignore_default=False):  # pylint: disable=too-many-locals
+        '''Retrieve @option from @section, convert raw text to
+        appropriate object type, and validate.'''
+        try:
+            checker = self.OPTION_CHECKER[section][option]
+        except KeyError:
+            logging.error('Requesting invalid section=%s and/or option=%s', section, option)
+            raise
+
+        default = checker.get('default', None)
+
+        try:
+            text = self._config.get(section=section, option=option)
+        except (configparser.NoSectionError, configparser.NoOptionError, KeyError):
+            return None if ignore_default else self._defaults.get((section, option), default)
+
+        return self._check(text, section, option, default)
+
+    tron = property(functools.partial(get_option, section='Global', option='tron'))
+    kato = property(functools.partial(get_option, section='Global', option='kato'))
+    ip_family = property(functools.partial(get_option, section='Global', option='ip-family'))
+    queue_size = property(functools.partial(get_option, section='Global', option='queue-size'))
+    hdr_digest = property(functools.partial(get_option, section='Global', option='hdr-digest'))
+    data_digest = property(functools.partial(get_option, section='Global', option='data-digest'))
+    ignore_iface = property(functools.partial(get_option, section='Global', option='ignore-iface'))
+    pleo_enabled = property(functools.partial(get_option, section='Global', option='pleo'))
+    nr_io_queues = property(functools.partial(get_option, section='Global', option='nr-io-queues'))
+    ctrl_loss_tmo = property(functools.partial(get_option, section='Global', option='ctrl-loss-tmo'))
+    disable_sqflow = property(functools.partial(get_option, section='Global', option='disable-sqflow'))
+    nr_poll_queues = property(functools.partial(get_option, section='Global', option='nr-poll-queues'))
+    nr_write_queues = property(functools.partial(get_option, section='Global', option='nr-write-queues'))
+    reconnect_delay = property(functools.partial(get_option, section='Global', option='reconnect-delay'))
+
+    zeroconf_enabled = property(functools.partial(get_option, section='Service Discovery', option='zeroconf'))
+
+    zeroconf_persistence_sec = property(
+        functools.partial(
+            get_option, section='Discovery controller connection management', option='zeroconf-connections-persistence'
+        )
+    )
+
+    disconnect_scope = property(
+        functools.partial(get_option, section='I/O controller connection management', option='disconnect-scope')
+    )
+    disconnect_trtypes = property(
+        functools.partial(get_option, section='I/O controller connection management', option='disconnect-trtypes')
+    )
+    connect_attempts_on_ncc = property(
+        functools.partial(get_option, section='I/O controller connection management', option='connect-attempts-on-ncc')
+    )
+
+    @property
+    def stypes(self):
+        '''@brief Get the DNS-SD/mDNS service types.'''
+        return ['_nvme-disc._tcp', '_nvme-disc._udp'] if self.zeroconf_enabled else list()
+
+    @property
+    def persistent_connections(self):
+        '''@brief return the "persistent-connections" config parameter'''
+        section = 'Discovery controller connection management'
+        option = 'persistent-connections'
+
+        value = self.get_option(section, option, ignore_default=True)
+        legacy = self.get_option('Global', 'persistent-connections', ignore_default=True)
+
+        if value is None and legacy is None:
+            return self._defaults.get((section, option), True)
+
+        return value or legacy
+
+    def get_controllers(self):
+        '''@brief Return the list of controllers in the config file.
+        Each controller is in the form of a dictionary as follows.
+        Note that some of the keys are optional.
+        {
+            'transport':          [TRANSPORT],
+            'traddr':             [TRADDR],
+            'trsvcid':            [TRSVCID],
+            'host-traddr':        [TRADDR],
+            'host-iface':         [IFACE],
+            'subsysnqn':          [NQN],
+            'dhchap-ctrl-secret': [KEY],
+            'hdr-digest':         [BOOL]
+            'data-digest':        [BOOL]
+            'nr-io-queues':       [NUMBER]
+            'nr-write-queues':    [NUMBER]
+            'nr-poll-queues':     [NUMBER]
+            'queue-size':         [SIZE]
+            'kato':               [KATO]
+            'reconnect-delay':    [SECONDS]
+            'ctrl-loss-tmo':      [SECONDS]
+            'disable-sqflow':     [BOOL]
+        }
+        '''
+        controller_list = self.get_option('Controllers', 'controller')
+        cids = [_parse_controller(controller) for controller in controller_list]
+        for cid in cids:
+            try:
+                # replace 'nqn' key by 'subsysnqn', if present.
+                cid['subsysnqn'] = cid.pop('nqn')
+            except KeyError:
+                pass
+
+            # Verify values of the options used to overload the matching [Global] options
+            for option in cid:
+                if option in self.OPTION_CHECKER['Global']:
+                    value = self._check(cid[option], 'Global', option, None)
+                    if value is not None:
+                        cid[option] = value
+
+        return cids
+
+    def get_excluded(self):
+        '''@brief Return the list of excluded controllers in the config file.
+        Each excluded controller is in the form of a dictionary
+        as follows. All the keys are optional.
+        {
+            'transport':  [TRANSPORT],
+            'traddr':     [TRADDR],
+            'trsvcid':    [TRSVCID],
+            'host-iface': [IFACE],
+            'subsysnqn':  [NQN],
+        }
+        '''
+        controller_list = self.get_option('Controllers', 'exclude')
+
+        # 2022-09-20: Look for "blacklist". This is for backwards compatibility
+        # with releases 1.0 to 1.1.6. This is to be phased out (i.e. remove by 2024)
+        controller_list += self.get_option('Controllers', 'blacklist')
+
+        excluded = [_parse_controller(controller) for controller in controller_list]
+        for controller in excluded:
+            controller.pop('host-traddr', None)  # remove host-traddr
+            try:
+                # replace 'nqn' key by 'subsysnqn', if present.
+                controller['subsysnqn'] = controller.pop('nqn')
+            except KeyError:
+                pass
+        return excluded
+
+    def _check(self, text, section, option, default):
+        checker = self.OPTION_CHECKER[section][option]
+        text_checker = checker.get('txt-chk', None)
+        if text_checker is not None and not text_checker(text):
+            logging.warning(
+                'File:%s [%s]: %s - Text check found invalid value "%s". Default will be used',
+                self.conf_file,
+                section,
+                option,
+                text,
+            )
+            return self._defaults.get((section, option), default)
+
+        converter = checker.get('convert', None)
+        try:
+            value = converter(text)
+        except InvalidOption:
+            logging.warning(
+                'File:%s [%s]: %s - Data converter found invalid value "%s". Default will be used',
+                self.conf_file,
+                section,
+                option,
+                text,
+            )
+            return self._defaults.get((section, option), default)
+
+        value_in_range = checker.get('rng-chk', None)
+        if value_in_range is not None:
+            expected_range = value_in_range(value)
+            if expected_range is not None:
+                logging.warning(
+                    'File:%s [%s]: %s - "%s" is not within range %s..%s. Default will be used',
+                    self.conf_file,
+                    section,
+                    option,
+                    value,
+                    min(expected_range),
+                    max(expected_range),
+                )
+                return self._defaults.get((section, option), default)
+
+        list_checker = checker.get('lst-chk', None)
+        if list_checker:
+            values = set()
+            for item in value:
+                if item not in list_checker:
+                    logging.warning(
+                        'File:%s [%s]: %s - List checker found invalid item "%s" will be ignored.',
+                        self.conf_file,
+                        section,
+                        option,
+                        item,
+                    )
+                else:
+                    values.add(item)
+
+            if len(values) == 0:
+                return self._defaults.get((section, option), default)
+
+            value = list(values)
+
+        return value
+
+    def _read_conf_file(self):
+        '''@brief Read the configuration file if the file exists.'''
+        config = configparser.ConfigParser(
+            default_section=None,
+            allow_no_value=True,
+            delimiters=('='),
+            interpolation=None,
+            strict=False,
+            dict_type=OrderedMultisetDict,
+        )
+        if self._conf_file and os.path.isfile(self._conf_file):
+            config.read(self._conf_file)
+
+        # Parse Configuration and validate.
+        if self._valid_conf is not None:
+            invalid_sections = set()
+            for section in config.sections():
+                if section not in self._valid_conf:
+                    invalid_sections.add(section)
+                else:
+                    invalid_options = set()
+                    for option in config.options(section):
+                        if option not in self._valid_conf.get(section, []):
+                            invalid_options.add(option)
+
+                    if len(invalid_options) != 0:
+                        logging.error(
+                            'File:%s [%s] contains invalid options: %s',
+                            self.conf_file,
+                            section,
+                            invalid_options,
+                        )
+
+            if len(invalid_sections) != 0:
+                logging.error(
+                    'File:%s contains invalid sections: %s',
+                    self.conf_file,
+                    invalid_sections,
+                )
+
+        return config
+
+
+# ******************************************************************************
+class SysConf(metaclass=singleton.Singleton):
+    '''Read and cache the host configuration file.'''
+
+    def __init__(self, conf_file=defs.SYS_CONF_FILE):
+        self._config = None
+        self._conf_file = conf_file
+        self.reload()
+
+    def reload(self):
+        '''@brief Reload the configuration file.'''
+        self._config = self._read_conf_file()
+
+    @property
+    def conf_file(self):
+        '''Return the configuration file name'''
+        return self._conf_file
+
+    def set_conf_file(self, fname):
+        '''Set the configuration file name and reload config'''
+        self._conf_file = fname
+        self.reload()
+
+    def as_dict(self):
+        '''Return configuration as a dictionary'''
+        return {
+            'hostnqn': self.hostnqn,
+            'hostid': self.hostid,
+            'hostkey': self.hostkey,
+            'symname': self.hostsymname,
+        }
+
+    @property
+    def hostnqn(self):
+        '''@brief return the host NQN
+        @return: Host NQN
+        @raise: Host NQN is mandatory. The program will terminate if a
+                Host NQN cannot be determined.
+        '''
+        try:
+            value = self.__get_value('Host', 'nqn', defs.NVME_HOSTNQN)
+        except FileNotFoundError as ex:
+            sys.exit(f'Error reading mandatory Host NQN (see stasadm --help): {ex}')
+
+        if value is not None and not value.startswith('nqn.'):
+            sys.exit(f'Error Host NQN "{value}" should start with "nqn."')
+
+        return value
+
+    @property
+    def hostid(self):
+        '''@brief return the host ID
+        @return: Host ID
+        @raise: Host ID is mandatory. The program will terminate if a
+                Host ID cannot be determined.
+        '''
+        try:
+            value = self.__get_value('Host', 'id', defs.NVME_HOSTID)
+        except FileNotFoundError as ex:
+            sys.exit(f'Error reading mandatory Host ID (see stasadm --help): {ex}')
+
+        return value
+
+    @property
+    def hostkey(self):
+        '''@brief return the host key
+        @return: Host key
+        @raise: Host key is optional, but mandatory if authorization will be performed.
+        '''
+        try:
+            value = self.__get_value('Host', 'key', defs.NVME_HOSTKEY)
+        except FileNotFoundError as ex:
+            logging.info('Host key undefined: %s', ex)
+            value = None
+
+        return value
+
+    @property
+    def hostsymname(self):
+        '''@brief return the host symbolic name (or None)
+        @return: symbolic name or None
+        '''
+        try:
+            value = self.__get_value('Host', 'symname')
+        except FileNotFoundError as ex:
+            logging.warning('Error reading host symbolic name (will remain undefined): %s', ex)
+            value = None
+
+        return value
+
+    def _read_conf_file(self):
+        '''@brief Read the configuration file if the file exists.'''
+        config = configparser.ConfigParser(
+            default_section=None, allow_no_value=True, delimiters=('='), interpolation=None, strict=False
+        )
+        if os.path.isfile(self._conf_file):
+            config.read(self._conf_file)
+        return config
+
+    def __get_value(self, section, option, default_file=None):
+        '''@brief A configuration file consists of sections, each led by a
+               [section] header, followed by key/value entries separated
+               by a equal sign (=). This method retrieves the value
+               associated with the key @option from the section @section.
+               If the value starts with the string "file://", then the value
+               will be retrieved from that file.
+
+        @param section:      Configuration section
+        @param option:       The key to look for
+        @param default_file: A file that contains the default value
+
+        @return: On success, the value associated with the key. On failure,
+                 this method will return None is a default_file is not
+                 specified, or will raise an exception if a file is not
+                 found.
+
+        @raise: This method will raise the FileNotFoundError exception if
+                the value retrieved is a file that does not exist.
+        '''
+        try:
+            value = self._config.get(section=section, option=option)
+            if not value.startswith('file://'):
+                return value
+            file = value[7:]
+        except (configparser.NoSectionError, configparser.NoOptionError, KeyError):
+            if default_file is None:
+                return None
+            file = default_file
+
+        try:
+            with open(file) as f:  # pylint: disable=unspecified-encoding
+                return f.readline().split()[0]
+        except IndexError:
+            return None
+
+
+# ******************************************************************************
+class NvmeOptions(metaclass=singleton.Singleton):
+    '''Object used to read and cache contents of file /dev/nvme-fabrics.
+    Note that this file was not readable prior to Linux 5.16.
+    '''
+
+    def __init__(self):
+        # Supported options can be determined by looking at the kernel version
+        # or by reading '/dev/nvme-fabrics'. The ability to read the options
+        # from '/dev/nvme-fabrics' was only introduced in kernel 5.17, but may
+        # have been backported to older kernels. In any case, if the kernel
+        # version meets the minimum version for that option, then we don't
+        # even need to read '/dev/nvme-fabrics'.
+        self._supported_options = {
+            'discovery': defs.KERNEL_VERSION >= defs.KERNEL_TP8013_MIN_VERSION,
+            'host_iface': defs.KERNEL_VERSION >= defs.KERNEL_IFACE_MIN_VERSION,
+            'dhchap_secret': defs.KERNEL_VERSION >= defs.KERNEL_HOSTKEY_MIN_VERSION,
+            'dhchap_ctrl_secret': defs.KERNEL_VERSION >= defs.KERNEL_CTRLKEY_MIN_VERSION,
+        }
+
+        # If some of the options are False, we need to check wether they can be
+        # read from '/dev/nvme-fabrics'. This method allows us to determine that
+        # an older kernel actually supports a specific option because it was
+        # backported to that kernel.
+        if not all(self._supported_options.values()):  # At least one option is False.
+            try:
+                with open('/dev/nvme-fabrics') as f:  # pylint: disable=unspecified-encoding
+                    options = [option.split('=')[0].strip() for option in f.readline().rstrip('\n').split(',')]
+            except PermissionError:  # Must be root to read this file
+                raise
+            except (OSError, FileNotFoundError):
+                logging.warning('Cannot determine which NVMe options the kernel supports')
+            else:
+                for option, supported in self._supported_options.items():
+                    if not supported:
+                        self._supported_options[option] = option in options
+
+    def __str__(self):
+        return f'supported options: {self._supported_options}'
+
+    def get(self):
+        '''get the supported options as a dict'''
+        return self._supported_options
+
+    @property
+    def discovery_supp(self):
+        '''This option adds support for TP8013'''
+        return self._supported_options['discovery']
+
+    @property
+    def host_iface_supp(self):
+        '''This option allows forcing connections to go over
+        a specific interface regardless of the routing tables.
+        '''
+        return self._supported_options['host_iface']
+
+    @property
+    def dhchap_hostkey_supp(self):
+        '''This option allows specifying the host DHCHAP key used for authentication.'''
+        return self._supported_options['dhchap_secret']
+
+    @property
+    def dhchap_ctrlkey_supp(self):
+        '''This option allows specifying the controller DHCHAP key used for authentication.'''
+        return self._supported_options['dhchap_ctrl_secret']