Adding upstream version 2.2.1.upstream/2.2.1

Signed-off-by: Benjamin Drung <bdrung@debian.org>

1 files changed, 652 insertions, 0 deletions

diff --git a/doc/stacd.conf.xml b/doc/stacd.conf.xml
new file mode 100644
index 0000000..e8c6a64
--- /dev/null
+++ b/doc/stacd.conf.xml
@@ -0,0 +1,652 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--*-nxml-*-->
+<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd">
+<!--
+    SPDX-License-Identifier: Apache-2.0
+    Copyright (c) 2021, Dell Inc. or its subsidiaries.  All rights reserved.
+-->
+<refentry id="stacd.conf" xmlns:xi="http://www.w3.org/2001/XInclude">
+    <refentryinfo>
+        <title>stacd.conf</title>
+        <productname>nvme-stas</productname>
+
+        <author>
+            <personname>
+                <honorific>Mr</honorific>
+                <firstname>Martin</firstname>
+                <surname>Belanger</surname>
+            </personname>
+
+            <affiliation>
+                <orgname>Dell, Inc.</orgname>
+            </affiliation>
+        </author>
+    </refentryinfo>
+
+    <refmeta>
+        <refentrytitle>stacd.conf</refentrytitle>
+        <manvolnum>5</manvolnum>
+    </refmeta>
+
+    <refnamediv>
+        <refname>stacd.conf</refname>
+        <refpurpose>
+            <citerefentry project="man-pages">
+                <refentrytitle>stacd</refentrytitle>
+                <manvolnum>8</manvolnum>
+            </citerefentry>
+            configuration file
+        </refpurpose>
+    </refnamediv>
+
+    <refsynopsisdiv>
+        <para>
+            <filename>/etc/stas/stacd.conf</filename>
+        </para>
+    </refsynopsisdiv>
+
+    <refsect1>
+        <title>Description</title>
+
+        <para>
+            When <citerefentry project="man-pages"><refentrytitle>stacd</refentrytitle>
+            <manvolnum>8</manvolnum></citerefentry> starts up, it reads its
+            configuration from <filename>stacd.conf</filename>.
+        </para>
+    </refsect1>
+
+    <refsect1>
+        <title>Configuration File Format</title>
+        <para>
+            <filename>stacd.conf</filename> is a plain text file divided into
+            sections, with configuration entries in the style
+            <replaceable>key</replaceable>=<replaceable>value</replaceable>.
+            Spaces immediately before or after the <literal>=</literal> are
+            ignored. Empty lines are ignored as well as lines starting with
+            <literal>#</literal>, which may be used for commenting.
+        </para>
+    </refsect1>
+
+    <refsect1>
+        <title>Options</title>
+
+        <refsect2>
+            <title>[Global] section</title>
+            <para>
+                The following options are available in the
+                <literal>[Global]</literal> section:
+            </para>
+
+            <variablelist>
+                <xi:include href="standard-conf.xml" xpointer="tron"/>
+                <xi:include href="standard-conf.xml" xpointer="hdr-digest"/>
+                <xi:include href="standard-conf.xml" xpointer="data-digest"/>
+                <xi:include href="standard-conf.xml" xpointer="kato"/>
+                <xi:include href="standard-conf.xml" xpointer="ip-family"/>
+
+                <varlistentry>
+                    <term><varname>nr-io-queues=</varname></term>
+
+                    <listitem>
+                        <para>
+                            Takes a value in the range 1...N. Overrides the
+                            default number of I/O queues create by the driver.
+                        </para>
+
+                        <para>Note: This parameter is identical to that provided by nvme-cli.</para>
+                        <para>
+                            Default: Depends on kernel and other run
+                            time factors (e.g. number of CPUs).
+                        </para>
+                    </listitem>
+                </varlistentry>
+
+                <varlistentry>
+                    <term><varname>nr-write-queues=</varname></term>
+
+                    <listitem>
+                        <para>
+                            Takes a value in the range 1...N. Adds additional
+                            queues that will be used for write I/O.
+                        </para>
+
+                        <para>Note: This parameter is identical to that provided by nvme-cli.</para>
+
+                        <para>
+                            Default: Depends on kernel and other run
+                            time factors (e.g. number of CPUs).
+                        </para>
+                    </listitem>
+                </varlistentry>
+
+                <varlistentry>
+                    <term><varname>nr-poll-queues=</varname></term>
+
+                    <listitem>
+                        <para>
+                            Takes a value in the range 1...N. Adds additional
+                            queues that will be used for polling latency
+                            sensitive I/O.
+                        </para>
+
+                        <para>Note: This parameter is identical to that provided by nvme-cli.</para>
+
+                        <para>
+                            Default: Depends on kernel and other run
+                            time factors (e.g. number of CPUs).
+                        </para>
+                    </listitem>
+                </varlistentry>
+
+                <xi:include href="standard-conf.xml" xpointer="queue-size"/>
+                <xi:include href="standard-conf.xml" xpointer="reconnect-delay"/>
+                <xi:include href="standard-conf.xml" xpointer="ctrl-loss-tmo"/>
+                <xi:include href="standard-conf.xml" xpointer="disable-sqflow"/>
+
+                <varlistentry>
+                    <term><varname>ignore-iface=</varname></term>
+                    <listitem>
+                        <para>
+                            Takes a boolean argument. This option controls how
+                            connections with I/O Controllers (IOC) are made.
+                        </para>
+
+                        <para>
+                            There is no guarantee that there will be a route to
+                            reach that IOC. However, we can use the socket
+                            option SO_BINDTODEVICE to force the connection to be
+                            made on a specific interface instead of letting the
+                            routing tables decide where to make the connection.
+                        </para>
+
+                        <para>
+                            This option determines whether <code>stacd</code> will use
+                            SO_BINDTODEVICE to force connections on an interface
+                            or just rely on the routing tables. The default is
+                            to use SO_BINDTODEVICE, in other words, <code>stacd</code> does
+                            not ignore the interface.
+                        </para>
+
+                        <para>
+                            BACKGROUND:
+                            By default, <code>stacd</code> will connect to IOCs on the same
+                            interface that was used to retrieve the discovery
+                            log pages. If stafd discovers a DC on an interface
+                            using mDNS, and stafd connects to that DC and
+                            retrieves the log pages, it is expected that the
+                            storage subsystems listed in the log pages are
+                            reachable on the same interface where the DC was
+                            discovered.
+                        </para>
+
+                        <para>
+                            For example, let's say a DC is discovered on
+                            interface ens102. Then all the subsystems listed in
+                            the log pages retrieved from that DC must be
+                            reachable on interface ens102. If this doesn't work,
+                            for example you cannot "ping -I ens102 [storage-ip]",
+                            then the most likely explanation is that proxy arp
+                            is not enabled on the switch that the host is
+                            connected to on interface ens102. Whatever you do,
+                            resist the temptation to manually set up the routing
+                            tables or to add alternate routes going over a
+                            different interface than the one where the DC is
+                            located. That simply won't work. Make sure proxy arp
+                            is enabled on the switch first.
+                        </para>
+
+                        <para>
+                            Setting routes won't work because, by default, <code>stacd</code>
+                            uses the SO_BINDTODEVICE socket option when it
+                            connects to IOCs. This option is used to force a
+                            socket connection to be made on a specific interface
+                            instead of letting the routing tables decide where
+                            to connect the socket. Even if you were to manually
+                            configure an alternate route on a different interface,
+                            the connections (i.e. host to IOC) will still be
+                            made on the interface where the DC was discovered by
+                            stafd.
+                        </para>
+
+                        <para>
+                            Defaults to <parameter>false</parameter>.
+                        </para>
+                    </listitem>
+                </varlistentry>
+            </variablelist>
+        </refsect2>
+
+        <refsect2>
+            <title>[I/O controller connection management] section</title>
+            <para>
+                Connectivity between hosts and subsystems in a fabric is
+                controlled by Fabric Zoning. Entities that share a common
+                zone (i.e., are zoned together) are allowed to discover each
+                other and establish connections between them. Fabric Zoning is
+                configured on Discovery Controllers (DC). Users can add/remove
+                controllers and/or hosts to/from zones.
+            </para>
+
+            <para>
+                Hosts have no direct knowledge of the Fabric Zoning configuration
+                that is active on a given DC. As a result, if a host is impacted
+                by a Fabric Zoning configuration change, it will be notified of
+                the connectivity configuration change by the DC via Asynchronous
+                Event Notifications (AEN).
+            </para>
+
+            <table frame='all'>
+                <title>List of terms used in this section:</title>
+                <tgroup cols="2" align='left' colsep='1' rowsep='1'>
+                    <thead>
+                        <row>
+                            <entry>Term</entry>
+                            <entry>Description</entry>
+                        </row>
+                    </thead>
+
+                    <tbody>
+                        <row>
+                            <entry>AEN</entry>
+                            <entry>Asynchronous Event Notification. A CQE (Completion Queue Entry) for an Asynchronous Event Request that was previously transmitted by the host to a Discovery Controller. AENs are used by DCs to notify hosts that a change (e.g., a connectivity configuration change) has occurred.</entry>
+                        </row>
+
+                        <row>
+                            <entry>DC</entry>
+                            <entry>Discovery Controller.</entry>
+                        </row>
+
+                        <row>
+                            <entry>DLP</entry>
+                            <entry>Discovery Log Page. A host will issue a Get Log Page command to retrieve the list of controllers it may connect to.</entry>
+                        </row>
+
+                        <row>
+                            <entry>DLPE</entry>
+                            <entry><simpara>
+                                       Discovery Log Page Entry. The response
+                                       to a Get Log Page command contains a list of DLPEs identifying
+                                       each controller that the host is allowed to connect with.
+                                   </simpara><simpara>
+                                       Note that DLPEs may contain both I/O Controllers (IOCs)
+                                       and Discovery Controllers (DCs). DCs listed in DLPEs
+                                       are called referrals. <code>stacd</code> only deals with IOCs.
+                                       Referrals (DCs) are handled by <code>stafd</code>.
+                                   </simpara>
+                            </entry>
+                        </row>
+
+                        <row>
+                            <entry>IOC</entry>
+                            <entry>I/O Controller.</entry>
+                        </row>
+
+                        <row>
+                            <entry>Manual Config</entry>
+                            <entry>Refers to manually adding entries to <filename>stacd.conf</filename> with the <varname>controller=</varname> parameter.</entry>
+                        </row>
+
+                        <row>
+                            <entry>Automatic Config</entry>
+                            <entry>Refers to receiving configuration from a DC as DLPEs</entry>
+                        </row>
+
+                        <row>
+                            <entry>External Config</entry>
+                            <entry>Refers to configuration done outside of the <code>nvme-stas</code> framework, for example using <code>nvme-cli</code> commands</entry>
+                        </row>
+                    </tbody>
+                </tgroup>
+            </table>
+
+
+            <para>
+                DCs notify hosts of connectivity configuration changes by sending
+                AENs indicating a "Discovery Log" change. The host uses these AENs as
+                a trigger to issue a Get Log Page command. The response to this command
+                is used to update the list of DLPEs containing the controllers
+                the host is allowed to access.
+                Upon reception of the current DLPEs, the host will determine
+                whether DLPEs were added and/or removed, which will trigger the
+                addition and/or removal of controller connections. This happens in real time
+                and may affect active connections to controllers including controllers
+                that support I/O operations (IOCs). A host that was previously
+                connected to an IOC may suddenly be told that it is no longer
+                allowed to connect to that IOC and should disconnect from it.
+            </para>
+
+            <formalpara><title>IOC connection creation</title>
+                <para>
+                    There are 3 ways to configure IOC connections on a host:
+                </para>
+
+                <orderedlist>
+                    <listitem>
+                        <para>
+                            Manual Config by adding <varname>controller=</varname> entries
+                            to the <literal>[Controllers]</literal> section (see below).
+                        </para>
+                    </listitem>
+                    <listitem>
+                        <para>
+                            Automatic Config received in the form of
+                            DLPEs from a remote DC.
+                        </para>
+                    </listitem>
+                    <listitem>
+                        <para>
+                            External Config using <code>nvme-cli</code> (e.g. "<code>nvme connect</code>")
+                        </para>
+                    </listitem>
+                </orderedlist>
+            </formalpara>
+
+            <formalpara><title>IOC connection removal/prevention</title>
+                <para>
+                    There are 3 ways to remove (or prevent) connections to an IOC:
+                </para>
+
+                <orderedlist>
+                    <listitem>
+                        <para>
+                            Manual Config.
+                            <orderedlist numeration='lowerroman'>
+                                <listitem>
+                                    <para>
+                                        by adding <varname>exclude=</varname> entries to
+                                        the <literal>[Controllers]</literal> section (see below).
+                                    </para>
+                                </listitem>
+                                <listitem>
+                                    <para>
+                                        by removing <varname>controller=</varname> entries
+                                        from the <literal>[Controllers]</literal> section.
+                                    </para>
+                                </listitem>
+                            </orderedlist>
+                        </para>
+                    </listitem>
+                    <listitem>
+                        <para>
+                            Automatic Config. As explained above, a host gets a
+                            new list of DLPEs upon connectivity configuration
+                            changes. On DLPE removal, the host should remove the
+                            connection to the IOC matching that DLPE. This
+                            behavior is configurable using the
+                            <varname>disconnect-scope=</varname> parameter
+                            described below.
+                        </para>
+                    </listitem>
+                    <listitem>
+                        <para>
+                            External Config using <code>nvme-cli</code> (e.g. "<code>nvme
+                            disconnect</code>" or "<code>nvme disconnect-all</code>")
+                        </para>
+                    </listitem>
+                </orderedlist>
+            </formalpara>
+
+            <para>
+                The decision by the host to automatically disconnect from an
+                IOC following connectivity configuration changes is controlled
+                by two parameters: <code>disconnect-scope</code>
+                and <code>disconnect-trtypes</code>.
+            </para>
+
+            <variablelist>
+                <varlistentry>
+                    <term><varname>disconnect-scope=</varname></term>
+                    <listitem>
+                        <para>
+                            Takes one of: <parameter>only-stas-connections</parameter>,
+                            <parameter>all-connections-matching-disconnect-trtypes</parameter>, or <parameter>no-disconnect</parameter>.
+                        </para>
+
+                        <para>
+                            In theory, hosts should only connect to IOCs that have
+                            been zoned for them. Connections to IOCs that a host
+                            is not zoned to have access to should simply not exist.
+                            In practice, however, users may not want hosts to
+                            disconnect from all IOCs in reaction to connectivity
+                            configuration changes (or at least for some of the IOC
+                            connections).
+                        </para>
+
+                        <para>
+                            Some users may prefer for IOC connections to be "sticky" 
+                            and only be removed manually (<code>nvme-cli</code> or 
+                            <varname>exclude=</varname>) or removed by a system 
+                            reboot. Specifically, they don't want IOC connections 
+                            to be removed unexpectedly on DLPE removal. These users 
+                            may want to set <varname>disconnect-scope</varname>
+                            to <parameter>no-disconnect</parameter>.
+                        </para>
+
+                        <para>
+                            It is important to note that when IOC connections 
+                            are removed, ongoing I/O transactions will be 
+                            terminated immediately. There is no way to tell what
+                            happens to the data being exchanged when such an abrupt
+                            termination happens. If a host was in the middle of writing
+                            to a storage subsystem, there is a chance that outstanding
+                            I/O operations may not successfully complete.
+                        </para>
+
+                        <refsect3>
+                            <title>Values:</title>
+                            <variablelist>
+                                <varlistentry>
+                                    <term><parameter>only-stas-connections</parameter></term>
+                                    <listitem>
+                                        <para>
+                                            Only remove connections previously made by <code>stacd</code>.
+                                        </para>
+                                        <para>
+                                            In this mode, when a DLPE is removed as a result of
+                                            connectivity configuration changes, the corresponding
+                                            IOC connection will be removed by <code>stacd</code>.
+                                        </para>
+                                        <para>
+                                            Connections to IOCs made externally, e.g. using <code>nvme-cli</code>,
+                                            will not be affected, unless they happen to be duplicates
+                                            of connections made by <code>stacd</code>. It's simply not
+                                            possible for <code>stacd</code> to tell that a connection
+                                            was previously made with <code>nvme-cli</code> (or any other external tool).
+                                            So, it's good practice to avoid duplicating
+                                            configuration between <code>stacd</code> and external tools.
+                                        </para>
+                                        <para>
+                                            Users wanting to persist some of their IOC connections
+                                            regardless of connectivity configuration changes should not use
+                                            <code>nvme-cli</code> to make those connections. Instead, 
+                                            they should hard-code them in <filename>stacd.conf</filename>
+                                            with the <varname>controller=</varname> parameter. Using the
+                                            <varname>controller=</varname> parameter is the only way for a user
+                                            to tell <code>stacd</code> that a connection must be made and
+                                            not be deleted "<emphasis>no-matter-what</emphasis>".
+                                        </para>
+                                    </listitem>
+                                </varlistentry>
+
+                                <varlistentry>
+                                    <term><parameter>all-connections-matching-disconnect-trtypes</parameter></term>
+                                    <listitem>
+                                        <para>
+                                            All connections that match the transport type specified by 
+                                            <varname>disconnect-trtypes=</varname>, whether they were 
+                                            made automatically by <code>stacd</code> or externally 
+                                            (e.g., <code>nvme-cli</code>), will be audited and are 
+                                            subject to removal on DLPE removal.
+                                        </para>
+                                        <para>
+                                            In this mode, as DLPEs are removed as a result of
+                                            connectivity configuration changes, the corresponding
+                                            IOC connections will be removed by the host immediately
+                                            whether they were made by <code>stacd</code>, <code>nvme-cli</code>,
+                                            or any other way. Basically, <code>stacd</code> audits 
+                                            <emphasis>all</emphasis> IOC connections matching the 
+                                            transport type specified by <varname>disconnect-trtypes=</varname>.
+                                        </para>
+                                        <formalpara><title><emphasis>NOTE</emphasis></title>
+                                            <para>
+                                                This mode implies that <code>stacd</code> will
+                                                only allow Manually Configured or Automatically
+                                                Configured IOC connections to exist. Externally
+                                                Configured connections using <code>nvme-cli</code>
+                                                (or other external mechanism)
+                                                that do not match any Manual Config
+                                                (<filename>stacd.conf</filename>)
+                                                or Automatic Config (DLPEs) will get deleted
+                                                immediately by <code>stacd</code>.
+                                            </para>
+                                        </formalpara>
+                                    </listitem>
+                                </varlistentry>
+
+                                <varlistentry>
+                                    <term><parameter>no-disconnect</parameter></term>
+                                    <listitem>
+                                        <para>
+                                            <code>stacd</code> does not disconnect from IOCs 
+                                            when a DPLE is removed or a <varname>controller=</varname> 
+                                            entry is removed from <filename>stacd.conf</filename>.
+                                            All IOC connections are "sticky".
+                                        </para>
+
+                                        <para>
+                                            Instead, users can remove connections 
+                                            by issuing the <code>nvme-cli</code>
+                                            command "<code>nvme disconnect</code>", add an
+                                            <varname>exclude=</varname> entry to
+                                            <filename>stacd.conf</filename>, or wait
+                                            until the next system reboot at which time all
+                                            connections will be removed.
+                                        </para>
+                                    </listitem>
+                                </varlistentry>
+                            </variablelist>
+                        </refsect3>
+
+                        <para>
+                            Defaults to <parameter>only-stas-connections</parameter>.
+                        </para>
+                    </listitem>
+                </varlistentry>
+
+                <varlistentry>
+                    <term><varname>disconnect-trtypes=</varname></term>
+                    <listitem>
+                        <para>
+                            This parameter only applies when <varname>disconnect-scope</varname>
+                            is set to <parameter>all-connections-matching-disconnect-trtypes</parameter>. 
+                            It limits the scope of the audit to specific transport types.
+                        </para>
+
+                        <para>
+                            Can take the values <parameter>tcp</parameter>,
+                            <parameter>rdma</parameter>, <parameter>fc</parameter>, or
+                            a combination thereof by separating them with a plus (+) sign.
+                            For example: <parameter>tcp+fc</parameter>. No spaces
+                            are allowed between values and the plus (+) sign.
+                        </para>
+
+                        <refsect3>
+                            <title>Values:</title>
+                            <variablelist>
+                                <varlistentry>
+                                    <term><parameter>tcp</parameter></term>
+                                    <listitem>
+                                        <para>
+                                            Audit TCP connections.
+                                        </para>
+                                    </listitem>
+                                </varlistentry>
+                                <varlistentry>
+                                    <term><parameter>rdma</parameter></term>
+                                    <listitem>
+                                        <para>
+                                            Audit RDMA connections.
+                                        </para>
+                                    </listitem>
+                                </varlistentry>
+                                <varlistentry>
+                                    <term><parameter>fc</parameter></term>
+                                    <listitem>
+                                        <para>
+                                            Audit Fibre Channel connections.
+                                        </para>
+                                    </listitem>
+                                </varlistentry>
+                            </variablelist>
+                        </refsect3>
+
+                        <para>
+                            Defaults to <parameter>tcp</parameter>.
+                        </para>
+                    </listitem>
+                </varlistentry>
+
+                <varlistentry>
+                    <term><varname>connect-attempts-on-ncc=</varname></term>
+                    <listitem>
+                        <para>
+                            The NCC bit (Not Connected to CDC) is a bit returned
+                            by the CDC in the EFLAGS field of the DLPE. Only CDCs
+                            will set the NCC bit. DDCs will always clear NCC to
+                            0. The NCC bit is a way for the CDC to let hosts
+                            know that the subsystem is currently not reachable
+                            by the CDC. This may indicate that the subsystem is
+                            currently down or that there is an outage on the
+                            section of the network connecting the CDC to the
+                            subsystem.
+                        </para>
+
+                        <para>
+                            If a host is currently failing to connect to an I/O
+                            controller and if the NCC bit associated with that
+                            I/O controller is asserted, the host can decide to
+                            stop trying to connect to that subsystem until
+                            connectivity is restored. This will be indicated by
+                            the CDC when it clears the NCC bit.
+                        </para>
+
+                        <para>
+                            The parameter <varname>connect-attempts-on-ncc=</varname>
+                            controls whether <code>stacd</code> will take the
+                            NCC bit into account when attempting to connect to
+                            an I/O Controller. Setting <varname>connect-attempts-on-ncc=</varname>
+                            to 0 means that <code>stacd</code> will ignore
+                            the NCC bit and will keep trying to connect. Setting
+                            <varname>connect-attempts-on-ncc=</varname> to a
+                            non-zero value indicates the number of connection
+                            attempts that will be made before <code>stacd</code>
+                            gives up trying. Note that this value should be set
+                            to a value greater than 1. In fact, when set to 1,
+                            <code>stacd</code> will automatically use 2 instead.
+                            The reason for this is simple. It is possible that a
+                            first connect attempt may fail.
+                        </para>
+
+
+                        <para>
+                            Defaults to <parameter>0</parameter>.
+                        </para>
+                    </listitem>
+                </varlistentry>
+            </variablelist>
+        </refsect2>
+
+        <xi:include href="standard-conf.xml" xpointer="controller"/>
+    </refsect1>
+
+    <refsect1>
+        <title>See Also</title>
+        <para>
+            <citerefentry>
+                <refentrytitle>stacd</refentrytitle>
+                <manvolnum>8</manvolnum>
+            </citerefentry>
+        </para>
+    </refsect1>
+</refentry>