summaryrefslogtreecommitdiffstats
path: root/doc/stacd.conf.xml
diff options
context:
space:
mode:
authorBenjamin Drung <bdrung@debian.org>2023-06-10 08:55:33 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2023-06-10 09:21:49 +0000
commit88837172f69eabc408ae3945d82e0270b8e07440 (patch)
treed6b7fa06694f45d25f54f6ea9ded93c981e51f6f /doc/stacd.conf.xml
parentInitial commit. (diff)
downloadnvme-stas-88837172f69eabc408ae3945d82e0270b8e07440.tar.xz
nvme-stas-88837172f69eabc408ae3945d82e0270b8e07440.zip
Adding upstream version 2.2.1.upstream/2.2.1
Signed-off-by: Benjamin Drung <bdrung@debian.org>
Diffstat (limited to 'doc/stacd.conf.xml')
-rw-r--r--doc/stacd.conf.xml652
1 files changed, 652 insertions, 0 deletions
diff --git a/doc/stacd.conf.xml b/doc/stacd.conf.xml
new file mode 100644
index 0000000..e8c6a64
--- /dev/null
+++ b/doc/stacd.conf.xml
@@ -0,0 +1,652 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--*-nxml-*-->
+<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd">
+<!--
+ SPDX-License-Identifier: Apache-2.0
+ Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved.
+-->
+<refentry id="stacd.conf" xmlns:xi="http://www.w3.org/2001/XInclude">
+ <refentryinfo>
+ <title>stacd.conf</title>
+ <productname>nvme-stas</productname>
+
+ <author>
+ <personname>
+ <honorific>Mr</honorific>
+ <firstname>Martin</firstname>
+ <surname>Belanger</surname>
+ </personname>
+
+ <affiliation>
+ <orgname>Dell, Inc.</orgname>
+ </affiliation>
+ </author>
+ </refentryinfo>
+
+ <refmeta>
+ <refentrytitle>stacd.conf</refentrytitle>
+ <manvolnum>5</manvolnum>
+ </refmeta>
+
+ <refnamediv>
+ <refname>stacd.conf</refname>
+ <refpurpose>
+ <citerefentry project="man-pages">
+ <refentrytitle>stacd</refentrytitle>
+ <manvolnum>8</manvolnum>
+ </citerefentry>
+ configuration file
+ </refpurpose>
+ </refnamediv>
+
+ <refsynopsisdiv>
+ <para>
+ <filename>/etc/stas/stacd.conf</filename>
+ </para>
+ </refsynopsisdiv>
+
+ <refsect1>
+ <title>Description</title>
+
+ <para>
+ When <citerefentry project="man-pages"><refentrytitle>stacd</refentrytitle>
+ <manvolnum>8</manvolnum></citerefentry> starts up, it reads its
+ configuration from <filename>stacd.conf</filename>.
+ </para>
+ </refsect1>
+
+ <refsect1>
+ <title>Configuration File Format</title>
+ <para>
+ <filename>stacd.conf</filename> is a plain text file divided into
+ sections, with configuration entries in the style
+ <replaceable>key</replaceable>=<replaceable>value</replaceable>.
+ Spaces immediately before or after the <literal>=</literal> are
+ ignored. Empty lines are ignored as well as lines starting with
+ <literal>#</literal>, which may be used for commenting.
+ </para>
+ </refsect1>
+
+ <refsect1>
+ <title>Options</title>
+
+ <refsect2>
+ <title>[Global] section</title>
+ <para>
+ The following options are available in the
+ <literal>[Global]</literal> section:
+ </para>
+
+ <variablelist>
+ <xi:include href="standard-conf.xml" xpointer="tron"/>
+ <xi:include href="standard-conf.xml" xpointer="hdr-digest"/>
+ <xi:include href="standard-conf.xml" xpointer="data-digest"/>
+ <xi:include href="standard-conf.xml" xpointer="kato"/>
+ <xi:include href="standard-conf.xml" xpointer="ip-family"/>
+
+ <varlistentry>
+ <term><varname>nr-io-queues=</varname></term>
+
+ <listitem>
+ <para>
+ Takes a value in the range 1...N. Overrides the
+ default number of I/O queues create by the driver.
+ </para>
+
+ <para>Note: This parameter is identical to that provided by nvme-cli.</para>
+ <para>
+ Default: Depends on kernel and other run
+ time factors (e.g. number of CPUs).
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term><varname>nr-write-queues=</varname></term>
+
+ <listitem>
+ <para>
+ Takes a value in the range 1...N. Adds additional
+ queues that will be used for write I/O.
+ </para>
+
+ <para>Note: This parameter is identical to that provided by nvme-cli.</para>
+
+ <para>
+ Default: Depends on kernel and other run
+ time factors (e.g. number of CPUs).
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term><varname>nr-poll-queues=</varname></term>
+
+ <listitem>
+ <para>
+ Takes a value in the range 1...N. Adds additional
+ queues that will be used for polling latency
+ sensitive I/O.
+ </para>
+
+ <para>Note: This parameter is identical to that provided by nvme-cli.</para>
+
+ <para>
+ Default: Depends on kernel and other run
+ time factors (e.g. number of CPUs).
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <xi:include href="standard-conf.xml" xpointer="queue-size"/>
+ <xi:include href="standard-conf.xml" xpointer="reconnect-delay"/>
+ <xi:include href="standard-conf.xml" xpointer="ctrl-loss-tmo"/>
+ <xi:include href="standard-conf.xml" xpointer="disable-sqflow"/>
+
+ <varlistentry>
+ <term><varname>ignore-iface=</varname></term>
+ <listitem>
+ <para>
+ Takes a boolean argument. This option controls how
+ connections with I/O Controllers (IOC) are made.
+ </para>
+
+ <para>
+ There is no guarantee that there will be a route to
+ reach that IOC. However, we can use the socket
+ option SO_BINDTODEVICE to force the connection to be
+ made on a specific interface instead of letting the
+ routing tables decide where to make the connection.
+ </para>
+
+ <para>
+ This option determines whether <code>stacd</code> will use
+ SO_BINDTODEVICE to force connections on an interface
+ or just rely on the routing tables. The default is
+ to use SO_BINDTODEVICE, in other words, <code>stacd</code> does
+ not ignore the interface.
+ </para>
+
+ <para>
+ BACKGROUND:
+ By default, <code>stacd</code> will connect to IOCs on the same
+ interface that was used to retrieve the discovery
+ log pages. If stafd discovers a DC on an interface
+ using mDNS, and stafd connects to that DC and
+ retrieves the log pages, it is expected that the
+ storage subsystems listed in the log pages are
+ reachable on the same interface where the DC was
+ discovered.
+ </para>
+
+ <para>
+ For example, let's say a DC is discovered on
+ interface ens102. Then all the subsystems listed in
+ the log pages retrieved from that DC must be
+ reachable on interface ens102. If this doesn't work,
+ for example you cannot "ping -I ens102 [storage-ip]",
+ then the most likely explanation is that proxy arp
+ is not enabled on the switch that the host is
+ connected to on interface ens102. Whatever you do,
+ resist the temptation to manually set up the routing
+ tables or to add alternate routes going over a
+ different interface than the one where the DC is
+ located. That simply won't work. Make sure proxy arp
+ is enabled on the switch first.
+ </para>
+
+ <para>
+ Setting routes won't work because, by default, <code>stacd</code>
+ uses the SO_BINDTODEVICE socket option when it
+ connects to IOCs. This option is used to force a
+ socket connection to be made on a specific interface
+ instead of letting the routing tables decide where
+ to connect the socket. Even if you were to manually
+ configure an alternate route on a different interface,
+ the connections (i.e. host to IOC) will still be
+ made on the interface where the DC was discovered by
+ stafd.
+ </para>
+
+ <para>
+ Defaults to <parameter>false</parameter>.
+ </para>
+ </listitem>
+ </varlistentry>
+ </variablelist>
+ </refsect2>
+
+ <refsect2>
+ <title>[I/O controller connection management] section</title>
+ <para>
+ Connectivity between hosts and subsystems in a fabric is
+ controlled by Fabric Zoning. Entities that share a common
+ zone (i.e., are zoned together) are allowed to discover each
+ other and establish connections between them. Fabric Zoning is
+ configured on Discovery Controllers (DC). Users can add/remove
+ controllers and/or hosts to/from zones.
+ </para>
+
+ <para>
+ Hosts have no direct knowledge of the Fabric Zoning configuration
+ that is active on a given DC. As a result, if a host is impacted
+ by a Fabric Zoning configuration change, it will be notified of
+ the connectivity configuration change by the DC via Asynchronous
+ Event Notifications (AEN).
+ </para>
+
+ <table frame='all'>
+ <title>List of terms used in this section:</title>
+ <tgroup cols="2" align='left' colsep='1' rowsep='1'>
+ <thead>
+ <row>
+ <entry>Term</entry>
+ <entry>Description</entry>
+ </row>
+ </thead>
+
+ <tbody>
+ <row>
+ <entry>AEN</entry>
+ <entry>Asynchronous Event Notification. A CQE (Completion Queue Entry) for an Asynchronous Event Request that was previously transmitted by the host to a Discovery Controller. AENs are used by DCs to notify hosts that a change (e.g., a connectivity configuration change) has occurred.</entry>
+ </row>
+
+ <row>
+ <entry>DC</entry>
+ <entry>Discovery Controller.</entry>
+ </row>
+
+ <row>
+ <entry>DLP</entry>
+ <entry>Discovery Log Page. A host will issue a Get Log Page command to retrieve the list of controllers it may connect to.</entry>
+ </row>
+
+ <row>
+ <entry>DLPE</entry>
+ <entry><simpara>
+ Discovery Log Page Entry. The response
+ to a Get Log Page command contains a list of DLPEs identifying
+ each controller that the host is allowed to connect with.
+ </simpara><simpara>
+ Note that DLPEs may contain both I/O Controllers (IOCs)
+ and Discovery Controllers (DCs). DCs listed in DLPEs
+ are called referrals. <code>stacd</code> only deals with IOCs.
+ Referrals (DCs) are handled by <code>stafd</code>.
+ </simpara>
+ </entry>
+ </row>
+
+ <row>
+ <entry>IOC</entry>
+ <entry>I/O Controller.</entry>
+ </row>
+
+ <row>
+ <entry>Manual Config</entry>
+ <entry>Refers to manually adding entries to <filename>stacd.conf</filename> with the <varname>controller=</varname> parameter.</entry>
+ </row>
+
+ <row>
+ <entry>Automatic Config</entry>
+ <entry>Refers to receiving configuration from a DC as DLPEs</entry>
+ </row>
+
+ <row>
+ <entry>External Config</entry>
+ <entry>Refers to configuration done outside of the <code>nvme-stas</code> framework, for example using <code>nvme-cli</code> commands</entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+
+ <para>
+ DCs notify hosts of connectivity configuration changes by sending
+ AENs indicating a "Discovery Log" change. The host uses these AENs as
+ a trigger to issue a Get Log Page command. The response to this command
+ is used to update the list of DLPEs containing the controllers
+ the host is allowed to access.
+ Upon reception of the current DLPEs, the host will determine
+ whether DLPEs were added and/or removed, which will trigger the
+ addition and/or removal of controller connections. This happens in real time
+ and may affect active connections to controllers including controllers
+ that support I/O operations (IOCs). A host that was previously
+ connected to an IOC may suddenly be told that it is no longer
+ allowed to connect to that IOC and should disconnect from it.
+ </para>
+
+ <formalpara><title>IOC connection creation</title>
+ <para>
+ There are 3 ways to configure IOC connections on a host:
+ </para>
+
+ <orderedlist>
+ <listitem>
+ <para>
+ Manual Config by adding <varname>controller=</varname> entries
+ to the <literal>[Controllers]</literal> section (see below).
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ Automatic Config received in the form of
+ DLPEs from a remote DC.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ External Config using <code>nvme-cli</code> (e.g. "<code>nvme connect</code>")
+ </para>
+ </listitem>
+ </orderedlist>
+ </formalpara>
+
+ <formalpara><title>IOC connection removal/prevention</title>
+ <para>
+ There are 3 ways to remove (or prevent) connections to an IOC:
+ </para>
+
+ <orderedlist>
+ <listitem>
+ <para>
+ Manual Config.
+ <orderedlist numeration='lowerroman'>
+ <listitem>
+ <para>
+ by adding <varname>exclude=</varname> entries to
+ the <literal>[Controllers]</literal> section (see below).
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ by removing <varname>controller=</varname> entries
+ from the <literal>[Controllers]</literal> section.
+ </para>
+ </listitem>
+ </orderedlist>
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ Automatic Config. As explained above, a host gets a
+ new list of DLPEs upon connectivity configuration
+ changes. On DLPE removal, the host should remove the
+ connection to the IOC matching that DLPE. This
+ behavior is configurable using the
+ <varname>disconnect-scope=</varname> parameter
+ described below.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ External Config using <code>nvme-cli</code> (e.g. "<code>nvme
+ disconnect</code>" or "<code>nvme disconnect-all</code>")
+ </para>
+ </listitem>
+ </orderedlist>
+ </formalpara>
+
+ <para>
+ The decision by the host to automatically disconnect from an
+ IOC following connectivity configuration changes is controlled
+ by two parameters: <code>disconnect-scope</code>
+ and <code>disconnect-trtypes</code>.
+ </para>
+
+ <variablelist>
+ <varlistentry>
+ <term><varname>disconnect-scope=</varname></term>
+ <listitem>
+ <para>
+ Takes one of: <parameter>only-stas-connections</parameter>,
+ <parameter>all-connections-matching-disconnect-trtypes</parameter>, or <parameter>no-disconnect</parameter>.
+ </para>
+
+ <para>
+ In theory, hosts should only connect to IOCs that have
+ been zoned for them. Connections to IOCs that a host
+ is not zoned to have access to should simply not exist.
+ In practice, however, users may not want hosts to
+ disconnect from all IOCs in reaction to connectivity
+ configuration changes (or at least for some of the IOC
+ connections).
+ </para>
+
+ <para>
+ Some users may prefer for IOC connections to be "sticky"
+ and only be removed manually (<code>nvme-cli</code> or
+ <varname>exclude=</varname>) or removed by a system
+ reboot. Specifically, they don't want IOC connections
+ to be removed unexpectedly on DLPE removal. These users
+ may want to set <varname>disconnect-scope</varname>
+ to <parameter>no-disconnect</parameter>.
+ </para>
+
+ <para>
+ It is important to note that when IOC connections
+ are removed, ongoing I/O transactions will be
+ terminated immediately. There is no way to tell what
+ happens to the data being exchanged when such an abrupt
+ termination happens. If a host was in the middle of writing
+ to a storage subsystem, there is a chance that outstanding
+ I/O operations may not successfully complete.
+ </para>
+
+ <refsect3>
+ <title>Values:</title>
+ <variablelist>
+ <varlistentry>
+ <term><parameter>only-stas-connections</parameter></term>
+ <listitem>
+ <para>
+ Only remove connections previously made by <code>stacd</code>.
+ </para>
+ <para>
+ In this mode, when a DLPE is removed as a result of
+ connectivity configuration changes, the corresponding
+ IOC connection will be removed by <code>stacd</code>.
+ </para>
+ <para>
+ Connections to IOCs made externally, e.g. using <code>nvme-cli</code>,
+ will not be affected, unless they happen to be duplicates
+ of connections made by <code>stacd</code>. It's simply not
+ possible for <code>stacd</code> to tell that a connection
+ was previously made with <code>nvme-cli</code> (or any other external tool).
+ So, it's good practice to avoid duplicating
+ configuration between <code>stacd</code> and external tools.
+ </para>
+ <para>
+ Users wanting to persist some of their IOC connections
+ regardless of connectivity configuration changes should not use
+ <code>nvme-cli</code> to make those connections. Instead,
+ they should hard-code them in <filename>stacd.conf</filename>
+ with the <varname>controller=</varname> parameter. Using the
+ <varname>controller=</varname> parameter is the only way for a user
+ to tell <code>stacd</code> that a connection must be made and
+ not be deleted "<emphasis>no-matter-what</emphasis>".
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term><parameter>all-connections-matching-disconnect-trtypes</parameter></term>
+ <listitem>
+ <para>
+ All connections that match the transport type specified by
+ <varname>disconnect-trtypes=</varname>, whether they were
+ made automatically by <code>stacd</code> or externally
+ (e.g., <code>nvme-cli</code>), will be audited and are
+ subject to removal on DLPE removal.
+ </para>
+ <para>
+ In this mode, as DLPEs are removed as a result of
+ connectivity configuration changes, the corresponding
+ IOC connections will be removed by the host immediately
+ whether they were made by <code>stacd</code>, <code>nvme-cli</code>,
+ or any other way. Basically, <code>stacd</code> audits
+ <emphasis>all</emphasis> IOC connections matching the
+ transport type specified by <varname>disconnect-trtypes=</varname>.
+ </para>
+ <formalpara><title><emphasis>NOTE</emphasis></title>
+ <para>
+ This mode implies that <code>stacd</code> will
+ only allow Manually Configured or Automatically
+ Configured IOC connections to exist. Externally
+ Configured connections using <code>nvme-cli</code>
+ (or other external mechanism)
+ that do not match any Manual Config
+ (<filename>stacd.conf</filename>)
+ or Automatic Config (DLPEs) will get deleted
+ immediately by <code>stacd</code>.
+ </para>
+ </formalpara>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term><parameter>no-disconnect</parameter></term>
+ <listitem>
+ <para>
+ <code>stacd</code> does not disconnect from IOCs
+ when a DPLE is removed or a <varname>controller=</varname>
+ entry is removed from <filename>stacd.conf</filename>.
+ All IOC connections are "sticky".
+ </para>
+
+ <para>
+ Instead, users can remove connections
+ by issuing the <code>nvme-cli</code>
+ command "<code>nvme disconnect</code>", add an
+ <varname>exclude=</varname> entry to
+ <filename>stacd.conf</filename>, or wait
+ until the next system reboot at which time all
+ connections will be removed.
+ </para>
+ </listitem>
+ </varlistentry>
+ </variablelist>
+ </refsect3>
+
+ <para>
+ Defaults to <parameter>only-stas-connections</parameter>.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term><varname>disconnect-trtypes=</varname></term>
+ <listitem>
+ <para>
+ This parameter only applies when <varname>disconnect-scope</varname>
+ is set to <parameter>all-connections-matching-disconnect-trtypes</parameter>.
+ It limits the scope of the audit to specific transport types.
+ </para>
+
+ <para>
+ Can take the values <parameter>tcp</parameter>,
+ <parameter>rdma</parameter>, <parameter>fc</parameter>, or
+ a combination thereof by separating them with a plus (+) sign.
+ For example: <parameter>tcp+fc</parameter>. No spaces
+ are allowed between values and the plus (+) sign.
+ </para>
+
+ <refsect3>
+ <title>Values:</title>
+ <variablelist>
+ <varlistentry>
+ <term><parameter>tcp</parameter></term>
+ <listitem>
+ <para>
+ Audit TCP connections.
+ </para>
+ </listitem>
+ </varlistentry>
+ <varlistentry>
+ <term><parameter>rdma</parameter></term>
+ <listitem>
+ <para>
+ Audit RDMA connections.
+ </para>
+ </listitem>
+ </varlistentry>
+ <varlistentry>
+ <term><parameter>fc</parameter></term>
+ <listitem>
+ <para>
+ Audit Fibre Channel connections.
+ </para>
+ </listitem>
+ </varlistentry>
+ </variablelist>
+ </refsect3>
+
+ <para>
+ Defaults to <parameter>tcp</parameter>.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term><varname>connect-attempts-on-ncc=</varname></term>
+ <listitem>
+ <para>
+ The NCC bit (Not Connected to CDC) is a bit returned
+ by the CDC in the EFLAGS field of the DLPE. Only CDCs
+ will set the NCC bit. DDCs will always clear NCC to
+ 0. The NCC bit is a way for the CDC to let hosts
+ know that the subsystem is currently not reachable
+ by the CDC. This may indicate that the subsystem is
+ currently down or that there is an outage on the
+ section of the network connecting the CDC to the
+ subsystem.
+ </para>
+
+ <para>
+ If a host is currently failing to connect to an I/O
+ controller and if the NCC bit associated with that
+ I/O controller is asserted, the host can decide to
+ stop trying to connect to that subsystem until
+ connectivity is restored. This will be indicated by
+ the CDC when it clears the NCC bit.
+ </para>
+
+ <para>
+ The parameter <varname>connect-attempts-on-ncc=</varname>
+ controls whether <code>stacd</code> will take the
+ NCC bit into account when attempting to connect to
+ an I/O Controller. Setting <varname>connect-attempts-on-ncc=</varname>
+ to 0 means that <code>stacd</code> will ignore
+ the NCC bit and will keep trying to connect. Setting
+ <varname>connect-attempts-on-ncc=</varname> to a
+ non-zero value indicates the number of connection
+ attempts that will be made before <code>stacd</code>
+ gives up trying. Note that this value should be set
+ to a value greater than 1. In fact, when set to 1,
+ <code>stacd</code> will automatically use 2 instead.
+ The reason for this is simple. It is possible that a
+ first connect attempt may fail.
+ </para>
+
+
+ <para>
+ Defaults to <parameter>0</parameter>.
+ </para>
+ </listitem>
+ </varlistentry>
+ </variablelist>
+ </refsect2>
+
+ <xi:include href="standard-conf.xml" xpointer="controller"/>
+ </refsect1>
+
+ <refsect1>
+ <title>See Also</title>
+ <para>
+ <citerefentry>
+ <refentrytitle>stacd</refentrytitle>
+ <manvolnum>8</manvolnum>
+ </citerefentry>
+ </para>
+ </refsect1>
+</refentry>