diff options
129 files changed, 16110 insertions, 0 deletions
diff --git a/.coveragerc.in b/.coveragerc.in new file mode 100644 index 0000000..c9f9105 --- /dev/null +++ b/.coveragerc.in @@ -0,0 +1,39 @@ +# .coveragerc to control coverage.py for combined stafd/stacd coverage +[run] +data_file = coverage/nvme-stas +parallel=True +concurrency=thread + +[report] +omit = + /usr/* + */test/test-*.py + subprojects/libnvme/* + +# Regexes for lines to exclude from consideration +exclude_lines = + # Have to re-enable the standard pragma + pragma: no cover + + # Coverage cannot cover code running in threads + def in_thread_exec + + # Don't complain if tests don't hit defensive assertion code: + raise AssertionError + raise RuntimeError + + # Don't complain if non-runnable code isn't run: + pass + if 0: + if __name__ *== *__main__ *: + sys\.exit\(\) + sys\.exit\(f?'.+\) + + # ImportError is usually OK because there will be a workaround import. + except ImportError + +skip_empty = True + +[html] +directory = coverage +title = nvme-stas coverage report diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..4a9bdf7 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +.git +.github +.build diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..5bb8a98 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +--- +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + - package-ecosystem: "docker" + directory: "/" + schedule: + interval: "weekly"
\ No newline at end of file diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 0000000..75f19fd --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,63 @@ +name: Docker + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +on: + push: + branches: [ main ] + # Publish semver tags as releases. + tags: [ 'v*.*.*' ] + pull_request: + branches: [ main ] + + workflow_dispatch: + +env: + # Use docker.io for Docker Hub if empty + REGISTRY: ghcr.io + # github.repository as <account>/<repo> + IMAGE_NAME: ${{ github.repository }} + + +jobs: + docker-publish: + if: ${{ !github.event.act }} # skip during local actions testing + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Login against a Docker registry except on PR + # https://github.com/docker/login-action + - name: Log into registry ${{ env.REGISTRY }} + if: github.event_name != 'pull_request' + uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Extract metadata (tags, labels) for Docker + # https://github.com/docker/metadata-action + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@507c2f2dc502c992ad446e3d7a5dfbe311567a96 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + # Build and push Docker image with Buildx (don't push on PR) + # https://github.com/docker/build-push-action + - name: Build and push Docker image + uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/docker-test.yml b/.github/workflows/docker-test.yml new file mode 100644 index 0000000..92284c0 --- /dev/null +++ b/.github/workflows/docker-test.yml @@ -0,0 +1,51 @@ +name: Test + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + + workflow_dispatch: + +jobs: + docker-run: + if: ${{ !github.event.act }} # skip during local actions testing + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install requirements + # make sure nvme-cli installed (we need it for /etc/nvme/hostnqn and /etc/nvme/hostid) + run: sudo apt-get install --yes --quiet nvme-cli + + - name: Load Kernel drivers + run: sudo modprobe -v nvme-fabrics + + - name: Fix D-BUS + run: | + sed 's/@STAFD_DBUS_NAME@/org.nvmexpress.staf/g' etc/dbus-1/system.d/org.nvmexpress.staf.in.conf | sudo tee /usr/share/dbus-1/system.d/org.nvmexpress.staf.conf + sed 's/@STACD_DBUS_NAME@/org.nvmexpress.stac/g' etc/dbus-1/system.d/org.nvmexpress.stac.in.conf | sudo tee /usr/share/dbus-1/system.d/org.nvmexpress.stac.conf + sudo systemctl reload dbus.service + + - name: Build & Start containers + run: docker-compose -f "docker-compose.yml" up --detach --build + + - name: Run tests + run: | + docker-compose ps + docker-compose exec -T stafd stafctl ls + docker-compose exec -T stafd stafctl status + docker-compose exec -T stacd stacctl ls + docker-compose exec -T stacd stacctl status + docker-compose logs + + - name: Logs + if: failure() + run: | + docker-compose ps || true + docker-compose logs || true + + - name: Stop containers + if: always() + run: docker-compose -f "docker-compose.yml" down diff --git a/.github/workflows/meson-test.yml b/.github/workflows/meson-test.yml new file mode 100644 index 0000000..2909ffa --- /dev/null +++ b/.github/workflows/meson-test.yml @@ -0,0 +1,76 @@ +name: Meson + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + + workflow_dispatch: + +jobs: + meson-build: + runs-on: ubuntu-latest + steps: + - name: "CHECKOUT: nvme-stas" + uses: actions/checkout@v3 + + - name: "INSTALL: Overall dependencies" + run: | + sudo apt-mark hold grub-efi-amd64-signed # Workaround for upstream issue + sudo apt-get update --yes --quiet + sudo apt-get upgrade --yes --quiet + sudo apt-get install --yes --quiet python3-pip cmake iproute2 + sudo python3 -m pip install --upgrade pip + sudo python3 -m pip install --upgrade wheel meson ninja + + - name: "INSTALL: nvme-stas dependencies" + run: | + sudo apt-get install --yes --quiet docbook-xml + sudo apt-get install --yes --quiet docbook-xsl + sudo apt-get install --yes --quiet xsltproc + sudo apt-get install --yes --quiet libglib2.0-dev + sudo apt-get install --yes --quiet libgirepository1.0-dev + sudo apt-get install --yes --quiet libsystemd-dev + sudo apt-get install --yes --quiet python3-systemd + sudo apt-get install --yes --quiet python3-pyudev + sudo apt-get install --yes --quiet python3-lxml + python3 -m pip install --upgrade dasbus pylint pyflakes PyGObject + python3 -m pip install --upgrade vermin pyfakefs importlib-resources + + - name: "INSTALL: libnvme" + run: | + sudo apt-get install --yes --quiet swig libjson-c-dev + meson subprojects download + meson setup .build subprojects/libnvme -Dlibdbus=disabled -Dopenssl=disabled -Dbuildtype=release -Dprefix=/usr -Dpython=true + ninja -C .build + sudo meson install -C .build + + - name: "CONFIG: PYTHONPATH" + run: | + echo "PYTHONPATH=.build:.build/subprojects/libnvme:/usr/lib/python3/dist-packages/" >> $GITHUB_ENV + + - name: "TEST: nvme-stas" + uses: BSFishy/meson-build@v1.0.3 + with: + action: test + directory: .build + setup-options: -Dman=true -Dhtml=true + options: --print-errorlogs --suite nvme-stas + + # Preserve meson's log file on failure + - uses: actions/upload-artifact@v3 + if: failure() + with: + name: "Linux_Meson_Testlog" + path: .build/meson-logs/* + + - name: "Generate coverage report" + run: | + python3 -m pip install pytest + python3 -m pip install pytest-cov + PYTHONPATH=.build:.build/subprojects/libnvme:/usr/lib/python3/dist-packages/ pytest --cov=./staslib --cov-report=xml test/test-*.py + + - uses: codecov/codecov-action@v3 + with: + fail_ci_if_error: false
\ No newline at end of file diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml new file mode 100644 index 0000000..abd3c1b --- /dev/null +++ b/.github/workflows/pylint.yml @@ -0,0 +1,104 @@ +name: Linters + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + + workflow_dispatch: + +jobs: + + docker-lint: + if: ${{ !github.event.act }} # skip during local actions testing + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v3 + - uses: hadolint/hadolint-action@v3.1.0 + with: + recursive: true + ignore: DL3041 + + python-lint: + runs-on: ubuntu-20.04 + + strategy: + fail-fast: false + matrix: + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] + + steps: + - name: "CHECKOUT: nvme-stas" + uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + +# - name: "UPGRADE: existing packages" +# run: | +# sudo apt-get update --yes --quiet || true +# sudo apt-get upgrade --yes --quiet || true + + - name: "INSTALL: additional packages" + run: | + sudo apt-get install --yes --quiet python3-pip || true + sudo apt-get install --yes --quiet cmake || true + sudo apt-get install --yes --quiet libgirepository1.0-dev || true + sudo apt-get install --yes --quiet libsystemd-dev || true + sudo apt-get install --yes --quiet python3-systemd || true + sudo python3 -m pip install --upgrade pip + sudo python3 -m pip install --upgrade wheel + sudo python3 -m pip install --upgrade meson + sudo python3 -m pip install --upgrade ninja + python3 -m pip install --upgrade dasbus + python3 -m pip install --upgrade pylint + python3 -m pip install --upgrade pyflakes + python3 -m pip install --upgrade PyGObject + python3 -m pip install --upgrade lxml + python3 -m pip install --upgrade pyudev + + - name: "BUILD: libnvme" + run: | + sudo apt-get install --yes --quiet swig libjson-c-dev || true + meson subprojects download + meson setup builddir subprojects/libnvme -Dlibdbus=disabled -Dopenssl=disabled -Dbuildtype=release -Dprefix=/usr -Dpython=true + ninja -C builddir + sudo meson install -C builddir + + - name: Set PYTHONPATH + run: | + echo "PYTHONPATH=builddir:builddir/subprojects/libnvme:/usr/lib/python3/dist-packages/" >> $GITHUB_ENV + + - name: Show test environment + run: | + echo -e "Build Directory:\n$(ls -laF builddir)" + python3 -VV + python3 -m site + python3 -m pylint --version + echo "pyflakes $(python3 -m pyflakes --version)" + + - name: Pylint + run: | + python3 -m pylint --rcfile=test/pylint.rc *.py staslib + + - name: Pyflakes + if: always() + run: | + python3 -m pyflakes *.py staslib + + python-black: + if: ${{ !github.event.act }} # skip during local actions testing + name: python-black formatter + runs-on: ubuntu-latest + steps: + - name: "CHECKOUT: nvme-stas" + uses: actions/checkout@v3 + + - name: "BLACK" + uses: psf/black@stable + with: + options: "--check --diff --color --line-length 120 --skip-string-normalization --extend-exclude (subprojects|debian|.build)" + src: "." diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c1f4487 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +.build +obj-x86_64-linux-gnu # DEBs Artifacts +redhat-linux-build # RPMs Artifacts +__pycache__ + +subprojects/* +!subprojects/*.wrap + +.vscode diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..8744afb --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,33 @@ +# Copyright (c) 2022, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). +# +# Authors: Martin Belanger <Martin.Belanger@dell.com> +# + +version: 2 + +python: + system_packages: true + +build: + os: ubuntu-22.04 + tools: + python: "3" + apt_packages: + - meson + - python3-lxml + - docbook-xsl + - xsltproc + - pandoc + jobs: + post_install: + - pip3 install lxml + pre_build: + - meson .build -Dreadthedocs=true || cat .build/meson-logs/meson-log.txt + - ninja -C .build + +sphinx: + configuration: .build/doc/readthedocs/conf.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..11e30b7 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,53 @@ +# Contributing guidelines + +Thanks for contributing to this project. We'd like to get your feedback and suggestions. + +## Issues + +Bugs, feature requests, or issues must be reported through GitHub's "[Issues](https://github.com/linux-nvme/nvme-stas/issues)". Make sure there is not an existing open issue (or recently closed) for the same problem or feature. Include all pertinent info: environment, nvme-stas version, how to reproduce, expected result, etc. + +## Contribution process + +All contributions should be made through pull requests. Before submitting make sure that you followed the coding style (below) and you ran and passed the unit tests. + +### How to submit contributions + +1. Fork the repo +2. Make changes. Try to split you changes into distinct commits and avoid making big reformatting as it makes it harder to review the changes. +3. If possible, add unit tests for new features. +4. Run `make black` to make sure the changes conform to coding styles. See [Coding styles]() below. +5. Run `make test` and make sure all tests pass. +6. Commit to your fork with descriptive message and use the "`--signoff, -s`" option +7. Send the pull request +8. Check for failures in the automated CI output. +9. Be involved in the conversation (if any). + +## Coding style + +nvme-stas uses [black](https://black.readthedocs.io/en/stable/), [pylint](https://pylint.readthedocs.io/en/latest/), and [pyflakes](https://pypi.org/project/pyflakes/) to check that the code meets minimum style requirements. We use `black` with the following options. + +```bash +black --diff --color --line-length 120 --skip-string-normalization [file or directory] +``` + +You can also use this convenience make command: + +``` +make black +``` + +## Minimum Python version required + +nvme-stas must be able to run with Python 3.6. Code changes cannot use Python features not supported by Python 3.6. The only exception is for Python scripts used during the build phase (e.g. scripts to generate the documentation) or test scripts. Those scripts can follow Python 3.8 syntax. + +nvme-stas uses [vermin](https://pypi.org/project/vermin/) to verify that the code submitted complies with the minimum version required. Vermin gets executed as part of the tests (see `make test` below). + +## Unit tests + +Unit tests can be run with this command: + +```bash +make test +``` + +This command not only runs the unit tests, but also pylint, pyflakes, and vermin. Make sure that these programs are installed otherwise the tests will be skipped. diff --git a/DISTROS.md b/DISTROS.md new file mode 100644 index 0000000..0f1522a --- /dev/null +++ b/DISTROS.md @@ -0,0 +1,403 @@ +# Notes to Linux distributors + +This document contains information about the packaging of nvme-stas. + +## Compile-time dependencies + +nvme-stas is a Python 3 project and does not require compile-time libraries per se. However, we use the meson build system for installation and testing. + +| Library / Program | Purpose | Mandatory / Optional | +| ----------------- | ------------------------------------------------- | -------------------- | +| meson | Project configuration, installation, and testing. | Mandatory | + +## Unit tests dependencies + +nvme-stas provides static code analysis (pylint, pyflakes), which can be run with "`meson test`". + +| Library / Program | Purpose | Mandatory / Optional | +| ----------------- | ------------------------------------------------------------ | -------------------- | +| pylint | Static code analysis | Optional | +| python3-pyflakes | Static code analysis | Optional | +| python3-pyfakefs | Static code analysis | Optional | +| vermin | Check that code meets minimum Python version requirement (3.6) | Optional | + +## Run-time dependencies + +Python 3.6 is the minimum version required to run nvme-stas. nvme-stas is built on top of libnvme, which is used to interact with the kernel's NVMe driver (i.e. `drivers/nvme/host/`). To support all the features of nvme-stas, several changes to the Linux kernel are required. nvme-stas can also operate with older kernels, but with limited functionality. Kernel 5.18 provides all the features needed by nvme-stas. nvme-stas can also work with older kernels that include back-ported changes to the NVMe driver. + +The next table shows different features that were added to the NVMe driver and in which version of the Linux kernel they were added (the list of git patches can be found in addendum). Note that the ability to query the NVMe driver to determine what options it supports was added in 5.17. This is needed if nvme-stas is to make the right decision on whether a feature is supported. Otherwise, nvme-stas can only rely on the kernel version to decide what is supported. This can greatly limit the features supported on back-ported kernels. + +| Feature | Introduced in kernel version | +| ------------------------------------------------------------ | ---------------------------- | +| **`host-iface` option** - Ability to force TCP connections over a specific interface. Needed for zeroconf provisioning. | 5.14 | +| **TP8013 Support** - Discovery Controller (DC) Unique NQN. Allow the creation of connections to DC with a NQN other than the default `nqn.2014-08.org.nvmexpress.discovery` | 5.16 | +| **Query supported options** - Allow user-space applications to query which options the NVMe driver supports | 5.17 | +| **TP8010 Support** - Ability for a Host to register with a Discovery Controller. This version of the kernel introduces a new event to indicate to user-space apps (e.g. nvme-stas) when a connection to a DC is restored. This is used to trigger a re-registration of the host. This kernel also exposes the DC Type (dctype) attribute through the sysfs, which is needed to determine whether registration is supported. | 5.18 | +| - Print actual source IP address (`src_addr`) through sysfs "address" attr. This is needed to verify that TCP connections were made on the right interface.<br />- Consider also `host_iface` when checking IP options.<br />- Send a rediscover uevent when a persistent discovery controller reconnects. | 6.1 | + +nvme-stas also depends on the following run-time libraries and modules. Note that versions listed are the versions that were tested with at the time the code was developed. + +| Package / Module | Min version | stafd | stacd | How to determine the currently installed version | +| ---------------------------------------------------------- | ----------- | ------------- | ------------- | ------------------------------------------------------------ | +| python3 | 3.6 | **Mandatory** | **Mandatory** | `python3 --version`<br />`nvme-stas` requires Python 3.6 as a minimum. | +| python3-dasbus | 1.6 | **Mandatory** | **Mandatory** | pip list \| grep dasbus | +| python3-pyudev | 0.22.0 | **Mandatory** | **Mandatory** | `python3 -c 'import pyudev; print(f"{pyudev.__version__}")'` | +| python3-systemd | 240 | **Mandatory** | **Mandatory** | `systemd --version` | +| python3-gi (Debian) OR python3-gobject (Fedora) | 3.36.0 | **Mandatory** | **Mandatory** | `python3 -c 'import gi; print(f"{gi.__version__}")'` | +| nvme-tcp (kernel module) | 5.18 * | **Mandatory** | **Mandatory** | N/A | +| dbus-daemon | 1.12.2 | **Mandatory** | **Mandatory** | `dbus-daemon --version` | +| avahi-daemon | 0.7 | **Mandatory** | Not required | `avahi-daemon --version` | +| python3-libnvme | 1.3 | **Mandatory** | **Mandatory** | `python3 -c 'import libnvme; print(f"{libnvme.__version__}")'` | +| importlib.resources.files() OR importlib_resources.files() | *** | Optional | Optional | `importlib.resources.files()` was introduced in Python 3.9 and backported to earlier versions as `importlib_resources.files()`. If neither modules can be found, `nvme-stas` will default to using the less efficient `pkg_resources.resource_string()` instead. When `nvme-stas` is no longer required to support Python 3.6 and is allowed a minimum of 3.9 or later, only `importlib.resources.files()` will be required. | + +* Kernel 5.18 provides full functionality. nvme-stas can work with older kernels, but with limited functionality, unless the kernels contain back-ported features (see Addendum for the list of kernel patches that could be back-ported to an older kernel). + +## Things to do post installation + +### D-Bus configuration + +We install D-Bus configuration files under `/usr/share/dbus-1/system.d`. One needs to run **`systemctl reload dbus-broker.service`** (Fedora) OR **`systemctl reload dbus.service`** (SuSE, Debian) for the new configuration to take effect. + +### Configuration shared with `libnvme` and `nvme-cli` + +`stafd` and `stacd` use the `libnvme` library to interact with the Linux kernel. And `libnvme` as well as `nvme-cli` rely on two configuration files, `/etc/nvme/hostnqn` and `/etc/nvme/hostid`, to retrieve the Host NQN and ID respectively. These files should be created post installation with the help of the `stadadm` utility. Here's an example for Debian-based systems: + +``` +if [ "$1" = "configure" ]; then + if [ ! -d "/etc/nvme" ] + mkdir /etc/nvme + fi + if [ ! -s "/etc/nvme/hostnqn" ]; then + stasadm hostnqn -f /etc/nvme/hostnqn + fi + if [ ! -s "/etc/nvme/hostid" ]; then + stasadm hostid -f /etc/nvme/hostid + fi +fi +``` + +The utility program `stasadm` gets installed with `nvme-stas`. `stasadm` also manages the creation (and updating) of `/etc/stas/sys.conf`, the `nvme-stas` system configuration file. + +### Configuration specific to nvme-stas + +The [README](./README.md) file defines the following three configuration files: + +- `/etc/stas/sys.conf` +- `/etc/stas/stafd.conf` +- `/etc/stas/stacd.conf` + +Care should be taken during upgrades to preserve customer configuration and not simply overwrite it. The process to migrate the configuration data and the list of parameters to migrate is still to be defined. + +### Enabling and starting the daemons + +Lastly, the two daemons, `stafd` and `stacd`, should be enabled (e.g. `systemctl enable stafd.service stacd.service`) and started (e.g. `systemctl start stafd.service stacd.service`). + +# Compatibility between nvme-stas and nvme-cli + +Udev rules are installed along with `nvme-cli` (e.g. `/usr/lib/udev/rules.d/70-nvmf-autoconnect.rules`). These udev rules allow `nvme-cli` to perform tasks similar to those performed by `nvme-stas`. However, the udev rules in `nvme-cli` version 2.1.2 and prior drop the `host-iface` parameter when making TCP connections to I/O controllers. `nvme-stas`, on the other hand, always makes sure that TCP connections to I/O controllers are made over the right interface using the `host-iface` parameter. + +We essentially have a race condition because `nvme-stas` and `nvme-cli` react to the same kernel events. Both try to perform the same task in parallel, which is to connect to I/O controllers. Because `nvme-stas` is written in Python and the udevd daemon (i.e. the process running the udev rules) in C, `nvme-stas` usually loses the race and TCP connections are made by the udev rules without specifying the `host-iface`. + +To remedy to this problem, `nvme-stas` disables `nvme-cli` udev rules and assumes the tasks performed by the udev rules. This way, only one process will take action on kernel events eliminating any race conditions. This also ensure that the right `host-iface` is used when making TCP connections. + +# Addendum + +## Kernel patches for nvme-stas 1.x + +Here's the list of kernel patches (added in kernels 5.14 to 5.18) that will enable all features of nvme-stas. + +``` +commit e3448b134426741902b6e2c07cbaf5f66cfd2ebc +Author: Martin Belanger <martin.belanger@dell.com> +Date: Tue Feb 8 14:18:02 2022 -0500 + + nvme: Expose cntrltype and dctype through sysfs + + TP8010 introduces the Discovery Controller Type attribute (dctype). + The dctype is returned in the response to the Identify command. This + patch exposes the dctype through the sysfs. Since the dctype depends on + the Controller Type (cntrltype), another attribute of the Identify + response, the patch also exposes the cntrltype as well. The dctype will + only be displayed for discovery controllers. + + A note about the naming of this attribute: + Although TP8010 calls this attribute the Discovery Controller Type, + note that the dctype is now part of the response to the Identify + command for all controller types. I/O, Discovery, and Admin controllers + all share the same Identify response PDU structure. Non-discovery + controllers as well as pre-TP8010 discovery controllers will continue + to set this field to 0 (which has always been the default for reserved + bytes). Per TP8010, the value 0 now means "Discovery controller type is + not reported" instead of "Reserved". One could argue that this + definition is correct even for non-discovery controllers, and by + extension, exposing it in the sysfs for non-discovery controllers is + appropriate. + + Signed-off-by: Martin Belanger <martin.belanger@dell.com> + +commit 68c483a105ce7107f1cf8e1ed6c2c2abb5baa551 +Author: Martin Belanger <martin.belanger@dell.com> +Date: Thu Feb 3 16:04:29 2022 -0500 + + nvme: send uevent on connection up + + When connectivity with a controller is lost, the driver will keep + trying to reconnect once every 10 sec. When connection is restored, + user-space apps need to be informed so that they can take proper + action. For example, TP8010 introduces the DIM PDU, which is used to + register with a discovery controller (DC). The DIM PDU is sent from + user-space. The DIM PDU must be sent every time a connection is + established with a DC. Therefore, the kernel must tell user-space apps + when connection is restored so that registration can happen. + + The uevent sent is a "change" uevent with environmental data + set to: "NVME_EVENT=connected". + + Signed-off-by: Martin Belanger <martin.belanger@dell.com> + Reviewed-by: Hannes Reinecke <hare@suse.de> + Reviewed-by: Sagi Grimberg <sagi@grimberg.me> + Reviewed-by: Chaitanya Kulkarni <kch@nvidia.com> + +commit f18ee3d988157ebcadc9b7e5fd34811938f50223 +Author: Hannes Reinecke <hare@suse.de> +Date: Tue Dec 7 14:55:49 2021 +0100 + + nvme-fabrics: print out valid arguments when reading from /dev/nvme-fabrics + + Currently applications have a hard time figuring out which + nvme-over-fabrics arguments are supported for any given kernel; + the ioctl will return an error code on failure, and the application + has to guess whether this was due to an invalid argument or due + to a connection or controller error. + With this patch applications can read a list of supported + arguments by simply reading from /dev/nvme-fabrics, allowing + them to validate the connection string. + + Signed-off-by: Hannes Reinecke <hare@suse.de> + Reviewed-by: Chaitanya Kulkarni <kch@nvidia.com> + Signed-off-by: Christoph Hellwig <hch@lst.de> + + +commit e5ea42faa773c6a6bb5d9e9f5c2cc808940b5a55 +Author: Hannes Reinecke <hare@suse.de> +Date: Wed Sep 22 08:35:25 2021 +0200 + + nvme: display correct subsystem NQN + + With discovery controllers supporting unique subsystem NQNs the + actual subsystem NQN might be different from that one passed in + via the connect args. So add a helper to display the resulting + subsystem NQN. + + Signed-off-by: Hannes Reinecke <hare@suse.de> + Reviewed-by: Chaitanya Kulkarni <kch@nvidia.com> + Signed-off-by: Christoph Hellwig <hch@lst.de> + +commit 20e8b689c9088027b7495ffd6f80812c11ecc872 +Author: Hannes Reinecke <hare@suse.de> +Date: Wed Sep 22 08:35:24 2021 +0200 + + nvme: Add connect option 'discovery' + + Add a connect option 'discovery' to specify that the connection + should be made to a discovery controller, not a normal I/O controller. + With discovery controllers supporting unique subsystem NQNs we + cannot easily distinguish by the subsystem NQN if this should be + a discovery connection, but we need this information to blank out + options not supported by discovery controllers. + + Signed-off-by: Hannes Reinecke <hare@suse.de> + Reviewed-by: Chaitanya Kulkarni <kch@nvidia.com> + Signed-off-by: Christoph Hellwig <hch@lst.de> + +commit 954ae16681f6bdf684f016ca626329302a38e177 +Author: Hannes Reinecke <hare@suse.de> +Date: Wed Sep 22 08:35:23 2021 +0200 + + nvme: expose subsystem type in sysfs attribute 'subsystype' + + With unique discovery controller NQNs we cannot distinguish the + subsystem type by the NQN alone, but need to check the subsystem + type, too. + So expose the subsystem type in a new sysfs attribute 'subsystype'. + + Signed-off-by: Hannes Reinecke <hare@suse.de> + Reviewed-by: Chaitanya Kulkarni <kch@nvidia.com> + Signed-off-by: Christoph Hellwig <hch@lst.de> + + +commit 3ede8f72a9a2825efca23a3552e80a1202ea88fd +Author: Martin Belanger <martin.belanger@dell.com> +Date: Thu May 20 15:09:34 2021 -0400 + + nvme-tcp: allow selecting the network interface for connections + + In our application, we need a way to force TCP connections to go out a + specific IP interface instead of letting Linux select the interface + based on the routing tables. + + Add the 'host-iface' option to allow specifying the interface to use. + When the option host-iface is specified, the driver uses the specified + interface to set the option SO_BINDTODEVICE on the TCP socket before + connecting. + + This new option is needed in addtion to the existing host-traddr for + the following reasons: + + Specifying an IP interface by its associated IP address is less + intuitive than specifying the actual interface name and, in some cases, + simply doesn't work. That's because the association between interfaces + and IP addresses is not predictable. IP addresses can be changed or can + change by themselves over time (e.g. DHCP). Interface names are + predictable [1] and will persist over time. Consider the following + configuration. + + 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state ... + link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 + inet 100.0.0.100/24 scope global lo + valid_lft forever preferred_lft forever + 2: enp0s3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc ... + link/ether 08:00:27:21:65:ec brd ff:ff:ff:ff:ff:ff + inet 100.0.0.100/24 scope global enp0s3 + valid_lft forever preferred_lft forever + 3: enp0s8: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc ... + link/ether 08:00:27:4f:95:5c brd ff:ff:ff:ff:ff:ff + inet 100.0.0.100/24 scope global enp0s8 + valid_lft forever preferred_lft forever + + The above is a VM that I configured with the same IP address + (100.0.0.100) on all interfaces. Doing a reverse lookup to identify the + unique interface associated with 100.0.0.100 does not work here. And + this is why the option host_iface is required. I understand that the + above config does not represent a standard host system, but I'm using + this to prove a point: "We can never know how users will configure + their systems". By te way, The above configuration is perfectly fine + by Linux. + + The current TCP implementation for host_traddr performs a + bind()-before-connect(). This is a common construct to set the source + IP address on a TCP socket before connecting. This has no effect on how + Linux selects the interface for the connection. That's because Linux + uses the Weak End System model as described in RFC1122 [2]. On the other + hand, setting the Source IP Address has benefits and should be supported + by linux-nvme. In fact, setting the Source IP Address is a mandatory + FedGov requirement (e.g. connection to a RADIUS/TACACS+ server). + Consider the following configuration. + + $ ip addr list dev enp0s8 + 3: enp0s8: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc ... + link/ether 08:00:27:4f:95:5c brd ff:ff:ff:ff:ff:ff + inet 192.168.56.101/24 brd 192.168.56.255 scope global enp0s8 + valid_lft 426sec preferred_lft 426sec + inet 192.168.56.102/24 scope global secondary enp0s8 + valid_lft forever preferred_lft forever + inet 192.168.56.103/24 scope global secondary enp0s8 + valid_lft forever preferred_lft forever + inet 192.168.56.104/24 scope global secondary enp0s8 + valid_lft forever preferred_lft forever + + Here we can see that several addresses are associated with interface + enp0s8. By default, Linux always selects the default IP address, + 192.168.56.101, as the source address when connecting over interface + enp0s8. Some users, however, want the ability to specify a different + source address (e.g., 192.168.56.102, 192.168.56.103, ...). The option + host_traddr can be used as-is to perform this function. + + In conclusion, I believe that we need 2 options for TCP connections. + One that can be used to specify an interface (host-iface). And one that + can be used to set the source address (host-traddr). Users should be + allowed to use one or the other, or both, or none. Of course, the + documentation for host_traddr will need some clarification. It should + state that when used for TCP connection, this option only sets the + source address. And the documentation for host_iface should say that + this option is only available for TCP connections. + + References: + [1] https://www.freedesktop.org/wiki/Software/systemd/PredictableNetworkInterfaceNames/ + [2] https://tools.ietf.org/html/rfc1122 + + Tested both IPv4 and IPv6 connections. + + Signed-off-by: Martin Belanger <martin.belanger@dell.com> + Reviewed-by: Sagi Grimberg <sagi@grimberg.me> + Reviewed-by: Hannes Reinecke <hare@suse.de> + Signed-off-by: Christoph Hellwig <hch@lst.de> +``` + +## Kernel patches for nvme-stas 2.x + +These patches are not essential for nvme-stas 2.x, but they allow nvme-stas to operate better. + +``` +commit f46ef9e87c9e8941b7acee45611c7c6a322592bb +Author: Sagi Grimberg <sagi@grimberg.me> +Date: Thu Sep 22 11:15:37 2022 +0300 + + nvme: send a rediscover uevent when a persistent discovery controller reconnects + + When a discovery controller is disconnected, no AENs will arrive to + notify the host about discovery log change events. + + In order to solve this, send a uevent notification when a + persistent discovery controller reconnects. We add a new ctrl + flag NVME_CTRL_STARTED_ONCE that will be set on the first + start, and consecutive calls will find it set, and send the + event to userspace if the controller is a discovery controller. + + Upon the event reception, userspace will re-read the discovery + log page and will act upon changes as it sees fit. + + Signed-off-by: Sagi Grimberg <sagi@grimberg.me> + Reviewed-by: Daniel Wagner <dwagner@suse.de> + Reviewed-by: James Smart <jsmart2021@gmail.com> + Signed-off-by: Christoph Hellwig <hch@lst.de> + +commit 02c57a82c0081141abc19150beab48ef47f97f18 (tag: nvme-6.1-2022-09-20) +Author: Martin Belanger <martin.belanger@dell.com> +Date: Wed Sep 7 08:27:37 2022 -0400 + + nvme-tcp: print actual source IP address through sysfs "address" attr + + TCP transport relies on the routing table to determine which source + address and interface to use when making a connection. Currently, there + is no way to tell from userspace where a connection was made. This + patch exposes the actual source address using a new field named + "src_addr=" in the "address" attribute. + + This is needed to diagnose and identify connectivity issues. With the + source address we can infer the interface associated with each + connection. + + This was tested with nvme-cli 2.0 to verify it does not have any + adverse effect. The new "src_addr=" field will simply be displayed in + the output of the "list-subsys" or "list -v" commands as shown here. + + $ nvme list-subsys + nvme-subsys0 - NQN=nqn.2014-08.org.nvmexpress.discovery + \ + +- nvme0 tcp traddr=192.168.56.1,trsvcid=8009,src_addr=192.168.56.101 live + + Signed-off-by: Martin Belanger <martin.belanger@dell.com> + Reviewed-by: Sagi Grimberg <sagi@grimberg.me> + Reviewed-by: Chaitanya Kulkarni <kch@nvidia.com> + Signed-off-by: Christoph Hellwig <hch@lst.de> + +commit 4cde03d82e2d0056d20fd5af6a264c7f5e6a3e76 +Author: Daniel Wagner <dwagner@suse.de> +Date: Fri Jul 29 16:26:30 2022 +0200 + + nvme: consider also host_iface when checking ip options + + It's perfectly fine to use the same traddr and trsvcid more than once + as long we use different host interface. This is used in setups where + the host has more than one interface but the target exposes only one + traddr/trsvcid combination. + + Use the same acceptance rules for host_iface as we have for + host_traddr. + + Signed-off-by: Daniel Wagner <dwagner@suse.de> + Reviewed-by: Chao Leng <lengchao@huawei.com> + Signed-off-by: Christoph Hellwig <hch@lst.de> +``` + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..e1ef30d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +FROM fedora:37 + +WORKDIR /root + +# first line for nvme-stas +# second line for libnvme +RUN dnf install -y python3-dasbus python3-pyudev python3-systemd python3-gobject meson \ + git gcc g++ cmake openssl-devel libuuid-devel json-c-devel swig python-devel meson && dnf clean all + +COPY . . +RUN meson .build && ninja -C .build && meson install -C .build + +ENTRYPOINT ["python3"] @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2021 STFS + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1d67940 --- /dev/null +++ b/Makefile @@ -0,0 +1,111 @@ +# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). +# +# Authors: Martin Belanger <Martin.Belanger@dell.com> +# +.DEFAULT_GOAL := stas +BUILD-DIR := .build +DEB-PKG-DIR := ${BUILD-DIR}/deb-pkg +RPM-BUILDROOT-DIR := ${BUILD-DIR}/rpmbuild + +ifneq (,$(strip $(filter $(MAKECMDGOALS),rpm deb dist))) + XTRA-MESON-OPTS := --wrap-mode=nodownload +endif + +.PHONY: update-subprojects +update-subprojects: + meson subprojects update + +${BUILD-DIR}: + BUILD_DIR=${BUILD-DIR} ./configure ${XTRA-MESON-OPTS} + @echo "Configuration located in: $@" + @echo "-------------------------------------------------------" + +.PHONY: stas +stas: ${BUILD-DIR} + ninja -C ${BUILD-DIR} + +.PHONY: clean +clean: +ifneq ("$(wildcard ${BUILD-DIR})","") + ninja -C ${BUILD-DIR} -t clean +endif + +.PHONY: purge +purge: +ifneq ("$(wildcard ${BUILD-DIR})","") + rm -rf ${BUILD-DIR} +endif + +.PHONY: install +install: stas + sudo meson $@ -C ${BUILD-DIR} --skip-subprojects + +.PHONY: uninstall +uninstall: ${BUILD-DIR} + sudo ninja -C ${BUILD-DIR} uninstall + +.PHONY: dist +dist: stas + meson $@ -C ${BUILD-DIR} --formats gztar + +.PHONY: test +test: stas + meson $@ -C ${BUILD-DIR} --suite nvme-stas + +.PHONY: loc +loc: + @cloc --by-file --exclude-dir=${BUILD-DIR},doc,subprojects,test,utils,debian,obj-x86_64-linux-gnu,.github --exclude-lang=Markdown,"NAnt script",XML,"Bourne Again Shell",make,"Bourne Shell",Meson,YAML,XSLT . + +.PHONY: loc-full +loc-full: + @cloc --by-file --exclude-dir=${BUILD-DIR},subprojects,debian,obj-x86_64-linux-gnu,.github . + +.PHONY: black +black: + black --diff --color --line-length 120 --skip-string-normalization --extend-exclude="(subprojects|debian|.build)" . + +# Coverage requirements: +# pip install coverage +.PHONY: coverage +coverage: stas + cd ${BUILD-DIR} && ./coverage.sh + +################################################################################ +# Debian (*.deb) +# Use "DEB_BUILD_OPTIONS=nocheck make deb" to skip unit testing. +# This requires: sudo apt install -y debhelper dh-python +ifeq (deb,$(strip $(MAKECMDGOALS))) + ifneq (SUCCESS,$(shell dpkg -s debhelper dh-python > /dev/null 2>&1 && echo "SUCCESS" || echo "FAIL")) + $(error Missing packages. Run -> "sudo apt install -y debhelper dh-python") + endif +endif + +.PHONY: deb +deb: ${BUILD-DIR} + mkdir -p ${DEB-PKG-DIR} + dpkg-buildpackage -us -uc + @mv ../nvme-stas_*.deb ${DEB-PKG-DIR} + @mv ../nvme-stas_*.buildinfo ${DEB-PKG-DIR} + @mv ../nvme-stas_*.changes ${DEB-PKG-DIR} + @mv ../nvme-stas_*.dsc ${DEB-PKG-DIR} + @mv ../nvme-stas_*.tar.gz ${DEB-PKG-DIR} + @echo "=======================================================" + @echo "Debian packages located in: ${DEB-PKG-DIR}/" + + +################################################################################ +# RedHat (*.rpm) +${BUILD-DIR}/nvme-stas.spec: ${BUILD-DIR} nvme-stas.spec.in + meson --wrap-mode=nodownload --reconfigure ${BUILD-DIR} + +${RPM-BUILDROOT-DIR}: ${BUILD-DIR}/nvme-stas.spec + rpmbuild -ba $< --build-in-place --clean --nocheck --define "_topdir $(abspath ${BUILD-DIR}/rpmbuild)" + @echo "=======================================================" + @echo "RPM packages located in: ${RPM-BUILDROOT-DIR}/" + +.PHONY: rpm +rpm: ${RPM-BUILDROOT-DIR} @@ -0,0 +1,143 @@ +# STorage Appliance Services (STAS) + +## Changes with release 2.2.1 + +Added a few more unit and coverage tests. Fixed the following bugs. + +Bug fixes: + +* Fix errors with some debug commands (e.g. `stafctl ls --detailed`) +* Fix setting controller DHCHAP key (this requires [corresponding changes in libnvme](https://github.com/linux-nvme/libnvme/pull/597)) + +## Changes with release 2.2 + +Support for in-band authentication. + +## Changes with release 2.1.3 + +This release is all about `udev rules`. As explained in [DISTROS.md](./DISTROS.md), `nvme-stas` and `nvme-cli` compete for the same kernel events (a.k.a. uevents or udev events). Those are events generated by the kernel related to Discovery Controller (DC) state changes. For example, an AEN indicating a change of Discovery Log Page (DLP), or an event indicating that the the connection to a DC was restored (event = `connected` or `rediscover`), which means that the DLP needs to be refreshed and connections to controllers listed in the DLP need to be updated. + +When both `nvme-stas` and `nvme-cli` are allowed to react and process these events, we have a race condition where both processes try to perform the same connections at the same time. Since the kernel will not allow duplicate connections, then one process will get an error. This is not a real problem since the connection does succeed, but the kernel will log an error and this can be irritating to users. + +We tried different ways to fix this issue. The simplest was to disable the `udev rules` installed by `nvme-cli`. This prevents `nvme-cli` from reacting to udev events and only `nvme-stas` gets to process the events. The downside to this is that `nvme-stas` only expects udev events from DCs that it manages. If a DC connection is made outside of `nvme-stas` (e.g. using `nvme-cli`) and `nvme-stas` receives an event for that DC, it won't know what to do with it and will simply ignore it. + +To solve this issue, and to eliminate the race condition, this release of `nvme-stas` includes changes that allows `nvme-stas` to react and process events even for DCs that are not managed by `nvme-stas`. In that case, `nvme-stas` invokes `nvme-cli's` standard event handler. While `nvme-stas` is running, `nvme-cli's` `udev rules` will be disabled and all event handling will be performed by `nvme-stas`. `nvme-cli's` `udev rules` are restored when `nvme-stas` is stopped. + +With this change we no longer need to provide the configuration parameter `udev-rule=[enabled|disabled]` in `stacd.conf`. This parameter is therefore deprecated. + +This release also adds the "[black](https://github.com/psf/black)" code formatter to the GitHub actions. From now on, code submitted as a pull request with GitHub must comply to black's code format. A new command, `make black`, has been added to allow users to verify their code before submitting a pull request. + +## Changes with release 2.1.2 + +* Bug fixes: + * Add support for RoCE and iWARP protocols in mDNS TXT field (i.e. `p=roce`, `p=iwarp`) + * Add `_nvme-disc._udp` to the list of supported mDNS service types (stype) + +## Changes with release 2.1.1 + +* Bug fixes: + * Fix handling of unresponsive zeroconf-discovered Discovery Controllers. Sometimes we could have a timeout during twice as long as normal. + * Set default value of legacy "[Global] persistent-connections=false" + * Add `ControllerTerminator` entity to deal with potential (rare) cases where Connect/Disconnect operations could be performed in reverse order. +* Add more unit tests +* Increase code coverage +* Improve name resolution algorithm +* Set udev event priority to high (for faster handling) + +## Changes with release 2.1 + +* Bug fixes: + * Immediately remove existing connection to Discovery Controllers (DC) discovered through zeroconf (mDNS) when added to `exclude=` in `stafd.conf`. Previously, adding DCs to `exclude=` would only take effect on new connections and would not apply to existing connections. + * When handling "key=value" pairs in the TXT field from Avahi, "keys" need to be case insensitive. + * Strip spaces from Discovery Log Page Entries (DLPE). Some DCs may append extra spaces to DLPEs (e.g. IP addresses with trailing spaces). The kernel driver does not expect extra spaces and therefore they need to be removed. +* In `stafd.conf` and `stacd.conf`, added new configuration parameters to provide parity with `nvme-cli`: + * `nr-io-queues`, `nr-write-queues`, `nr-poll-queues`, `queue-size`, `reconnect-delay`, `ctrl-loss-tmo`, `duplicate-connect`, `disable-sqflow` +* Changes to `stafd.conf`: + * Move `persistent-connections` from the `[Global]` section to a new section named `[Discovery controller connection management]`. `persistent-connections` will still be recognized from the `[Global]` section, but will be deprecated over time. + * Add new configuration parameter `zeroconf-connections-persistence` to section `[Discovery controller connection management]`. This parameter allows one to age Discovery Controllers discovered through zeroconf (mDNS) when they are no longer reachable and should be purged from the configuration. +* Added more configuration validation to identify invalid Sections and Options in configuration files (`stafd.conf` and `stacd.conf`). +* Improve dependencies in meson build environment so that missing subprojects won't prevent distros from packaging the `nvme-stas` (i.e. needed when invoking meson with the `--wrap-mode=nodownload` option) +* Improve Read-The-Docs documentation format. + +## Changes with release 2.0 + +Because of incompatibilities between 1.1.6 and 1.2 (ref. `sticky-connections`), it was decided to skip release 1.2 and have a 2.0 release instead. Release 2.0 contains everything listed in 1.2 (below) plus the following: + +* Add support for PLEO - Port-Local Entries Only, see TP8010. + * Add new configuration parameter to stafd.conf: `pleo=[enabled|disabled]` + * This requires `libnvme` 1.2 or later although nvme-stas can still operate with 1.1 (but PLEO will not be supported). + * Although `blacklist=` is deprecated, keep supporting it for a while. + * Target `udev-rule=` at TCP connections only. + * Read-the-docs will now build directly from source (instead of using a possibly stale copy) + * More unit tests were added + * Refactored the code that handles pyudev events in an effort to fix spurious lost events. + +## ~~Changes with release 1.2~~ (never released - use 2.0 instead) + +- In `stacd.conf`, add a new configuration section, `[I/O controller connection management]`. + - This is to replace `sticky-connections` by `disconnect-scope` and `disconnect-trtypes`, which is needed so that hosts can better react to Fabric Zoning changes at the CDC. + - Add `connect-attempts-on-ncc` to control how stacd will react to the NCC bit (Not Connected to CDC). +- When the host's symbolic name is changed in `sys.conf`, allow re-issuing the DIM command (register with DC) on a `reload` signal (`systemctl reload stafd`). +- Replace `blacklist=` by `exclude=` is `stafd.conf` and `stacd.conf`. Warning: this may create an incompatibility for people that were using `blacklist=`. They will need to manually migrate their configuration files. +- Change `TID.__eq__()` and `TID.__ne__()` to recognize a TID object even when the `host-iface` is not set. This is to fix system audits where `nvme-stas` would not recognize connections made by `nvme-cli`. The TID object, or Transport ID, contains all the parameters needed to establish a connection with a controller, e.g. (`trtype`, `traddr`, `trsvcid`, `nqn`, `host-traddr`, and `host-iface`). `nvme-stas` can scan the `sysfs` (`/sys/class/nvme/`) to find exiting NVMe connections. It relies on the `address` and other attributes for that. For example the attribute `/sys/class/nvme/nvme0/address` may contain something like: `traddr=192.168.56.1,trsvcid=8009,host_iface=enp0s8`. + + `nvme-stas` always specify the `host-iface` when making connections but `nvme-cli` typically does not. Instead, `nvme-cli` relies on the routing table to select the interface. This creates a discrepancy between the `address` attribute of connections made by `nvme-cli` and those made by `nvme-stas` (i.e. `host_iface=` is missing for `nvme-cli` connections). And this results in `nvme-stas` not being able to recognize connections made by `nvme-cli`. Two solutions have been proposed to workaround this problem: + + - First, a short term solution changes `TID.__eq__()` and `TID.__ne__()` so that the `host-iface` has a lesser weight when comparing two TIDs. This way, the TID of a connection created by `nvme-cli` can be compared to the TID of a connection made with `nvme-stas` and still result in a match. The downside to this approach is that a connection made with `nvme-cli` that is going over the wrong interface (e.g. bad routing table entry), will now be accepted by `nvme-stas` as a valid connection. + - Second, a long term solution that involves a change to the kernel NVMe driver will allow being able to determine the host interface for any NVMe connections, even those made without specifying the `host-iface` parameter. The kernel driver will now expose the source address of all NVMe connections through the `sysfs`. This will be identified by the key=value pair "`src-addr=[ip-address]`" in the `address` attribute. And from the source address one can infer the actual host interface. This actually will solve the shortcomings of the "short term" solution discussed above. Unfortunately, it may take several months before this kernel addition is available in a stock Distribution OS. So, the short term solution will need to suffice for now. + +## Changes with release 1.1.6 + +- Fix issues with I/O controller connection audits + - Eliminate pcie devices from list of I/O controller connections to audit + - Add soaking timer to workaround race condition between kernel and user-space applications on "add" uevents. When the kernel adds a new nvme device (e.g. `/dev/nvme7`) and sends a "add" uevent to notify user-space applications, the attributes associated with that device (e.g. `/sys/class/nvme/nvme7/cntrltype`) may not be fully initialized which can lead `stacd` to dismiss a device that should get audited. +- Make `sticky-connections=enabled` the default (see `stacd.conf`) + +## Changes with release 1.1.5 + +- Fix issues introduced in 1.1.3 when enabling Fibre Channel (FC) support. + - Eliminate pcie devices from discovery log pages. When enabling FC, pcie was accidentally enabled as well. + - Fix I/O controller scan and detect algorithm. Again, while adding support for FC, the I/O scan & detect algorithm was modified, but we accidentally made it detect Discovery Controllers as well as I/O controllers. + + +## ~~Changes with release 1.1.4~~ USE 1.1.5 INSTEAD. + +- Fix issues for Fibre Channel (FC) support. +- Add TESTING.md + +## Changes with release 1.1.3 + +**stacd**: Add I/O controller connection audits. Audits are enabled when the configuration parameter "`sticky-connections`" is disabled. + +**stafd**: Preserve and Reload last known configuration on restarts. This is for warm restarts of the `stafd` daemon. This does not apply to system reboots (cold restarts). This is needed to avoid deleting I/O controller (IOC) connections by mistake when restarting `stafd`. It prevents momentarily losing previously acquired Discovery Log Page Entries (DLPE). Since `stacd` relies on acquired DLPEs to determine which connection should be created or deleted, it's important that the list of DLPEs survives a `stafd` restart. Eventually, after `stafd` has restarted and reconnected to all Discovery Controllers (DC), the list will get refreshed and the DLPE cache will get updated. And as the cache gets updated, `stacd` will be able to determine which connections should remain and which one should get deleted. + +**`stafd`/`stacd`**: Fixed crash caused by `stafd`/`stacd` calling the wrong callback function during the normal disconnect of a controller. There are two callback functions that can be called after a controller is disconnected, but one of them must only be called on a final disconnect just before the process (`stafd` or `stacd`) exits. The wrong callback was being called on a normal disconnect, which led the process to think it was shutting down. + +## ~~Changes with release 1.1.2~~ USE 1.1.3 INSTEAD. + +stacd: Bug fix. Check that self._cfg_soak_tmr is not None before dereferencing it. + +## Changes with release 1.1.1 + +Make `sticky-connections=disabled` the default (see `stacd.conf`) + +## Changes with release 1.1 + +- Add `udev-rule` configuration parameter to `stacd.conf`. +- Add `sticky-connections` configuration parameter to `stacd.conf`. +- Add coverage testing (`make coverage`) +- Add `make uninstall` +- To `README.md`, add mDNS troubleshooting section. + +## Changes with release 1.0.1 + +- Install staslib as pure python package instead of arch-specific. + +## Changes with release 1.0 + +- First public release following TP8009 / TP8010 ratification and publication. + +## Changes with release 0.1: + +- Initial release + diff --git a/README.md b/README.md new file mode 100644 index 0000000..b2e09be --- /dev/null +++ b/README.md @@ -0,0 +1,288 @@ +# STorage Appliance Services (STAS) + + + +[](https://github.com/psf/black) +[](https://github.com/linux-nvme/nvme-stas/releases) +[](https://GitHub.com/linux-nvme/nvme-stas/commit/) +[](https://nvme-stas.readthedocs.io/en/latest/) +[](https://codecov.io/gh/linux-nvme/nvme-stas) +[](https://www.python.org/downloads/) + +What does nvme-stas provide? + +- A Central Discovery Controller (CDC) client for Linux +- Asynchronous Event Notifications (AEN) handling +- Automated NVMe subsystem connection controls +- Error handling and reporting +- Automatic (zeroconf) and Manual configuration + +## Overview + +STAS is composed of two services, STAF and STAC, running on the Host computer. + +**STAF** - **STorage Appliance Finder**. The tasks performed by STAF include: + +- Register with the Avahi daemon for service type `_nvme-disc._tcp`. This allows STAF to locate Central or Direct Discovery Controllers (CDC, DDC) with zero-touch provisioning (ZTP). STAF also allows users to manually enter CDCs and DDCs in a configuration file (`/etc/stas/stafd.conf`) when users prefer not to use ZTP. +- Connect to discovered or configured CDCs or DDCs. +- Retrieve the list of storage subsystems using the "get log page" command. +- Maintain a cache of the discovered storage subsystems. +- Provide a D-Bus interface where 3rd party applications can retrieve the data about the Discovery Controller connections (e.g. log pages). + +**STAC** - **STorage Appliance Connector**. The tasks performed by STAC include: + +- Read the list of storage subsystems from STAF over D-Bus. +- Similar to STAF, STAC can also read a list of storage subsystems to connect to from a configuration file. +- Set up the I/O controller connections. +- Provide a D-Bus interface where 3rd party applications can retrieve data about the I/O controller connections. + + + +## Design + +**`stafd`** and **`stacd`** use the [GLib main loop](https://docs.gtk.org/glib/main-loop.html). The [GLib](https://docs.gtk.org/glib/index.html) Python module provides several low-level building blocks that are needed by **`stafd`** and **`stacd`**. In addition, many Python modules "play nice" with GLib such as `dasbus` and `pyudev`. GLib also provides additional components such as timers, signal handlers, and much more. + +**`stafd`** connects to the `avahi-daemon`, which it uses to detect Central Discovery Controllers (CDC) and Direct Discovery Controllers (DDC). When Discovery Controllers (DC) are found with Avahi's help, **`stafd`** uses `libnvme` to set up persistent connections and retrieve the discovery log pages. + +## Daemonization + +**`stafd`** and **`stacd`** are managed as `systemd` services. The following operations are supported (here showing only `stafd`, but the same operations apply to `stacd`): + +- `systemctl start stafd`. Start daemon. +- `systemctl stop stafd`. Stop daemon. The `SIGTERM` signal is used to tell the daemon to stop. +- `systemctl restart stafd`. Effectively a `stop` + `start`. +- `systemctl reload stafd`. Reload configuration. This is done in real time without restarting the daemon. The `SIGHUP` signal is used to tell the daemon to reload its configuration file. + +## Configuration + +As stated before, **`stafd`** can automatically locate discovery controllers with the help of Avahi and connect to them, and **`stacd`** can automatically set up the I/O connections to discovered storage subsystems. However, **`stafd`** and **`stacd`** can also operate in a non-automatic mode based on manually entered configuration. In other words, discovery controllers and/or storage subsystems can be entered manually. This is to provide customers with more flexibility. The configuration for each daemon is found in **`/etc/stas/stafd.conf`** and **`/etc/stas/stacd.conf`** respectively. The configuration files also provide additional parameters, such as log-level attributes used mainly for debugging purposes. + +The following configuration files are defined: + +| File | Consumer | Purpose | +| ---------------------- | ----------------- | ------------------------------------------------------------ | +| `/etc/stas/sys.conf` | `stafd` + `stacd` | Contains system-wide (i.e. host) configuration such as the Host NQN, the Host ID, and the Host Symbolic Name. Changes to this file can be made manually or with the help of the `stasadm` utility as described in the previous section. <br /><br />For example, `stasadm hostnqn -f /etc/nvme/hostnqn` writes the Host NQN to the file `/etc/nvme/hostnqn`, but also adds an entry to `/etc/stas/sys.conf` to indicate where the Host NQN has been saved. <br /><br />This gives nvme-stas the flexibility of defining its own Host parameters or to use the same parameters defined by `libnvme` and `nvme-cli`. | +| `/etc/stas/stafd.conf` | `stafd` | Contains configuration specific to `stafd`. Discovery controllers can be manually added or excluded in this file. | +| `/etc/stas/stacd.conf` | `stacd` | Contains configuration specific to `stacd`. I/O controllers can be manually added or excluded in this file. | + +## D-Bus interface + +The interface to **`stafd`** and **`stacd`** is D-Bus. This allows other programs, such as **`stafctl`** and **`stacctl`**, to communicate with the daemons. This also provides third parties the ability to write their own applications that can interact with **`stafd`** and **`stacd`**. For example, someone could decide to write a GUI where they would display the discovery controllers as well as the all the discovery log pages in a "pretty" window. The next table provides info about the two D-Bus interfaces. + +| Component | D-Bus address | +| --------- | ------------------------------ | +| `stafd` | **`org.nvmexpress.staf.conf`** | +| `stacd` | **`org.nvmexpress.stac.conf`** | + +## Companion programs: `stafctl` and `stacctl` + +**`stafctl`** and **`stacctl`** are utilities that allow users to interact with **`stafd`** and **`stacd`** respectively. This is a model used by several programs, such as `systemctl` with `systemd`. + +At a minimum, these utilities provide debug tools, but they could also provide some configuration capabilities (TBD). + +## Packages + +**`stafd`** and **`stacd`** as well as their companion programs **`stafctl`** and **`stacctl`** are released together in a package called "**`nvme-stas`**" for **ST**orage **A**pplicance **S**ervices (e.g. `stas-1.0.0-1.x86_64.rpm` or `stas_1.0.0_amd64.deb`). + +## Dependencies + + **`stafd`**/**`stacd`** require Linux kernel 5.14 or later. + +The following packages must be installed to use **`stafd`**/**`stacd`** + +**Debian packages (tested on Ubuntu 20.04):** + +```bash +sudo apt-get install -y python3-pyudev python3-systemd python3-gi +sudo apt-get install -y python3-dasbus # Ubuntu 22.04 +OR: +sudo pip3 install dasbus # Ubuntu 20.04 +``` + +**RPM packages (tested on Fedora 34..35 and SLES15):** + +```bash +sudo dnf install -y python3-dasbus python3-pyudev python3-systemd python3-gobject +``` + +# STAF - STorage Appliance Finder + + +| Component | Description | +| --------------- | -------------------------------------------------------- | +| **`/usr/sbin/stafd`** | A daemon that finds (discovers) NVMe storage appliances. | +| **`/usr/bin/stafctl`** | A companion shell utility for `stafd`. | +| **`/etc/stas/stafd.conf`** | Configuration file | + +## stafd configuration file + +The configuration file is named `/etc/stas/stafd.conf`. This file contains configuration parameters for the **`stafd`** daemon. One of the things you may want to configure is the IP address of the discovery controller(s) you want **`stafd`** to connect to. The configuration file contains a description of all the parameters that can be configured. + +## Service discovery with Avahi + +**`stafd`** can automatically find and set up connections to Discovery Controllers. To do this, **`stafd`** registers with the [Avahi](https://www.avahi.org/), the mDNS/DNS-SD (Service Discovery) daemon. Discovery Controllers that advertise themselves with service type `_nvme-disc._tcp` will be recognized by Avahi, which will inform **`stafd`**. + +### Not receiving mDNS packets? + +If **`stafd`** is not detecting any discovery controllers through Avahi, it could simply be that the mDNS packets are being suppressed by your firewall. If you know for a fact that the discovery controllers are advertizing themselves with mDNS packets, make sure that the Avahi daemon is receiving them as follows: + +```bash +avahi-browse -t -r _nvme-disc._tcp +``` + +If you're not seeing anything, then check whether your firewall allows mDNS packets. + +### Why is Avahi failing to discover services on some interfaces? + +Linux limits the number of multicast group memberships that a host can belong to. The default is 20. For Avahi to monitor mDNS (multicast DNS) packets on all interfaces, the host computer must be able to register one multicast group per interface. This can be physical or logical interfaces. For example, configuring 10 VLANs on a physical interface increases the total number of interfaces by 10. If the total number of interfaces is greater than the limit of 20, then Avahi won't be able to monitor all interfaces. + +The limit can be changed by configuring the variable **`igmp_max_memberships`**. This variable is defined [here](https://sysctl-explorer.net/net/ipv4/igmp_max_memberships/) in the kernel documentation. And this [StackExchange page](https://unix.stackexchange.com/questions/23832/is-there-a-way-to-increase-the-20-multicast-group-limit-per-socket) describes how one can increase the limit. + +# STAC - STorage Appliance Connector + + +| File name | Description | +| -------------------------- | -------------------------------------------------- | +| **`/usr/sbin/stacd`** | A daemon that connects to NVMe storage appliances. | +| **`/usr/bin/stacctl`** | A companion shell utility for `stacd`. | +| **`/etc/stas/stacd.conf`** | Configuration file | + +## stacd configuration file + +The configuration file is named `/etc/stas/stacd.conf`. In this file you can configure storage appliances that **`stacd`** will connect to. By default, **`stacd`** uses information (log pages) collected from **`stafd`** to connect to storage appliances. However, you can also manually enter IP addresses of storage appliances in this file. + +# System configuration + +A host must be provided with a Host NQN and a Host ID. `nvme-stas` will not run without these two mandatory configuration parameters. To follow in the footsteps of `nvme-cli` and `libnvme`, `nvme-stas` will use the same Host NQN and ID that `nvme-cli` and `libnvme` use by default. In other words, `nvme-stas` will read the Host NQN and ID from these two files by default: + +1. `/etc/nvme/hostnqn` +2. `/etc/nvme/hostid` + +Using the same configuration files will ensure consistency between `nvme-stas`, `nvme-cli`, and `libnvme`. On the other hand, `nvme-stas` can operate with a different Host NQN and/or ID. In that case, one can specify them in `/etc/stas/sys.conf`. + +A new optional configuration parameters introduced in TP8010, the Host Symbolic Name, can also be specified in `/etc/stas/sys.conf`. The schema/documentation for `/etc/stas/sys.conf` can be found [`/etc/stas/sys.conf.doc`](./etc/stas/sys.conf.doc). + +# Build, install, unit tests + +STAS uses the `meson` build system. Since STAS is a Python project, there is no code to build. However, the code needs to be installed using `meson`. Unit tests can also be run with `meson`. + +## Using meson + +Invoke `meson` to configure the project: + +```bash +meson .build +``` + +The command `meson .build` need only be called once. This analyzes the project and the host computer to determine if all the necessary tools and dependencies are available. The result is saved to the directory named `.build`. + +To compile the code: + +```bash +cd .build +ninja +``` + +To install / uninstall the code: + +```bash +cd .build +meson install +ninja uninstall +``` + +To run the unit tests: + +```bash +cd .build +meson test +``` + +For more information about testing, please refer to: [TESTING.md](./TESTING.md) + +## Alternate approach using Good-ole make + +Recognizing that many people are not familiar with `meson`, we're providing a second way to install the code using the more familiar `configure` script combined with a `make`. + +```bash +./configure +make +``` + +This performs the same operations as the meson approach described above. The `configure` script is automatically invoked when running `make` by itself. + +| make command | Description | +| -------------------- | :----------------------------------------------------------- | +| **`make`** | Invoke the `.configure` script and build the code. | +| **`make install`** | Install the code. Requires root privileges (you will be asked to enter your password). | +| **`make uninstall`** | Uninstall the code. Requires root privileges (you will be asked to enter your password). | +| **`make test`** | Run the unit tests | +| **`make clean`** | Clean build artifacts, but does not remove the meson's configuration. That is, the configuration in `.build` is preserved. | +| **`make purge`** | Remove all build artifacts including the `.build` directory. | + +## Compiling and running nvme-stas in a docker container + +Use published image (optional) +```bash +docker pull ghcr.io/linux-nvme/nvme-stas:main +``` + +Build your own image (optional) +```bash +docker-compose up --build +``` + +Run services using docker-compose like this +```bash +docker-compose up +``` + +Run companion programs **`stafctl`** and **`stacctl`** like this +```bash +docker-compose exec stafd stafctl ls +docker-compose exec stafd stafctl status + +docker-compose exec stacd stacctl ls +docker-compose exec stacd stacctl status +``` + +dependencies: dbus, avahi. + +## Generating man and html pages + +nvme-stas uses the following programs to generate the documentation. These can be installed as shown in the "dependencies" section below. + +- `xsltproc` - Used to convert DocBook XML notation to "man pages" and "html pages". +- `gdbus-codegen` - Used to convert D-Bus IDL to DocBook XML notation. + +### Dependencies + +The following packages must be installed to generate the documentation + +**Debian packages (tested on Ubuntu 20.04):** + +```bash +sudo apt-get install -y docbook-xml docbook-xsl xsltproc libglib2.0-dev +``` + +**RPM packages (tested on Fedora 34..35 and SLES15):** + +```bash +sudo dnf install -y docbook-style-xsl libxslt glib2-devel +``` + +### Configuring and building the man and html pages + +By default, the documentation is not built. You need to run the `configure` as follows to tell meson that you want to build the documentation. You may need to first purge any previous configuration. + +```bash +make purge +./configure -Dman=true -Dhtml=true +make +``` + +## Generating RPM and/or DEB packages +```bash +make rpm +make deb +``` diff --git a/TESTING.md b/TESTING.md new file mode 100644 index 0000000..0fa928f --- /dev/null +++ b/TESTING.md @@ -0,0 +1,221 @@ +--- +author: Martin Belanger +title: Testing nvme-stas +--- + +# Overview + +For quick an easy testing, it's possible to run a storage subsystem simulator using the `nvmet` driver. This is how most of the testing was done during `nvme-stas` development. The main feature that cannot be tested this way is mDNS discovery. + +There are two ways to run the tests. + +- The first one involves starting all the components manually and using the nvmet driver as the storage appliance. +- The second one is fully automated and can be invoked simply by running `make coverage`. + +[toc] + +# Manual testing using the nvmet driver + +A script is provided (`utils/nvmet/nvmet.py`) to simplify the configuration of the `nvmet` driver. The script comes with a companion configuration file (`utils/nvmet/nvmet.conf`). The configuration file is where you configure the port(s) and subsystem(s) to create. The default configuration will create 3 subsystems under port 1. This is mapped to the local IPv6 loopback address (`::1`). + +Since nvmet doesn't provide a mDNS responder, you will need to manually configure `stafd` (`/etc/stas/stafd.conf`) so that it connects to the DDC that the nvmet driver creates by adding the DDC's address under the `[Controllers]` section. For example: + +```bash +[Controllers] +controller=transport=tcp;traddr=localhost +``` + +## Monitoring + +While testing it's a good idea to follow the journal in real time to see how `stafd` and `stacd` are performing. In a terminal (e.g. `bash`) do: + +```bash +$ sudo journalctl --system --full -o short-precise --follow +``` + +You probably don't really need all these options, but they will give you full view of the messages with a millisecond time resolution. I personally define an alias `alias j='sudo journalctl --system --full -o short-precise'` and then I need only invoke `j -f`. Or even better, I add my user to the `systemd-journal` group so that I don't have to use `sudo` to see system-level log messages (Ref: [systemd-journal.service](https://www.freedesktop.org/software/systemd/man/systemd-journald.service.html#Access%20Control)). + +## Test startup + +Here's a step-by-step guide to start `stafd` and `stacd` and connect to the `nvmet` driver. Open a second terminal and enter the following commands (these commands assume that nvme-stas will be cloned under `~/work/nvme-stas` referred to as `$STAS_DIR`): + +### Clone nvme-stas (if not done already) + +```bash +$ mkdir ~/work +$ cd ~/work +$ git clone https://github.com/linux-nvme/nvme-stas.git +$ STAS_DIR=~/work/nvme-stas +``` + +### Build and install nvme-stas + +```bash +$ cd $STAS_DIR +$ make install +``` + +### Create a post-install script + +Create an executable shell script (call it `stas-config.sh`) with the following contents. These are post-installation configuration steps required every time `nvme-stas` is reinstalled. Place the script in a directory that is in the search `$PATH` so that it can be invoked easily. + +```bash +#!/usr/bin/env bash +##################################################################### +# Must run daemon-reload after installing nvme-stas +sudo systemctl daemon-reload + +##################################################################### +# Make sure Host NQN and ID are configured +if [ ! -d "/etc/nvme" ]; then + sudo mkdir /etc/nvme +fi + +if [ ! -s /etc/nvme/hostnqn ]; then + sudo stasadm hostnqn -f /etc/nvme/hostnqn +fi + +if [ ! -s /etc/nvme/hostid ]; then + sudo stasadm hostid -f /etc/nvme/hostid +fi + +##################################################################### +# Edit /etc/stas/stafd.conf to enable tracing and add the local +# nvmet driver as the Discovery Controller to connect to. +FILES="stafd.conf stacd.conf" +for file in ${FILES}; do + sudo sed -i '/^#tron=false/a tron=true' /etc/stas/${file} +done +sudo sed -i '/^#controller=$/a controller=transport=tcp;traddr=localhost' /etc/stas/stafd.conf + +``` + +### Run the post-install script + +```bash +$ stas-config.sh +``` + +### Start the nvmet driver + +```bash +$ cd $STAS_DIR/utils/nvmet +$ sudo ./nvmet.py create +``` + +### Start stafd and stacd + +```bash +$ sudo systemctl start stafd stacd +``` + +## So, is it running yet? + +You should have seen `stafd` and `stacd` starting in the first terminal where `journalctl` is following the system log. At this point `stafd` should have connected to the `nvmet` discovery controller and retrieved the discovery log page entries (DLPE). And `stacd` should have retrieved the DLPEs from `stafd` and connected to the 3 subsystems defined in `nvmet.conf`. This can be confirmed as follows: + +```bash +$ stafctl ls +[{'device': 'nvme0', + 'host-iface': '', + 'host-traddr': '', + 'subsysnqn': 'nqn.2014-08.org.nvmexpress.discovery', + 'traddr': '::1', + 'transport': 'tcp', + 'trsvcid': '8009'}] +``` + +And: + +```bash +$ stacctl ls +[{'device': 'nvme1', + 'host-iface': '', + 'host-traddr': '', + 'subsysnqn': 'klingons', + 'traddr': '::1', + 'transport': 'tcp', + 'trsvcid': '8009'}, + {'device': 'nvme2', + 'host-iface': '', + 'host-traddr': '', + 'subsysnqn': 'nqn.1988-11.com.dell:PowerSANxxx:01:20210225100113-454f73093ceb4847a7bdfc6e34ae8e28', + 'traddr': '::1', + 'transport': 'tcp', + 'trsvcid': '8009'}, + {'device': 'nvme3', + 'host-iface': '', + 'host-traddr': '', + 'subsysnqn': 'starfleet', + 'traddr': '::1', + 'transport': 'tcp', + 'trsvcid': '8009'}] +``` + +You can also use `nvme-cli` to list the connections. For example: `nvme list -v`. + +## Generating Asynchronous Event Notifications (AEN) + +You can use the `nvmet.py` script to simulate the removal of a subsystem, which results in an AEN being sent to indicate a "Change of Discovery Log Page". Here's how: + +```bash +$ cd $STAS_DIR/utils/nvmet +$ sudo ./nvmet.py unlink -p 1 -s klingons +``` + +Observe what happens in the journal. `stafd` will receive the AEN and update the DLPEs by performing a Get Discovery Log Page command. And `stacd` will disconnect from the "`klingons`" subsystem (use `stacctl ls` to confirm). + +Then, add the subsystem back as follows: + +```bash +$ sudo ./nvmet.py link -p 1 -s klingons +``` + +**NOTE**: I know, "`klingons`" is not a valid NQN, but it sure is easier to remember and to type than a valid NQN. Fortunately, the `nvmet` driver doesn't care what the actual subsystem's NQN looks like. :smile: + +## Stopping nvmet + +```bash +$ cd $STAS_DIR/utils/nvmet +$ sudo ./nvmet.py clean +``` + +# Automated testing using the coverage test + +This requires the [Python coverage package](https://coverage.readthedocs.io/en/6.4.1/), which can be installed as follows: + +```bash +$ sudo pip install coverage +``` + +Note that this test cannot be run while `stafd` and `stacd` are running. Make sure to stop `stafd` and `stacd` if they are running (`systemctl stop [stafd|stacd]`). You may also need to mask those services (`systemctl mask [stafd|stacd]`) if coverage fails to start. + +To run the coverage test, from the root of the `nvme-stas` git repo: + +```bash +$ make coverage +``` + +This will start `stafd`, `stacd`, and the `nvmet` target. At the end, if all goes well, you should get an output similar to this: + +```bash +Name Stmts Miss Cover +---------------------------------------- +stacctl 53 0 100% +stacd 190 3 98% +stafctl 75 0 100% +stafd 246 21 91% +staslib/avahi.py 185 19 90% +staslib/defs.py 22 0 100% +staslib/stas.py 858 51 94% +staslib/version.py 31 0 100% +---------------------------------------- +TOTAL 1660 94 94% +``` + +Note that the Python coverage package has trouble tracking code executed in threads. And since nvme-stas uses threads, some of the code will not be accounted for (in other words, you'll never get 100% coverage). + +Also note, that some of the code (e.g. explicit registration per TP8010) only gets executed when connected to a CDC (not a DDC). So, depending on your environment you will most likely get different coverage result. The above test was done on a system where mDNS discovery with a CDC was available, which provides more coverage than using the `nvmet` driver alone. + +An HTML output is also available where you can click on each file and which lines of code got executed and which ones were missed. In your web browser, simply type `file:///[$STAS_DIR]/.build/coverage/index.html` (you must replace `[$STAS_DIR]` by the actual location of the nvme-stas repo where `make coverage` was run) . You should get something like this: + +
\ No newline at end of file diff --git a/configure b/configure new file mode 100755 index 0000000..c221a28 --- /dev/null +++ b/configure @@ -0,0 +1,17 @@ +#!/bin/bash -e +# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). +# +# Authors: Martin Belanger <Martin.Belanger@dell.com> +# +BUILD_DIR="${BUILD_DIR:-.build}" + +if [ ! -d ${BUILD_DIR} ]; then + exec meson ${BUILD_DIR} "$@" +else + exec meson configure ${BUILD_DIR} "$@" +fi + diff --git a/coverage.sh.in b/coverage.sh.in new file mode 100755 index 0000000..b75ae5c --- /dev/null +++ b/coverage.sh.in @@ -0,0 +1,653 @@ +#!/usr/bin/env bash + +PRIMARY_GRP=$( id -ng ) +PRIMARY_USR=$( id -nu ) +PYTHON_PATH=.:./subprojects/libnvme +AVAHI_PUBLISHER=mdns_publisher.service + +file=/tmp/stafd.conf.XXXXXX +stafd_conf_fname=$(mktemp $file) + +file=/tmp/stacd.conf.XXXXXX +stacd_conf_fname=$(mktemp $file) + +CYAN="[1;36m" +RED="[1;31m" +YELLOW="[1;33m" +NORMAL="[0m" + +log() { + msg="$1" + printf "%b%s%s%b[0m\n" "\0033" ${CYAN} "${msg}" "\0033" + sudo logger -t COVERAGE -i "@@@@@ " -p warning -- "${msg}" +} + +log_file_contents() { + rc=$1 + file=$2 + + if [ $rc -eq 0 ]; then + color=${NORMAL} + level="info" + else + color=${YELLOW} + level="error" + fi + + while IFS= read -r line; do + msg=" ${line}" + printf "%b%s%s%b[0m\n" "\0033" ${color} "${msg}" "\0033" + sudo logger -t COVERAGE -i "@@@@@ " -p ${level} -- "${msg}" + done < ${file} +} + +systemctl-exists() { + unit="$1" + [ $(systemctl list-unit-files "${unit}" | wc -l) -gt 3 ] +} + +sd_stop() { + app="$1" + unit="${app}"-cov.service + if systemctl-exists "${unit}" >/dev/null 2>&1; then + log "Stop ${app}" + sudo systemctl stop "${unit}" >/tmp/output.txt 2>&1 + if [ -s /tmp/output.txt ]; then + log_file_contents $? /tmp/output.txt + else + printf " sudo systemctl stop %s\n" "${unit}" + fi + sudo systemctl reset-failed "${unit}" >/dev/null 2>&1 + printf "\n" + sleep 1 + fi +} + +sd_start() { + app="$1" + dbus="$2" + conf="$3" + unit="${app}"-cov.service + + if [ -z "${conf}" ]; then + cmd="${app} --syslog" + else + cmd="${app} --syslog -f ${conf}" + fi + + RUNTIME_DIRECTORY=/tmp/${app} + rm -rf ${RUNTIME_DIRECTORY} + mkdir ${RUNTIME_DIRECTORY} + + # Clear previous failure status (if any) + sudo systemctl reset-failed "${unit}" >/dev/null 2>&1 + + log "Start ${app}" + sudo systemd-run --unit="${unit}" --working-directory=. --property=Type=dbus --property=BusName="${dbus}" --property="SyslogIdentifier=${app}" --property="ExecReload=/bin/kill -HUP \$MAINPID" --setenv=PYTHONPATH=${PYTHON_PATH} --setenv=RUNTIME_DIRECTORY=${RUNTIME_DIRECTORY} coverage run --rcfile=.coveragerc ${cmd} >/tmp/output.txt 2>&1 + log_file_contents $? /tmp/output.txt + printf "\n" + sleep 1 +} + +sd_restart() { + app="$1" + unit="${app}"-cov.service + + if systemctl is-active "${unit}" >/dev/null 2>&1; then + log "Restart ${app}" + sudo systemctl restart "${unit}" && printf "systemctl restart %s\n" "${unit}" >/tmp/output.txt 2>&1 + log_file_contents $? /tmp/output.txt + sleep 1 + else + msg="Cannot restart ${app}, which is not currently running." + printf "%b%s%s%b[0m\n\n" "\0033" ${RED} "${msg}" "\0033" + fi + printf "\n" +} + +reload_cfg() { + app="$1" + unit="${app}"-cov.service + log "Reload config ${app}" + sudo systemctl reload "${unit}" && printf "systemctl reload %s\n" "${unit}" >/tmp/output.txt 2>&1 + #pid=$( systemctl show --property MainPID --value "${unit}" ) + #sudo kill -HUP "${pid}" >/tmp/output.txt 2>&1 + log_file_contents $? /tmp/output.txt + printf "\n" + sleep 1 +} + +run_unit_test() { + input=$@ + if [ "$1" == "sudo" ]; then + shift + COVERAGE="sudo coverage" + else + COVERAGE="coverage" + fi + test=$@ + log "Run unit test: ${input}" + PYTHONPATH=${PYTHON_PATH} ${COVERAGE} run --rcfile=.coveragerc ../test/${test} >/dev/null 2>&1 +} + +run_cmd_coverage() { + input=$@ + if [ "$1" == "sudo" ]; then + shift + COVERAGE="sudo coverage" + else + COVERAGE="coverage" + fi + cmd="$@" + log "Invoke: ${input}" + ${COVERAGE} run --rcfile=.coveragerc ${cmd} >/tmp/output.txt 2>&1 + log_file_contents $? /tmp/output.txt + printf "\n" +} + +run_cmd() { + cmd="$@" + ${cmd} >/tmp/output.txt 2>&1 + if [ -s /tmp/output.txt ]; then + log_file_contents $? /tmp/output.txt + else + printf " %s\n" "${cmd}" + fi +} + +prerun_setup() { + if [ ! -d coverage ]; then + mkdir coverage + fi + + for file in staf stac; do + if [ ! -f "/usr/share/dbus-1/system.d/org.nvmexpress.${file}.conf" -a \ + ! -f "/etc/dbus-1/system.d/org.nvmexpress.${file}.conf" ]; then + log "hardlink /etc/dbus-1/system.d/org.nvmexpress.${file}.conf -> @BUILD_DIR@/etc/dbus-1/system.d/org.nvmexpress.${file}.conf" + sudo ln @BUILD_DIR@/etc/dbus-1/system.d/org.nvmexpress.${file}.conf /etc/dbus-1/system.d/org.nvmexpress.${file}.conf + if [ $? -ne 0 ]; then + log "hardlink failed" + exit 1 + fi + fi + done + sudo systemctl reload dbus.service +} + +postrun_cleanup() { + sd_stop "stafd" + sd_stop "stacd" + + log "Stop nvmet" + sudo ../utils/nvmet/nvmet.py clean >/tmp/output.txt 2>&1 + log_file_contents $? /tmp/output.txt + printf "\n" + + log "nvme disconnect-all" + run_cmd sudo nvme disconnect-all + printf "\n" + + log "Remove ${stafd_conf_fname} and ${stacd_conf_fname}" + rm "${stafd_conf_fname}" + rm "${stacd_conf_fname}" + printf "\n" + + for file in staf stac; do + if [ -f "/etc/dbus-1/system.d/org.nvmexpress.${file}.conf" ]; then + if [ "$(stat -c %h -- "/etc/dbus-1/system.d/org.nvmexpress.${file}.conf")" -gt 1 ]; then + log "Remove hardlink /etc/dbus-1/system.d/org.nvmexpress.${file}.conf" + sudo rm "/etc/dbus-1/system.d/org.nvmexpress.${file}.conf" + fi + fi + done + sudo systemctl reload dbus.service + + sudo systemctl unmask avahi-daemon.service + sudo systemctl unmask avahi-daemon.socket + sudo systemctl start avahi-daemon.service + sudo systemctl start avahi-daemon.socket + + sudo systemctl stop ${AVAHI_PUBLISHER} >/dev/null 2>&1 + sudo systemctl reset-failed ${AVAHI_PUBLISHER} >/dev/null 2>&1 + + log "All done!!!" + log "FINISHED-FINISHED-FINISHED-FINISHED-FINISHED-FINISHED-FINISHED-FINISHED" +} + +trap postrun_cleanup EXIT +trap postrun_cleanup SIGINT + +################################################################################ +################################################################################ +################################################################################ + +log "START-START-START-START-START-START-START-START-START-START-START-START" + +if systemctl is-active stafd.service >/dev/null 2>&1 || systemctl is-active stacd.service >/dev/null 2>&1; then + msg="Stopping because stafd and/or stacd is/are currently running." + printf "%b%s%s%b[0m\n" "\0033" ${RED} "${msg}" "\0033" + exit 1 +fi + +prerun_setup + +#******************************************************************************* +# Load nvme kernel module +log "modprobe nvme_tcp" +run_cmd sudo /usr/sbin/modprobe nvme_tcp + +log "nvme disconnect-all" +run_cmd sudo nvme disconnect-all +printf "\n" + +sd_stop stafd # make sure it's not running already +sd_stop stacd # make sure it's not running already + +#******************************************************************************* +# Create a dummy config file for stafd +log "Create dummy config file ${stafd_conf_fname}" +cat > "${stafd_conf_fname}" <<'EOF' +[Global] +tron = true +ip-family = ipv6 +johnny = be-good +queue-size = 2000000 +reconnect-delay = NaN +ctrl-loss-tmo = 10 +disable-sqflow = true + +[Discovery controller connection management] +persistent-connections = false +zeroconf-connections-persistence = -1 + +[Hello] +hello = bye +EOF +log_file_contents 0 "${stafd_conf_fname}" +printf "\n" + +#******************************************************************************* +# Create a dummy config file for stacd +log "Create dummy config file ${stacd_conf_fname}" +cat > "${stacd_conf_fname}" <<'EOF' +[Global] +tron = true +kato = 10 +nr-io-queues = 4 +nr-write-queues = NaN +nr-poll-queues = NaN +queue-size = 2000000 +reconnect-delay = 1 +ctrl-loss-tmo = 1 +disable-sqflow = true + +[I/O controller connection management] +disconnect-scope = blah-blah +disconnect-trtypes = boing-boing +EOF +log_file_contents 0 "${stacd_conf_fname}" +printf "\n" + +log "Stop & Mask Avahi daemon" +run_cmd sudo systemctl stop avahi-daemon.service +run_cmd sudo systemctl stop avahi-daemon.socket +run_cmd sudo systemctl mask avahi-daemon.service +run_cmd sudo systemctl mask avahi-daemon.socket +printf "\n" +sleep 1 + +log ">>>>>>>>>>>>>>>>>>>>> Marker [1] <<<<<<<<<<<<<<<<<<<<<" +printf "\n" + +run_cmd_coverage stafctl ls +run_cmd_coverage stafctl invalid-command +run_cmd_coverage stacctl ls +run_cmd_coverage stacctl invalid-command + +#******************************************************************************* +# Start nvme target simulator +log "Start nvmet" +sudo ../utils/nvmet/nvmet.py clean >/dev/null 2>&1 +sudo ../utils/nvmet/nvmet.py create -f ../utils/nvmet/nvmet.conf >/tmp/output.txt 2>&1 +log_file_contents $? /tmp/output.txt +printf "\n" + +sleep 2 + +log ">>>>>>>>>>>>>>>>>>>>> Marker [2] <<<<<<<<<<<<<<<<<<<<<" +printf "\n" + +#******************************************************************************* +# Start stafd and stacd +sd_start "stafd" "@STAFD_DBUS_NAME@" "${stafd_conf_fname}" +sd_start "stacd" "@STACD_DBUS_NAME@" "${stacd_conf_fname}" +sleep 2 + +run_cmd_coverage stafctl status + +reload_cfg "stafd" +sleep 1 + +log "Restart Avahi daemon" +run_cmd sudo systemctl unmask avahi-daemon.socket +run_cmd sudo systemctl unmask avahi-daemon.service +run_cmd sudo systemctl start avahi-daemon.socket +run_cmd sudo systemctl start avahi-daemon.service +printf "\n" +sleep 2 + +log ">>>>>>>>>>>>>>>>>>>>> Marker [3] <<<<<<<<<<<<<<<<<<<<<" +printf "\n" + +log "Change stafd config [1]:" +cat > "${stafd_conf_fname}" <<'EOF' +[Global] +tron = true + +[Discovery controller connection management] +persistent-connections = false +zeroconf-connections-persistence = 0.5 + +[Service Discovery] +zeroconf = enabled +EOF +log_file_contents 0 "${stafd_conf_fname}" +printf "\n" + +reload_cfg "stafd" +sleep 1 + +log "Change stafd config [2]:" +cat > "${stafd_conf_fname}" <<'EOF' +[Global] +tron = true +ip-family = ipv4 +queue-size = 2000000 +reconnect-delay = 1 +ctrl-loss-tmo = 1 +disable-sqflow = true +pleo = disable + +[Discovery controller connection management] +persistent-connections = false +zeroconf-connections-persistence = 1:01 + +[Controllers] +controller = transport = tcp ; traddr = localhost ; ; ; kato=31; dhchap-ctrl-secret=not-so-secret +controller=transport=tcp;traddr=1.1.1.1 +controller=transport=tcp;traddr=100.100.100.100 +controller=transport=tcp;traddr=2607:f8b0:4002:c2c::71 + +exclude=transport=tcp;traddr=1.1.1.1 +EOF +log_file_contents 0 "${stafd_conf_fname}" +printf "\n" + +reload_cfg "stafd" +sleep 5 + + +log "Change stacd config [1]:" +cat > "${stacd_conf_fname}" <<'EOF' +[Global] +tron=true +nr-io-queues=4 +nr-write-queues=4 +queue-size=2000000 +reconnect-delay=1 +ctrl-loss-tmo=1 +disable-sqflow=true + +[I/O controller connection management] +disconnect-scope=all-connections-matching-disconnect-trtypes +disconnect-trtypes=tcp+rdma +EOF +log_file_contents 0 "${stacd_conf_fname}" +printf "\n" + +reload_cfg "stacd" +sleep 5 + +log ">>>>>>>>>>>>>>>>>>>>> Marker [4] <<<<<<<<<<<<<<<<<<<<<" +printf "\n" + +run_cmd_coverage stafctl status + +#******************************************************************************* +# Fake mDNS packets from a CDC +log "Start Avahi publisher" +run_cmd sudo systemctl stop ${AVAHI_PUBLISHER} +run_cmd sudo systemctl reset-failed ${AVAHI_PUBLISHER} +run_cmd sudo systemd-run --unit=${AVAHI_PUBLISHER} --working-directory=. avahi-publish -s SFSS _nvme-disc._tcp 8009 "p=tcp" +printf "\n" +sleep 1 + +#******************************************************************************* +run_cmd_coverage stafd --version +run_cmd_coverage stacd --version + +#******************************************************************************* +# Stimulate D-Bus activity +run_cmd_coverage sudo stafctl --version +run_cmd_coverage sudo stafctl blah +run_cmd_coverage sudo stafctl troff +run_cmd_coverage stafctl status +run_cmd_coverage sudo stafctl tron +run_cmd_coverage stafctl ls -d +run_cmd_coverage stafctl adlp -d +run_cmd_coverage stafctl dlp -t tcp -a ::1 -s 8009 + +run_cmd_coverage sudo stacctl --version +run_cmd_coverage sudo stacctl blah +run_cmd_coverage sudo stacctl troff +run_cmd_coverage stacctl status +run_cmd_coverage sudo stacctl tron +run_cmd_coverage stacctl ls -d + +log ">>>>>>>>>>>>>>>>>>>>> Marker [5] <<<<<<<<<<<<<<<<<<<<<" +printf "\n" + +#******************************************************************************* +# Stimulate AENs activity by removing/restoring namespaces +log "Remove namespace: klingons" +run_cmd sudo ../utils/nvmet/nvmet.py unlink -p 1 -s klingons +printf "\n" +sleep 2 +run_cmd_coverage stacctl ls + +log "Restore namespace: klingons" +run_cmd sudo ../utils/nvmet/nvmet.py link -p 1 -s klingons +printf "\n" +sleep 2 +run_cmd_coverage stacctl ls + +log ">>>>>>>>>>>>>>>>>>>>> Marker [6] <<<<<<<<<<<<<<<<<<<<<" +printf "\n" + +#******************************************************************************* +# Stop Avahi Publisher +log "Stop Avahi publisher" +run_cmd sudo systemctl stop ${AVAHI_PUBLISHER} +printf "\n" +sleep 1 + +#******************************************************************************* +log "Restart Avahi publisher" +run_cmd sudo systemd-run --unit=${AVAHI_PUBLISHER} --working-directory=. avahi-publish -s SFSS _nvme-disc._tcp 8009 "p=tcp" +printf "\n" +sleep 2 + +log ">>>>>>>>>>>>>>>>>>>>> Marker [7] <<<<<<<<<<<<<<<<<<<<<" +printf "\n" + +#******************************************************************************* +# Make config changes for stafd +log "Change stafd config [3]:" +cat > "${stafd_conf_fname}" <<'EOF' +[Global] +tron = true +queue-size = 2000000 +reconnect-delay = 1 +ctrl-loss-tmo = 1 +disable-sqflow = true + +[Discovery controller connection management] +persistent-connections=false +zeroconf-connections-persistence=0.5 + +[Service Discovery] +zeroconf=disabled +EOF +log_file_contents 0 "${stafd_conf_fname}" +printf "\n" + +reload_cfg "stafd" +sleep 3 + +#******************************************************************************* +# Make more config changes for stafd +log "Change stafd config [4]:" +cat > "${stafd_conf_fname}" <<'EOF' +[Global] +tron=true +queue-size=2000000 +reconnect-delay=1 +ctrl-loss-tmo=0 +disable-sqflow=true +ip-family=ipv6 + +[Discovery controller connection management] +persistent-connections=false +zeroconf-connections-persistence=0 + +[Controllers] +controller=transport=tcp;traddr=localhost;trsvcid=8009 +controller=transport=tcp;traddr=abracadabra +controller=transport=tcp;traddr=google.com +controller= +controller=trsvcid +controller=transport=rdma;traddr=!@#$ +controller=transport=fc;traddr=21:00:00:00:00:00:00:00;host-traddr=20:00:00:00:00:00:00:00 +controller=transport=XM;traddr=2.2.2.2 +controller=transport=tcp;traddr=555.555.555.555 +EOF +log_file_contents 0 "${stafd_conf_fname}" +printf "\n" + +log ">>>>>>>>>>>>>>>>>>>>> Marker [8] <<<<<<<<<<<<<<<<<<<<<" +printf "\n" + +reload_cfg "stafd" +sleep 2 + +#******************************************************************************* +# Stop Avahi Publisher +log "Stop Avahi publisher" +run_cmd sudo systemctl stop ${AVAHI_PUBLISHER} +printf "\n" +sleep 2 + +log ">>>>>>>>>>>>>>>>>>>>> Marker [9] <<<<<<<<<<<<<<<<<<<<<" +printf "\n" + +#******************************************************************************* +# Remove one of the NVMe device's +file=/tmp/getdev-XXX.py +getdev=$(mktemp $file) +cat > "${getdev}" <<'EOF' +import sys +from dasbus.connection import SystemMessageBus + +bus = SystemMessageBus() +iface = bus.get_proxy(sys.argv[1], sys.argv[2]) +controllers = iface.list_controllers(False) +if len(controllers) > 0: + controller = controllers[0] + print(controller['device']) + sys.exit(0) +sys.exit(1) +EOF + +# Find a Discovery Controller and issue a "nvme disconnect" +if dev=$(python3 ${getdev} @STAFD_DBUS_NAME@ @STAFD_DBUS_PATH@); then + log "Remove connection (disconnect) to Discovery Controller ${dev}" + run_cmd sudo nvme disconnect -d ${dev} + printf "\n" +else + msg="Failed to find a connection to a Discovery Controller" + printf "%b%s%s%b[0m\n" "\0033" ${RED} "${msg}" "\0033" + sudo logger -t COVERAGE -i "@@@@@ " -p warning -- "${msg}" +fi + +# Find an I/O Controller and issue a "nvme disconnect" +if dev=$(python3 ${getdev} @STACD_DBUS_NAME@ @STACD_DBUS_PATH@); then + log "Remove connection (disconnect) to I/O Controller ${dev}" + run_cmd sudo nvme disconnect -d ${dev} + printf "\n" +else + msg="Failed to find a connection to an I/O Controller" + printf "%b%s%s%b[0m\n" "\0033" ${RED} "${msg}" "\0033" + sudo logger -t COVERAGE -i "@@@@@ " -p warning -- "${msg}" +fi + +sleep 3 + +rm "${getdev}" + + +#******************************************************************************* +log ">>>>>>>>>>>>>>>>>>>>> Marker [10] <<<<<<<<<<<<<<<<<<<<<" +printf "\n" + +sd_restart "stafd" +sd_restart "stacd" +sleep 4 + +log "Create invalid conditions for saving/loading stafd's last known config" +rm -rf "/tmp/stafd" +sd_restart "stafd" +sleep 2 + +log "Remove invalid conditions for saving/loading stafd's last known config" +mkdir -p "/tmp/stafd" +sd_restart "stafd" +sleep 2 + +#******************************************************************************* +# Change ownership of files that were created as root +sudo chown -R "${PRIMARY_USR}":"${PRIMARY_GRP}" coverage >/dev/null 2>&1 +sudo chown -R "${PRIMARY_USR}":"${PRIMARY_GRP}" staslib/__pycache__ >/dev/null 2>&1 +sudo chown -R "${PRIMARY_USR}":"${PRIMARY_GRP}" subprojects/libnvme/libnvme/__pycache__ >/dev/null 2>&1 + +#******************************************************************************* +# Run unit tests +run_unit_test test-avahi.py +run_unit_test test-avahi.py +run_unit_test test-config.py +run_unit_test test-controller.py +run_unit_test test-gtimer.py +run_unit_test test-iputil.py +run_unit_test test-log.py +run_unit_test sudo test-nvme_options.py # Test both with super user... +run_unit_test test-nvme_options.py # ... and with regular user +run_unit_test test-service.py +run_unit_test test-timeparse.py +run_unit_test test-transport_id.py +run_unit_test test-udev.py +run_unit_test test-version.py + +#******************************************************************************* +# Stop nvme target simulator + +log "Collect all coverage data" +coverage combine --rcfile=.coveragerc +printf "\n" + +log "Generating coverage report" +coverage report -i --rcfile=.coveragerc +printf "\n" + +log "Generating coverage report (HTML)" +coverage html -i --rcfile=.coveragerc +printf "\n" + diff --git a/doc/.gitignore b/doc/.gitignore new file mode 100644 index 0000000..186e12e --- /dev/null +++ b/doc/.gitignore @@ -0,0 +1 @@ +*.xml~ diff --git a/doc/dbus-idl-to-docbooks.py b/doc/dbus-idl-to-docbooks.py new file mode 100755 index 0000000..7865ffb --- /dev/null +++ b/doc/dbus-idl-to-docbooks.py @@ -0,0 +1,118 @@ +#!/usr/bin/python3 +import os +import sys +import pathlib +import tempfile +import subprocess +from argparse import ArgumentParser +from lxml import etree + + +def parse_args(): + parser = ArgumentParser(description='Generate DocBook documentation from D-Bus IDL.') + parser.add_argument( + '--idl', + action='store', + help='IDL file', + required=True, + type=str, + metavar='FILE', + ) + parser.add_argument( + '--output-directory', + action='store', + help='Output directory where DocBook files will be saved', + required=True, + type=str, + metavar='DIR', + ) + parser.add_argument( + '--tmp', + action='store', + help='Temporary directory for intermediate files', + required=True, + type=str, + metavar='DIR', + ) + return parser.parse_args() + + +ARGS = parse_args() + +pathlib.Path(ARGS.output_directory).mkdir(parents=True, exist_ok=True) + +REF_ENTRY_INFO = '''\ + <refentryinfo> + <title>stafctl</title> + <productname>nvme-stas</productname> + <author> + <personname> + <honorific>Mr</honorific> + <firstname>Martin</firstname> + <surname>Belanger</surname> + </personname> + <affiliation> + <orgname>Dell, Inc.</orgname> + </affiliation> + </author> + </refentryinfo> +''' + +MANVOLNUM = '<manvolnum>5</manvolnum>' +PURPOSE = '<refpurpose>DBus interface</refpurpose>' + +PARSER = etree.XMLParser(remove_blank_text=True) + + +def add_missing_info(fname, stem): + xml = etree.parse(fname, PARSER) + root = xml.getroot() + if root.tag != 'refentry': + return + + if xml.find('refentryinfo'): + return + + root.insert(0, etree.fromstring(REF_ENTRY_INFO)) + + refmeta = xml.find('refmeta') + if refmeta is not None: + if refmeta.find('refentrytitle') is None: + refmeta.append(etree.fromstring(f'<refentrytitle>{stem}</refentrytitle>')) + refmeta.append(etree.fromstring(MANVOLNUM)) + + refnamediv = xml.find('refnamediv') + if refnamediv is not None: + refpurpose = refnamediv.find('refpurpose') + if refpurpose is not None: + refnamediv.remove(refpurpose) + + refnamediv.append(etree.fromstring(PURPOSE)) + + et = etree.ElementTree(root) + et.write(fname, pretty_print=True) + + +FILE_PREFIX = 'nvme-stas' +FINAL_PREFIX = FILE_PREFIX + '-' + + +pathlib.Path(ARGS.tmp).mkdir(parents=True, exist_ok=True) +with tempfile.TemporaryDirectory(dir=ARGS.tmp) as tmpdirname: + try: + subprocess.run(['gdbus-codegen', '--output-directory', tmpdirname, '--generate-docbook', FILE_PREFIX, ARGS.idl]) + except subprocess.CalledProcessError as ex: + sys.exit(f'Failed to generate DocBook file. {ex}') + + stems = [] + with os.scandir(tmpdirname) as it: + for entry in it: + if entry.is_file() and entry.name.endswith('.xml') and entry.name.startswith(FINAL_PREFIX): + fname = entry.name[len(FINAL_PREFIX) :] # Strip prefix + stem = fname[0:-4] # Strip '.xml' suffix + stems.append(stem) + tmp_file = os.path.join(tmpdirname, entry.name) + add_missing_info(tmp_file, stem) + os.replace(tmp_file, os.path.join(ARGS.output_directory, fname)) + + print(';'.join(stems)) diff --git a/doc/genlist-from-docbooks.py b/doc/genlist-from-docbooks.py new file mode 100755 index 0000000..f094e09 --- /dev/null +++ b/doc/genlist-from-docbooks.py @@ -0,0 +1,36 @@ +#!/usr/bin/python3 +import glob +from lxml import etree + +exclude_list = list(glob.glob('standard-*.xml')) + +PARSER = etree.XMLParser(remove_blank_text=True) + + +def extract_data(fname): + et = etree.parse(fname, PARSER) + + manvolnum = et.find('./refmeta/manvolnum') + manvolnum = manvolnum.text if manvolnum is not None else 0 + + deps = set() + for elem in et.iter(): + keys = elem.keys() + if 'href' in keys and 'xpointer' in keys: + dep = elem.values()[0] + if dep in exclude_list: + deps.add(dep) + + return manvolnum, list(deps) + + +output = list() +file_list = glob.glob('*.xml') +for fname in file_list: + if fname not in exclude_list: + stem = fname[0:-4] + manvolnum, deps = extract_data(fname) + deps = ':'.join(deps) if deps else 'None' + output.append(','.join([stem, manvolnum, fname, deps])) + +print(';'.join(output)) diff --git a/doc/html.xsl b/doc/html.xsl new file mode 100644 index 0000000..9d2097c --- /dev/null +++ b/doc/html.xsl @@ -0,0 +1,63 @@ +<?xml version='1.0'?> <!--*-nxml-*--> +<!-- + SPDX-License-Identifier: Apache-2.0 + Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +--> + +<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0"> + +<xsl:import href="http://docbook.sourceforge.net/release/xsl/current/html/docbook.xsl"/> + +<xsl:template match="citerefentry[not(@project)]"> + <a> + <xsl:attribute name="href"> + <xsl:value-of select="refentrytitle"/> + <xsl:text>.html#</xsl:text> + <xsl:value-of select="refentrytitle/@target"/> + </xsl:attribute> + <xsl:call-template name="inline.charseq"/> + </a> +</xsl:template> + +<xsl:template name="user.header.content"> + <style> + a.headerlink { + color: #c60f0f; + font-size: 0.8em; + padding: 0 4px 0 4px; + text-decoration: none; + visibility: hidden; + } + + a.headerlink:hover { + background-color: #c60f0f; + color: white; + } + + h1:hover > a.headerlink, h2:hover > a.headerlink, h3:hover > a.headerlink, dt:hover > a.headerlink { + visibility: visible; + } + </style> + + + <a> + <xsl:text xml:space="preserve" white-space="pre">   </xsl:text> + </a> + + <span style="float:right"> + <xsl:text>nvme-stas </xsl:text> + <xsl:value-of select="$nvme-stas.version"/> + </span> + <hr/> +</xsl:template> + +<xsl:template match="literal"> + <xsl:text>"</xsl:text> + <xsl:call-template name="inline.monoseq"/> + <xsl:text>"</xsl:text> +</xsl:template> + +<xsl:output method="html" encoding="UTF-8" indent="no"/> + +</xsl:stylesheet> + diff --git a/doc/images/Coverage.png b/doc/images/Coverage.png Binary files differnew file mode 100644 index 0000000..91cd0a2 --- /dev/null +++ b/doc/images/Coverage.png diff --git a/doc/images/STAF-STAC-libnvme.png b/doc/images/STAF-STAC-libnvme.png Binary files differnew file mode 100644 index 0000000..19a6704 --- /dev/null +++ b/doc/images/STAF-STAC-libnvme.png diff --git a/doc/man.xsl b/doc/man.xsl new file mode 100644 index 0000000..fb6bb57 --- /dev/null +++ b/doc/man.xsl @@ -0,0 +1,41 @@ +<?xml version='1.0'?> <!--*-nxml-*--> +<!-- + SPDX-License-Identifier: Apache-2.0 + Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +--> + +<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" + xmlns:exsl="http://exslt.org/common" + extension-element-prefixes="exsl" + version="1.0"> + +<xsl:import href="http://docbook.sourceforge.net/release/xsl/current/manpages/docbook.xsl"/> + +<xsl:template name="TH.title.line"> + <xsl:param name="title"/> + <xsl:param name="section"/> + + <xsl:call-template name="mark.subheading"/> + <xsl:text>.TH "</xsl:text> + + <xsl:call-template name="string.upper"> + <xsl:with-param name="string"> + <xsl:value-of select="normalize-space($title)"/> + </xsl:with-param> + </xsl:call-template> + + <xsl:text>" "</xsl:text> + <xsl:value-of select="normalize-space($section)"/> + + <xsl:text>" "" "nvme-stas </xsl:text> + <xsl:value-of select="$nvme-stas.version"/> + + <xsl:text>" "</xsl:text> + + <xsl:text>" </xsl:text> + <xsl:call-template name="mark.subheading"/> + +</xsl:template> + +</xsl:stylesheet> + diff --git a/doc/meson.build b/doc/meson.build new file mode 100644 index 0000000..a4b5786 --- /dev/null +++ b/doc/meson.build @@ -0,0 +1,126 @@ +# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). +# +# Authors: Martin Belanger <Martin.Belanger@dell.com> +# + + +if want_man or want_html or want_readthedocs + docbklst = find_program('genlist-from-docbooks.py') + dbus2doc = find_program('dbus-idl-to-docbooks.py') + dbusgen = find_program('gdbus-codegen', required: false) # Needed by dbus2doc + if not dbusgen.found() + error('gdbus-codegen missing: Install libglib2.0-dev (deb) / glib2-devel (rpm)') + endif + + # Get the list of DocBook files to process. The result will + # be saved to variable docbooks as a list of tuples as follows: + # docbooks = [ ['file1', 'manvolnum-from-file1.xml', 'file1.xml'], + # ['file2', 'manvolnum-from-file2.xml', 'file2.xml'], ... ] + docbooks = [] + rr = run_command(docbklst, check: true) + output = rr.stdout().strip() + if output != '' + foreach item : output.split(';') + items = item.split(',') + stem = items[0] + manvolnum = items[1] + fname = items[2] + deps = items[3] + if deps == 'None' + deps = [] + else + deps = deps.split(':') + endif + docbooks += [ [stem, manvolnum, fname, deps] ] + endforeach + endif + + # Generate DocBooks from IDL queried directly from the D-Bus services. + out_dir = conf.get('BUILD_DIR') / 'man-tmp' + env = environment({'PYTHONPATH': PYTHONPATH}) + idls = [ 'stafd.idl', 'stacd.idl' ] + foreach idl : idls + rr = run_command( + dbus2doc, + '--idl', conf.get('BUILD_DIR') / 'staslib' / idl, + '--output-directory', out_dir, + '--tmp', meson.current_build_dir(), + env: env, + check: true) + output = rr.stdout().strip() + if output != '' + foreach stem : output.split(';') + docbooks += [ [stem, '5', out_dir / stem + '.xml', []] ] + endforeach + endif + endforeach + + + xsltproc = find_program('xsltproc') + if xsltproc.found() + manpage_style = 'http://docbook.sourceforge.net/release/xsl/current/manpages/docbook.xsl' + if run_command(xsltproc, '--nonet', manpage_style, check: false).returncode() != 0 + error('Docbook style sheet missing: Install docbook-xsl (deb) / docbook-style-xsl (rpm)') + endif + endif + + xslt_cmd = [ + xsltproc, + '--nonet', + '--xinclude', + '--stringparam', 'man.output.quietly', '1', + '--stringparam', 'funcsynopsis.style', 'ansi', + '--stringparam', 'man.th.extra1.suppress', '1', + '--stringparam', 'man.authors.section.enabled', '0', + '--stringparam', 'man.copyright.section.enabled', '0', + '--stringparam', 'nvme-stas.version', '@0@'.format(meson.project_version()), + '-o', '@OUTPUT@', + ] + + man_xsl = files('man.xsl') + html_xsl = files('html.xsl') + + + html_files = [] # Will be used as input to readthedocs + foreach tuple: docbooks + stem = tuple[0] + sect = tuple[1] + file = files(tuple[2]) + deps = tuple[3] + + if want_man + man = stem + '.' + sect + custom_target( + man, + input: file, + output: man, + depend_files: deps, + command: xslt_cmd + [man_xsl, '@INPUT@'], + install: true, + install_dir: mandir / ('man' + sect) + ) + endif + + if want_html or want_readthedocs + html = stem + '.html' + html_file = custom_target( + html, + input: file, + output: html, + depend_files: deps, + command: xslt_cmd + [html_xsl, '@INPUT@'], + install: want_html, + install_dir: docdir / 'html' + ) + html_files += [ [stem, html_file ] ] + endif + endforeach +endif + +if want_readthedocs + subdir('readthedocs') +endif diff --git a/doc/nvme-stas.xml b/doc/nvme-stas.xml new file mode 100644 index 0000000..1e70c02 --- /dev/null +++ b/doc/nvme-stas.xml @@ -0,0 +1,185 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" +"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"> +<!-- + SPDX-License-Identifier: Apache-2.0 + Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +--> + +<refentry id="nvme-stas"> + + <refentryinfo> + <title>nvme-stas</title> + <productname>nvme-stas</productname> + <author> + <personname> + <honorific>Mr</honorific> + <firstname>Martin</firstname> + <surname>Belanger</surname> + </personname> + <affiliation> + <orgname>Dell, Inc.</orgname> + </affiliation> + </author> + </refentryinfo> + + <refmeta> + <refentrytitle>nvme-stas</refentrytitle> + <manvolnum>7</manvolnum> + </refmeta> + + <refnamediv> + <refname>nvme-stas</refname> + <refpurpose>NVMe over Fabrics STorage Appliance Services</refpurpose> + </refnamediv> + + <refsect1> + <title>Introduction</title> + + <para> + This page describes the services provided by the <code>nvme-stas</code> package. + </para> + + <para> + <code>nvme-stas</code> is composed of two services, <citerefentry><refentrytitle>stafd</refentrytitle><manvolnum>8</manvolnum></citerefentry> + and <citerefentry><refentrytitle>stacd</refentrytitle><manvolnum>8</manvolnum></citerefentry>, + running on a host computer (the NVMe Host). + </para> + + <refsect2> + <title>STorage Appliance Finder (<code>stafd</code>)</title> + <para> + The tasks performed by <code>stafd</code> include: + </para> + <itemizedlist mark='opencircle'> + <listitem> + <para> + Register for mDNS service type <parameter>_nvme-disc._tcp</parameter> + with Avahi, the service discovery daemon. This allows <code>stafd</code> + to automatically locate Central or Direct Discovery + Controllers (CDC, DDC) with zero-configuration networking + (zeroconf). <code>stafd</code> also allows users to manually enter CDCs + and DDCs in a configuration file + (<filename>/etc/stas/stafd.conf</filename>) when users + prefer not to enable mDNS-based zeroconf. + </para> + </listitem> + <listitem> + <para> + Connect to discovered or configured CDCs or DDCs. + </para> + </listitem> + <listitem> + <para> + Retrieve the list of NVMe subsystem IO Controllers or + Discovery Controller referrals from the Discovery Log Page + using the NVMe command "Get Log Page". + </para> + </listitem> + <listitem> + <para> + Maintain a cache of the discovery log pages. + </para> + </listitem> + <listitem> + <para> + Provide a D-Bus API where other applications can interact + with <code>stafd</code>. This API can be used, for example, to retrieve + the list of cached discovery log pages. + </para> + </listitem> + </itemizedlist> + </refsect2> + + <refsect2> + <title>STorage Appliance Connector (<code>stacd</code>)</title> + <para> + The tasks performed by <code>stacd</code> include: + </para> + <itemizedlist mark='opencircle'> + <listitem> + <para> + Read the list of storage subsystems (i.e., discovery log pages) + from <code>stafd</code> over the D-Bus API. + </para> + </listitem> + <listitem> + <para> + Similar to <code>stafd</code>, <code>stacd</code> can also read a list of storage + subsystems to connect to from a configuration + file: (<filename>/etc/stas/stacd.conf</filename>). + </para> + </listitem> + <listitem> + <para> + Set up the I/O controller connections to each storage subsystem. + </para> + </listitem> + <listitem> + <para> + Provide a D-Bus API where other applications can interact + with <code>stacd</code>. For example, an application could retrieve the + list of I/O controllers that <code>stacd</code> connected to. + </para> + </listitem> + </itemizedlist> + </refsect2> + + <refsect2> + <title>System configuration</title> + <para> + A host must be provided with a Host NQN and a Host ID. <code>nvme-stas</code> + will not run without these two mandatory configuration parameters. + To follow in the footsteps of <code>nvme-cli</code> and <code>libnvme</code>, + <code>nvme-stas</code> will use the same Host NQN and ID that + <code>nvme-cli</code> and <code>libnvme</code> use by default. + In other words, <code>nvme-stas</code> will read the Host NQN and ID + from these two files by default: + </para> + + <itemizedlist mark='opencircle'> + <listitem> + <para> + <filename>/etc/nvme/hostnqn</filename> + </para> + </listitem> + <listitem> + <para> + <filename>/etc/nvme/hostid</filename> + </para> + </listitem> + </itemizedlist> + + <para> + Using the same configuration files will ensure consistency between + <code>nvme-stas</code>, <code>nvme-cli</code>, and <code>libnvme</code>. + On the other hand, <code>nvme-stas</code> can operate with a + different Host NQN and/or ID. In that case, one can specify them + in <filename>/etc/stas/sys.conf</filename>. + </para> + A new optional configuration parameters introduced in TP8010, the + Host Symbolic Name, can also be specified in <filename>/etc/stas/sys.conf</filename>. + The documentation for <filename>/etc/stas/sys.conf</filename> + can be found <filename>/etc/stas/sys.conf.doc</filename>. + <para> + </para> + </refsect2> + + </refsect1> + + + <refsect1> + <title>See Also</title> + <para> + <citerefentry><refentrytitle>stacctl</refentrytitle><manvolnum>1</manvolnum></citerefentry>, + <citerefentry><refentrytitle>stacd.conf</refentrytitle><manvolnum>5</manvolnum></citerefentry>, + <citerefentry><refentrytitle>stacd.service</refentrytitle><manvolnum>8</manvolnum></citerefentry>, + <citerefentry><refentrytitle>stacd</refentrytitle><manvolnum>8</manvolnum></citerefentry>, + <citerefentry><refentrytitle>stafctl</refentrytitle><manvolnum>1</manvolnum></citerefentry>, + <citerefentry><refentrytitle>stafd.conf</refentrytitle><manvolnum>5</manvolnum></citerefentry>, + <citerefentry><refentrytitle>stafd.service</refentrytitle><manvolnum>8</manvolnum></citerefentry>, + <citerefentry><refentrytitle>stafd</refentrytitle><manvolnum>8</manvolnum></citerefentry>, + </para> + </refsect1> + +</refentry> diff --git a/doc/readthedocs/Makefile b/doc/readthedocs/Makefile new file mode 100644 index 0000000..76911e9 --- /dev/null +++ b/doc/readthedocs/Makefile @@ -0,0 +1,22 @@ +# Minimal makefile for Sphinx documentation +# +.DEFAULT_GOAL := help + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +.PHONY: help +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +.PHONY: Makefile +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/doc/readthedocs/conf.py b/doc/readthedocs/conf.py new file mode 100644 index 0000000..a5b3960 --- /dev/null +++ b/doc/readthedocs/conf.py @@ -0,0 +1,34 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + + +# -- Project information ----------------------------------------------------- + +project = 'nvme-stas' +copyright = 'Copyright (c) 2022, Dell Inc. or its subsidiaries. All rights reserved.' +author = 'Martin Belanger <martin.belanger@dell.com>' +master_doc = 'index' + +version = '@VERSION@' +release = '@VERSION@' + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autosummary', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['Thumbs.db', '.DS_Store'] diff --git a/doc/readthedocs/environment.txt b/doc/readthedocs/environment.txt new file mode 100644 index 0000000..5388c88 --- /dev/null +++ b/doc/readthedocs/environment.txt @@ -0,0 +1 @@ +sphinx==5.3.0 diff --git a/doc/readthedocs/index.rst b/doc/readthedocs/index.rst new file mode 100644 index 0000000..ad927fe --- /dev/null +++ b/doc/readthedocs/index.rst @@ -0,0 +1,30 @@ +Welcome to nvme-stas's documentation! +===================================== + +What does nvme-stas provide? + +* A Central Discovery Controller (CDC) client for Linux +* Asynchronous Event Notifications (AEN) handling +* Automated NVMe subsystem connection controls +* Error handling and reporting +* Automatic (zeroconf) and Manual configuration + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + installation.rst + nvme-stas.rst + stafd-index.rst + stacd-index.rst + stasadm.rst + sys.conf.rst + stas-config.target.rst + stas-config@.service.rst + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/doc/readthedocs/installation.rst b/doc/readthedocs/installation.rst new file mode 100644 index 0000000..95750e2 --- /dev/null +++ b/doc/readthedocs/installation.rst @@ -0,0 +1,28 @@ +Installation +============ + +Debian / Ubuntu: +---------------- + +.. code-block:: sh + + $ apt-get install nvme-stas + +Fedora / Red Hat: +----------------- + +.. code-block:: sh + + $ dnf install nvme-stas + +Python Version +-------------- + +The latest Python 3 version is always recommended, since it has all the latest bells and +whistles. libnvme supports Python 3.6 and above. + +Dependencies +------------ + +nvme-stas is built on top of libnvme, which is used to interact with the kernel's NVMe driver (i.e. drivers/nvme/host/). To support all the features of nvme-stas, several changes to the Linux kernel are required. nvme-stas can also operate with older kernels, but with limited functionality. Kernel 5.18 provides all the features needed by nvme-stas. nvme-stas can also work with older kernels that include back-ported changes to the NVMe driver. + diff --git a/doc/readthedocs/make.bat b/doc/readthedocs/make.bat new file mode 100644 index 0000000..153be5e --- /dev/null +++ b/doc/readthedocs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+ set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+ echo.
+ echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+ echo.installed, then set the SPHINXBUILD environment variable to point
+ echo.to the full path of the 'sphinx-build' executable. Alternatively you
+ echo.may add the Sphinx directory to PATH.
+ echo.
+ echo.If you don't have Sphinx installed, grab it from
+ echo.https://www.sphinx-doc.org/
+ exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd
diff --git a/doc/readthedocs/meson.build b/doc/readthedocs/meson.build new file mode 100644 index 0000000..a8e2305 --- /dev/null +++ b/doc/readthedocs/meson.build @@ -0,0 +1,64 @@ +# Copyright (c) 2022, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). +# +# Authors: Martin Belanger <Martin.Belanger@dell.com> +# + +pandoc = find_program('pandoc', required: true) + +components = [ + 'conf.py', + 'Makefile', + 'make.bat', + 'index.rst', + 'environment.txt', + 'installation.rst', + 'nvme-stas.rst', + 'org.nvmexpress.stac.debug.rst', + 'org.nvmexpress.stac.rst', + 'org.nvmexpress.staf.debug.rst', + 'org.nvmexpress.staf.rst', + 'stacctl.rst', + 'stacd-index.rst', + 'stacd.conf.rst', + 'stacd.rst', + 'stacd.service.rst', + 'stafctl.rst', + 'stafd-index.rst', + 'stafd.conf.rst', + 'stafd.rst', + 'stafd.service.rst', + 'stas-config.target.rst', + 'stas-config@.service.rst', + 'stasadm.rst', + 'sys.conf.rst', +] +foreach component : components + configure_file( + input: component, + output: component, + configuration: conf, + ) +endforeach + +foreach tuple: html_files + stem = tuple[0] + html_file = tuple[1] + rst = '_' + stem + '.rst' + custom_target( + rst, + input: html_file, + output: rst, + build_by_default: true, + command: [ + pandoc, + '-f', 'html', + '-t', 'rst', + '-o', '@OUTPUT@', + '@INPUT@' + ] + ) +endforeach diff --git a/doc/readthedocs/nvme-stas.rst b/doc/readthedocs/nvme-stas.rst new file mode 100644 index 0000000..e7383f9 --- /dev/null +++ b/doc/readthedocs/nvme-stas.rst @@ -0,0 +1,5 @@ +========================== +STorage Appliance Services +========================== + +.. include:: _nvme-stas.rst diff --git a/doc/readthedocs/org.nvmexpress.stac.debug.rst b/doc/readthedocs/org.nvmexpress.stac.debug.rst new file mode 100644 index 0000000..e82f2d7 --- /dev/null +++ b/doc/readthedocs/org.nvmexpress.stac.debug.rst @@ -0,0 +1,7 @@ +========================= +org.nvmexpress.stac.debug +========================= +.. module:: org.nvmexpress.stac.debug + +.. include:: _org.nvmexpress.stac.debug.rst + diff --git a/doc/readthedocs/org.nvmexpress.stac.rst b/doc/readthedocs/org.nvmexpress.stac.rst new file mode 100644 index 0000000..5132fad --- /dev/null +++ b/doc/readthedocs/org.nvmexpress.stac.rst @@ -0,0 +1,7 @@ +=================== +org.nvmexpress.stac +=================== +.. module:: org.nvmexpress.stac + +.. include:: _org.nvmexpress.stac.rst + diff --git a/doc/readthedocs/org.nvmexpress.staf.debug.rst b/doc/readthedocs/org.nvmexpress.staf.debug.rst new file mode 100644 index 0000000..8161db5 --- /dev/null +++ b/doc/readthedocs/org.nvmexpress.staf.debug.rst @@ -0,0 +1,7 @@ +========================= +org.nvmexpress.staf.debug +========================= +.. module:: org.nvmexpress.staf.debug + +.. include:: _org.nvmexpress.staf.debug.rst + diff --git a/doc/readthedocs/org.nvmexpress.staf.rst b/doc/readthedocs/org.nvmexpress.staf.rst new file mode 100644 index 0000000..26ef29d --- /dev/null +++ b/doc/readthedocs/org.nvmexpress.staf.rst @@ -0,0 +1,7 @@ +=================== +org.nvmexpress.staf +=================== +.. module:: org.nvmexpress.staf + +.. include:: _org.nvmexpress.staf.rst + diff --git a/doc/readthedocs/stacctl.rst b/doc/readthedocs/stacctl.rst new file mode 100644 index 0000000..60fe075 --- /dev/null +++ b/doc/readthedocs/stacctl.rst @@ -0,0 +1,7 @@ +======= +stacctl +======= +.. module:: stacctl + +.. include:: _stacctl.rst + diff --git a/doc/readthedocs/stacd-index.rst b/doc/readthedocs/stacd-index.rst new file mode 100644 index 0000000..cebd5f4 --- /dev/null +++ b/doc/readthedocs/stacd-index.rst @@ -0,0 +1,13 @@ +STorage Appliance Connector +=========================== + +.. toctree:: + :maxdepth: 1 + + stacd.rst + stacd.conf.rst + stacd.service.rst + stacctl.rst + org.nvmexpress.stac.rst + org.nvmexpress.stac.debug.rst + diff --git a/doc/readthedocs/stacd.conf.rst b/doc/readthedocs/stacd.conf.rst new file mode 100644 index 0000000..2a0cad9 --- /dev/null +++ b/doc/readthedocs/stacd.conf.rst @@ -0,0 +1,7 @@ +========== +stacd.conf +========== +.. module:: stacd.conf + +.. include:: _stacd.conf.rst + diff --git a/doc/readthedocs/stacd.rst b/doc/readthedocs/stacd.rst new file mode 100644 index 0000000..2cf809b --- /dev/null +++ b/doc/readthedocs/stacd.rst @@ -0,0 +1,8 @@ +===== +stacd +===== +.. module:: stacd + +.. include:: _stacd.rst + + diff --git a/doc/readthedocs/stacd.service.rst b/doc/readthedocs/stacd.service.rst new file mode 100644 index 0000000..7294710 --- /dev/null +++ b/doc/readthedocs/stacd.service.rst @@ -0,0 +1,7 @@ +============= +stacd.service +============= +.. module:: stacd.service + +.. include:: _stacd.service.rst + diff --git a/doc/readthedocs/stafctl.rst b/doc/readthedocs/stafctl.rst new file mode 100644 index 0000000..849e37c --- /dev/null +++ b/doc/readthedocs/stafctl.rst @@ -0,0 +1,7 @@ +======= +stafctl +======= +.. module:: stafctl + +.. include:: _stafctl.rst + diff --git a/doc/readthedocs/stafd-index.rst b/doc/readthedocs/stafd-index.rst new file mode 100644 index 0000000..f4d1292 --- /dev/null +++ b/doc/readthedocs/stafd-index.rst @@ -0,0 +1,13 @@ +STorage Appliance Finder +======================== + +.. toctree:: + :maxdepth: 1 + + stafd.rst + stafd.conf.rst + stafd.service.rst + stafctl.rst + org.nvmexpress.staf.rst + org.nvmexpress.staf.debug.rst + diff --git a/doc/readthedocs/stafd.conf.rst b/doc/readthedocs/stafd.conf.rst new file mode 100644 index 0000000..a03d93a --- /dev/null +++ b/doc/readthedocs/stafd.conf.rst @@ -0,0 +1,7 @@ +========== +stafd.conf +========== +.. module:: stafd.conf + +.. include:: _stafd.conf.rst + diff --git a/doc/readthedocs/stafd.rst b/doc/readthedocs/stafd.rst new file mode 100644 index 0000000..a12298c --- /dev/null +++ b/doc/readthedocs/stafd.rst @@ -0,0 +1,7 @@ +===== +stafd +===== +.. module:: stafd + +.. include:: _stafd.rst + diff --git a/doc/readthedocs/stafd.service.rst b/doc/readthedocs/stafd.service.rst new file mode 100644 index 0000000..771189c --- /dev/null +++ b/doc/readthedocs/stafd.service.rst @@ -0,0 +1,7 @@ +============= +stafd.service +============= +.. module:: stafd.service + +.. include:: _stafd.service.rst + diff --git a/doc/readthedocs/stas-config.target.rst b/doc/readthedocs/stas-config.target.rst new file mode 100644 index 0000000..cf88fa9 --- /dev/null +++ b/doc/readthedocs/stas-config.target.rst @@ -0,0 +1,7 @@ +================== +stas-config.target +================== +.. module:: stas-config.target + +.. include:: _stas-config.target.rst + diff --git a/doc/readthedocs/stas-config@.service.rst b/doc/readthedocs/stas-config@.service.rst new file mode 100644 index 0000000..7af78ed --- /dev/null +++ b/doc/readthedocs/stas-config@.service.rst @@ -0,0 +1,7 @@ +==================== +stas-config@.service +==================== +.. module:: stas-config@.service + +.. include:: _stas-config@.service.rst + diff --git a/doc/readthedocs/stasadm.rst b/doc/readthedocs/stasadm.rst new file mode 100644 index 0000000..54489a8 --- /dev/null +++ b/doc/readthedocs/stasadm.rst @@ -0,0 +1,7 @@ +======= +stasadm +======= +.. module:: stasadm + +.. include:: _stasadm.rst + diff --git a/doc/readthedocs/sys.conf.rst b/doc/readthedocs/sys.conf.rst new file mode 100644 index 0000000..05ab0f0 --- /dev/null +++ b/doc/readthedocs/sys.conf.rst @@ -0,0 +1,7 @@ +======== +sys.conf +======== +.. module:: sys.conf + +.. include:: _sys.conf.rst + diff --git a/doc/stacctl.xml b/doc/stacctl.xml new file mode 100644 index 0000000..1f07276 --- /dev/null +++ b/doc/stacctl.xml @@ -0,0 +1,223 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!--*-nxml-*--> +<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" +"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"> +<!-- + SPDX-License-Identifier: Apache-2.0 + Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +--> +<refentry id="stacctl" xmlns:xi="http://www.w3.org/2001/XInclude"> + <refentryinfo> + <title>stacctl</title> + <productname>nvme-stas</productname> + <author> + <personname> + <honorific>Mr</honorific> + <firstname>Martin</firstname> + <surname>Belanger</surname> + </personname> + <affiliation> + <orgname>Dell, Inc.</orgname> + </affiliation> + </author> + </refentryinfo> + + <refmeta> + <refentrytitle>stacctl</refentrytitle> + <manvolnum>1</manvolnum> + </refmeta> + + <refnamediv> + <refname>stacctl</refname> + <refpurpose>STorage Appliance Connector (STAC) utility program</refpurpose> + </refnamediv> + + <refsynopsisdiv> + <cmdsynopsis> + <command>stacctl</command> + <arg choice="opt" rep="repeat">OPTIONS</arg> + <arg choice="req">COMMAND</arg> + <arg choice="opt" rep="repeat">OPTIONS</arg> + </cmdsynopsis> + </refsynopsisdiv> + + <refsect1> + <title>Description</title> + + <para> + <command>stacctl</command> is a tool that can be used to communicate + with the <citerefentry><refentrytitle>stacd</refentrytitle> + <manvolnum>8</manvolnum></citerefentry> daemon to retrieve + operational data. + </para> + </refsect1> + + <refsect1> + <title>Commands</title> + <para>The following commands are understood:</para> + + <variablelist> + <xi:include href="standard-options.xml" xpointer="tron"/> + <xi:include href="standard-options.xml" xpointer="troff"/> + <xi:include href="standard-options.xml" xpointer="status"/> + + <varlistentry> + <term><command>ls</command></term> + <listitem> + <para> + Show the list of I/O controllers. This will list + all the I/O controllers configured in + <citerefentry> + <refentrytitle>stacd.conf</refentrytitle> + <manvolnum>5</manvolnum> + </citerefentry> as well as those discovered by the + <citerefentry> + <refentrytitle>stafd</refentrytitle> + <manvolnum>8</manvolnum> + </citerefentry> daemon. + </para> + </listitem> + </varlistentry> + </variablelist> + </refsect1> + + <refsect1> + <title>Options</title> + + <para>The following options are understood:</para> + + <variablelist> + <xi:include href="standard-options.xml" xpointer="help"/> + <xi:include href="standard-options.xml" xpointer="version"/> + <xi:include href="standard-options.xml" xpointer="detailed"/> + </variablelist> + </refsect1> + + <refsect1> + <title>Exit status</title> + <para> + On success, 0 is returned; otherwise, a non-zero failure code is + returned. + </para> + </refsect1> + + <refsect1> + <title>Examples</title> + + <example> + <title>List I/O controllers</title> + <programlisting>$ stacctl ls --detailed +[{'connect attempts': 0, + 'device': 'nvme1', + 'host-iface': '', + 'host-traddr': '', + 'hostid': '3e518ec3-72ec-46a5-a603-2510e3140e29', + 'hostnqn': 'nqn.2014-08.org.nvmexpress:uuid:13730573-e8d7-446e-81f6-042a497846d5', + 'model': 'Linux', + 'retry connect timer': '60.0s [off]', + 'serial': '8d22fa96da912fb13f5a', + 'subsysnqn': 'nqn.1988-11.com.dell:PowerSANxxx:01:20210225100113-454f73093ceb4847a7bdfc6e34aedead', + 'traddr': '::1', + 'transport': 'tcp', + 'trsvcid': '8009'}, + {'connect attempts': 0, + 'device': 'nvme2', + 'host-iface': '', + 'host-traddr': '', + 'hostid': '3e518ec3-72ec-46a5-a603-2510e3140e29', + 'hostnqn': 'nqn.2014-08.org.nvmexpress:uuid:13730573-e8d7-446e-81f6-042a497846d5', + 'model': 'Linux', + 'retry connect timer': '60.0s [off]', + 'serial': 'a9987ae2fd173d100fd0', + 'subsysnqn': 'nqn.1988-11.com.dell:PowerSANxxx:01:20210225100113-454f73093ceb4847a7bdfc6e34aebeef', + 'traddr': '::1', + 'transport': 'tcp', + 'trsvcid': '8009'}, + {'connect attempts': 0, + 'device': 'nvme3', + 'host-iface': '', + 'host-traddr': '', + 'hostid': '3e518ec3-72ec-46a5-a603-2510e3140e29', + 'hostnqn': 'nqn.2014-08.org.nvmexpress:uuid:13730573-e8d7-446e-81f6-042a497846d5', + 'model': 'Linux', + 'retry connect timer': '60.0s [off]', + 'serial': '13e122f1a8122bed5a8d', + 'subsysnqn': 'nqn.1988-11.com.dell:PowerSANxxx:01:20210225100113-454f73093ceb4847a7bdfc6e34ae8e28', + 'traddr': '::1', + 'transport': 'tcp', + 'trsvcid': '8009'}]</programlisting> + </example> + + <example> + <title>Disable tracing</title> + <programlisting>$ stacctl troff</programlisting> + </example> + + <example> + <title> + Show <citerefentry><refentrytitle>stacd</refentrytitle> + <manvolnum>8</manvolnum></citerefentry> operational status. + </title> + + <programlisting>$ stacctl status +{'config soak timer': '1.5s [off]', + 'controllers': [{'connect attempts': 0, + 'device': 'nvme1', + 'host-iface': '', + 'host-traddr': '', + 'hostid': '3e518ec3-72ec-46a5-a603-2510e3140e29', + 'hostnqn': 'nqn.2014-08.org.nvmexpress:uuid:13730573-e8d7-446e-81f6-042a497846d5', + 'model': 'Linux', + 'retry connect timer': '60.0s [off]', + 'serial': '8d22fa96da912fb13f5a', + 'subsysnqn': 'nqn.1988-11.com.dell:PowerSANxxx:01:20210225100113-454f73093ceb4847a7bdfc6e34aedead', + 'traddr': '::1', + 'transport': 'tcp', + 'trsvcid': '8009'}, + {'connect attempts': 0, + 'device': 'nvme2', + 'host-iface': '', + 'host-traddr': '', + 'hostid': '3e518ec3-72ec-46a5-a603-2510e3140e29', + 'hostnqn': 'nqn.2014-08.org.nvmexpress:uuid:13730573-e8d7-446e-81f6-042a497846d5', + 'model': 'Linux', + 'retry connect timer': '60.0s [off]', + 'serial': 'a9987ae2fd173d100fd0', + 'subsysnqn': 'nqn.1988-11.com.dell:PowerSANxxx:01:20210225100113-454f73093ceb4847a7bdfc6e34aebeef', + 'traddr': '::1', + 'transport': 'tcp', + 'trsvcid': '8009'}, + {'connect attempts': 0, + 'device': 'nvme3', + 'host-iface': '', + 'host-traddr': '', + 'hostid': '3e518ec3-72ec-46a5-a603-2510e3140e29', + 'hostnqn': 'nqn.2014-08.org.nvmexpress:uuid:13730573-e8d7-446e-81f6-042a497846d5', + 'model': 'Linux', + 'retry connect timer': '60.0s [off]', + 'serial': '13e122f1a8122bed5a8d', + 'subsysnqn': 'nqn.1988-11.com.dell:PowerSANxxx:01:20210225100113-454f73093ceb4847a7bdfc6e34ae8e28', + 'traddr': '::1', + 'transport': 'tcp', + 'trsvcid': '8009'}], + 'log-level': 'DEBUG', + 'tron': True}</programlisting> + </example> + </refsect1> + + <refsect1> + <title>See Also</title> + + <para> + <citerefentry> + <refentrytitle>stacd.conf</refentrytitle> + <manvolnum>5</manvolnum> + </citerefentry>, + <citerefentry> + <refentrytitle>stacd</refentrytitle> + <manvolnum>8</manvolnum> + </citerefentry> + <citerefentry project="man-pages"/> + </para> + </refsect1> +</refentry> diff --git a/doc/stacd.conf.xml b/doc/stacd.conf.xml new file mode 100644 index 0000000..e8c6a64 --- /dev/null +++ b/doc/stacd.conf.xml @@ -0,0 +1,652 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!--*-nxml-*--> +<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" +"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"> +<!-- + SPDX-License-Identifier: Apache-2.0 + Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +--> +<refentry id="stacd.conf" xmlns:xi="http://www.w3.org/2001/XInclude"> + <refentryinfo> + <title>stacd.conf</title> + <productname>nvme-stas</productname> + + <author> + <personname> + <honorific>Mr</honorific> + <firstname>Martin</firstname> + <surname>Belanger</surname> + </personname> + + <affiliation> + <orgname>Dell, Inc.</orgname> + </affiliation> + </author> + </refentryinfo> + + <refmeta> + <refentrytitle>stacd.conf</refentrytitle> + <manvolnum>5</manvolnum> + </refmeta> + + <refnamediv> + <refname>stacd.conf</refname> + <refpurpose> + <citerefentry project="man-pages"> + <refentrytitle>stacd</refentrytitle> + <manvolnum>8</manvolnum> + </citerefentry> + configuration file + </refpurpose> + </refnamediv> + + <refsynopsisdiv> + <para> + <filename>/etc/stas/stacd.conf</filename> + </para> + </refsynopsisdiv> + + <refsect1> + <title>Description</title> + + <para> + When <citerefentry project="man-pages"><refentrytitle>stacd</refentrytitle> + <manvolnum>8</manvolnum></citerefentry> starts up, it reads its + configuration from <filename>stacd.conf</filename>. + </para> + </refsect1> + + <refsect1> + <title>Configuration File Format</title> + <para> + <filename>stacd.conf</filename> is a plain text file divided into + sections, with configuration entries in the style + <replaceable>key</replaceable>=<replaceable>value</replaceable>. + Spaces immediately before or after the <literal>=</literal> are + ignored. Empty lines are ignored as well as lines starting with + <literal>#</literal>, which may be used for commenting. + </para> + </refsect1> + + <refsect1> + <title>Options</title> + + <refsect2> + <title>[Global] section</title> + <para> + The following options are available in the + <literal>[Global]</literal> section: + </para> + + <variablelist> + <xi:include href="standard-conf.xml" xpointer="tron"/> + <xi:include href="standard-conf.xml" xpointer="hdr-digest"/> + <xi:include href="standard-conf.xml" xpointer="data-digest"/> + <xi:include href="standard-conf.xml" xpointer="kato"/> + <xi:include href="standard-conf.xml" xpointer="ip-family"/> + + <varlistentry> + <term><varname>nr-io-queues=</varname></term> + + <listitem> + <para> + Takes a value in the range 1...N. Overrides the + default number of I/O queues create by the driver. + </para> + + <para>Note: This parameter is identical to that provided by nvme-cli.</para> + <para> + Default: Depends on kernel and other run + time factors (e.g. number of CPUs). + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><varname>nr-write-queues=</varname></term> + + <listitem> + <para> + Takes a value in the range 1...N. Adds additional + queues that will be used for write I/O. + </para> + + <para>Note: This parameter is identical to that provided by nvme-cli.</para> + + <para> + Default: Depends on kernel and other run + time factors (e.g. number of CPUs). + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><varname>nr-poll-queues=</varname></term> + + <listitem> + <para> + Takes a value in the range 1...N. Adds additional + queues that will be used for polling latency + sensitive I/O. + </para> + + <para>Note: This parameter is identical to that provided by nvme-cli.</para> + + <para> + Default: Depends on kernel and other run + time factors (e.g. number of CPUs). + </para> + </listitem> + </varlistentry> + + <xi:include href="standard-conf.xml" xpointer="queue-size"/> + <xi:include href="standard-conf.xml" xpointer="reconnect-delay"/> + <xi:include href="standard-conf.xml" xpointer="ctrl-loss-tmo"/> + <xi:include href="standard-conf.xml" xpointer="disable-sqflow"/> + + <varlistentry> + <term><varname>ignore-iface=</varname></term> + <listitem> + <para> + Takes a boolean argument. This option controls how + connections with I/O Controllers (IOC) are made. + </para> + + <para> + There is no guarantee that there will be a route to + reach that IOC. However, we can use the socket + option SO_BINDTODEVICE to force the connection to be + made on a specific interface instead of letting the + routing tables decide where to make the connection. + </para> + + <para> + This option determines whether <code>stacd</code> will use + SO_BINDTODEVICE to force connections on an interface + or just rely on the routing tables. The default is + to use SO_BINDTODEVICE, in other words, <code>stacd</code> does + not ignore the interface. + </para> + + <para> + BACKGROUND: + By default, <code>stacd</code> will connect to IOCs on the same + interface that was used to retrieve the discovery + log pages. If stafd discovers a DC on an interface + using mDNS, and stafd connects to that DC and + retrieves the log pages, it is expected that the + storage subsystems listed in the log pages are + reachable on the same interface where the DC was + discovered. + </para> + + <para> + For example, let's say a DC is discovered on + interface ens102. Then all the subsystems listed in + the log pages retrieved from that DC must be + reachable on interface ens102. If this doesn't work, + for example you cannot "ping -I ens102 [storage-ip]", + then the most likely explanation is that proxy arp + is not enabled on the switch that the host is + connected to on interface ens102. Whatever you do, + resist the temptation to manually set up the routing + tables or to add alternate routes going over a + different interface than the one where the DC is + located. That simply won't work. Make sure proxy arp + is enabled on the switch first. + </para> + + <para> + Setting routes won't work because, by default, <code>stacd</code> + uses the SO_BINDTODEVICE socket option when it + connects to IOCs. This option is used to force a + socket connection to be made on a specific interface + instead of letting the routing tables decide where + to connect the socket. Even if you were to manually + configure an alternate route on a different interface, + the connections (i.e. host to IOC) will still be + made on the interface where the DC was discovered by + stafd. + </para> + + <para> + Defaults to <parameter>false</parameter>. + </para> + </listitem> + </varlistentry> + </variablelist> + </refsect2> + + <refsect2> + <title>[I/O controller connection management] section</title> + <para> + Connectivity between hosts and subsystems in a fabric is + controlled by Fabric Zoning. Entities that share a common + zone (i.e., are zoned together) are allowed to discover each + other and establish connections between them. Fabric Zoning is + configured on Discovery Controllers (DC). Users can add/remove + controllers and/or hosts to/from zones. + </para> + + <para> + Hosts have no direct knowledge of the Fabric Zoning configuration + that is active on a given DC. As a result, if a host is impacted + by a Fabric Zoning configuration change, it will be notified of + the connectivity configuration change by the DC via Asynchronous + Event Notifications (AEN). + </para> + + <table frame='all'> + <title>List of terms used in this section:</title> + <tgroup cols="2" align='left' colsep='1' rowsep='1'> + <thead> + <row> + <entry>Term</entry> + <entry>Description</entry> + </row> + </thead> + + <tbody> + <row> + <entry>AEN</entry> + <entry>Asynchronous Event Notification. A CQE (Completion Queue Entry) for an Asynchronous Event Request that was previously transmitted by the host to a Discovery Controller. AENs are used by DCs to notify hosts that a change (e.g., a connectivity configuration change) has occurred.</entry> + </row> + + <row> + <entry>DC</entry> + <entry>Discovery Controller.</entry> + </row> + + <row> + <entry>DLP</entry> + <entry>Discovery Log Page. A host will issue a Get Log Page command to retrieve the list of controllers it may connect to.</entry> + </row> + + <row> + <entry>DLPE</entry> + <entry><simpara> + Discovery Log Page Entry. The response + to a Get Log Page command contains a list of DLPEs identifying + each controller that the host is allowed to connect with. + </simpara><simpara> + Note that DLPEs may contain both I/O Controllers (IOCs) + and Discovery Controllers (DCs). DCs listed in DLPEs + are called referrals. <code>stacd</code> only deals with IOCs. + Referrals (DCs) are handled by <code>stafd</code>. + </simpara> + </entry> + </row> + + <row> + <entry>IOC</entry> + <entry>I/O Controller.</entry> + </row> + + <row> + <entry>Manual Config</entry> + <entry>Refers to manually adding entries to <filename>stacd.conf</filename> with the <varname>controller=</varname> parameter.</entry> + </row> + + <row> + <entry>Automatic Config</entry> + <entry>Refers to receiving configuration from a DC as DLPEs</entry> + </row> + + <row> + <entry>External Config</entry> + <entry>Refers to configuration done outside of the <code>nvme-stas</code> framework, for example using <code>nvme-cli</code> commands</entry> + </row> + </tbody> + </tgroup> + </table> + + + <para> + DCs notify hosts of connectivity configuration changes by sending + AENs indicating a "Discovery Log" change. The host uses these AENs as + a trigger to issue a Get Log Page command. The response to this command + is used to update the list of DLPEs containing the controllers + the host is allowed to access. + Upon reception of the current DLPEs, the host will determine + whether DLPEs were added and/or removed, which will trigger the + addition and/or removal of controller connections. This happens in real time + and may affect active connections to controllers including controllers + that support I/O operations (IOCs). A host that was previously + connected to an IOC may suddenly be told that it is no longer + allowed to connect to that IOC and should disconnect from it. + </para> + + <formalpara><title>IOC connection creation</title> + <para> + There are 3 ways to configure IOC connections on a host: + </para> + + <orderedlist> + <listitem> + <para> + Manual Config by adding <varname>controller=</varname> entries + to the <literal>[Controllers]</literal> section (see below). + </para> + </listitem> + <listitem> + <para> + Automatic Config received in the form of + DLPEs from a remote DC. + </para> + </listitem> + <listitem> + <para> + External Config using <code>nvme-cli</code> (e.g. "<code>nvme connect</code>") + </para> + </listitem> + </orderedlist> + </formalpara> + + <formalpara><title>IOC connection removal/prevention</title> + <para> + There are 3 ways to remove (or prevent) connections to an IOC: + </para> + + <orderedlist> + <listitem> + <para> + Manual Config. + <orderedlist numeration='lowerroman'> + <listitem> + <para> + by adding <varname>exclude=</varname> entries to + the <literal>[Controllers]</literal> section (see below). + </para> + </listitem> + <listitem> + <para> + by removing <varname>controller=</varname> entries + from the <literal>[Controllers]</literal> section. + </para> + </listitem> + </orderedlist> + </para> + </listitem> + <listitem> + <para> + Automatic Config. As explained above, a host gets a + new list of DLPEs upon connectivity configuration + changes. On DLPE removal, the host should remove the + connection to the IOC matching that DLPE. This + behavior is configurable using the + <varname>disconnect-scope=</varname> parameter + described below. + </para> + </listitem> + <listitem> + <para> + External Config using <code>nvme-cli</code> (e.g. "<code>nvme + disconnect</code>" or "<code>nvme disconnect-all</code>") + </para> + </listitem> + </orderedlist> + </formalpara> + + <para> + The decision by the host to automatically disconnect from an + IOC following connectivity configuration changes is controlled + by two parameters: <code>disconnect-scope</code> + and <code>disconnect-trtypes</code>. + </para> + + <variablelist> + <varlistentry> + <term><varname>disconnect-scope=</varname></term> + <listitem> + <para> + Takes one of: <parameter>only-stas-connections</parameter>, + <parameter>all-connections-matching-disconnect-trtypes</parameter>, or <parameter>no-disconnect</parameter>. + </para> + + <para> + In theory, hosts should only connect to IOCs that have + been zoned for them. Connections to IOCs that a host + is not zoned to have access to should simply not exist. + In practice, however, users may not want hosts to + disconnect from all IOCs in reaction to connectivity + configuration changes (or at least for some of the IOC + connections). + </para> + + <para> + Some users may prefer for IOC connections to be "sticky" + and only be removed manually (<code>nvme-cli</code> or + <varname>exclude=</varname>) or removed by a system + reboot. Specifically, they don't want IOC connections + to be removed unexpectedly on DLPE removal. These users + may want to set <varname>disconnect-scope</varname> + to <parameter>no-disconnect</parameter>. + </para> + + <para> + It is important to note that when IOC connections + are removed, ongoing I/O transactions will be + terminated immediately. There is no way to tell what + happens to the data being exchanged when such an abrupt + termination happens. If a host was in the middle of writing + to a storage subsystem, there is a chance that outstanding + I/O operations may not successfully complete. + </para> + + <refsect3> + <title>Values:</title> + <variablelist> + <varlistentry> + <term><parameter>only-stas-connections</parameter></term> + <listitem> + <para> + Only remove connections previously made by <code>stacd</code>. + </para> + <para> + In this mode, when a DLPE is removed as a result of + connectivity configuration changes, the corresponding + IOC connection will be removed by <code>stacd</code>. + </para> + <para> + Connections to IOCs made externally, e.g. using <code>nvme-cli</code>, + will not be affected, unless they happen to be duplicates + of connections made by <code>stacd</code>. It's simply not + possible for <code>stacd</code> to tell that a connection + was previously made with <code>nvme-cli</code> (or any other external tool). + So, it's good practice to avoid duplicating + configuration between <code>stacd</code> and external tools. + </para> + <para> + Users wanting to persist some of their IOC connections + regardless of connectivity configuration changes should not use + <code>nvme-cli</code> to make those connections. Instead, + they should hard-code them in <filename>stacd.conf</filename> + with the <varname>controller=</varname> parameter. Using the + <varname>controller=</varname> parameter is the only way for a user + to tell <code>stacd</code> that a connection must be made and + not be deleted "<emphasis>no-matter-what</emphasis>". + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><parameter>all-connections-matching-disconnect-trtypes</parameter></term> + <listitem> + <para> + All connections that match the transport type specified by + <varname>disconnect-trtypes=</varname>, whether they were + made automatically by <code>stacd</code> or externally + (e.g., <code>nvme-cli</code>), will be audited and are + subject to removal on DLPE removal. + </para> + <para> + In this mode, as DLPEs are removed as a result of + connectivity configuration changes, the corresponding + IOC connections will be removed by the host immediately + whether they were made by <code>stacd</code>, <code>nvme-cli</code>, + or any other way. Basically, <code>stacd</code> audits + <emphasis>all</emphasis> IOC connections matching the + transport type specified by <varname>disconnect-trtypes=</varname>. + </para> + <formalpara><title><emphasis>NOTE</emphasis></title> + <para> + This mode implies that <code>stacd</code> will + only allow Manually Configured or Automatically + Configured IOC connections to exist. Externally + Configured connections using <code>nvme-cli</code> + (or other external mechanism) + that do not match any Manual Config + (<filename>stacd.conf</filename>) + or Automatic Config (DLPEs) will get deleted + immediately by <code>stacd</code>. + </para> + </formalpara> + </listitem> + </varlistentry> + + <varlistentry> + <term><parameter>no-disconnect</parameter></term> + <listitem> + <para> + <code>stacd</code> does not disconnect from IOCs + when a DPLE is removed or a <varname>controller=</varname> + entry is removed from <filename>stacd.conf</filename>. + All IOC connections are "sticky". + </para> + + <para> + Instead, users can remove connections + by issuing the <code>nvme-cli</code> + command "<code>nvme disconnect</code>", add an + <varname>exclude=</varname> entry to + <filename>stacd.conf</filename>, or wait + until the next system reboot at which time all + connections will be removed. + </para> + </listitem> + </varlistentry> + </variablelist> + </refsect3> + + <para> + Defaults to <parameter>only-stas-connections</parameter>. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><varname>disconnect-trtypes=</varname></term> + <listitem> + <para> + This parameter only applies when <varname>disconnect-scope</varname> + is set to <parameter>all-connections-matching-disconnect-trtypes</parameter>. + It limits the scope of the audit to specific transport types. + </para> + + <para> + Can take the values <parameter>tcp</parameter>, + <parameter>rdma</parameter>, <parameter>fc</parameter>, or + a combination thereof by separating them with a plus (+) sign. + For example: <parameter>tcp+fc</parameter>. No spaces + are allowed between values and the plus (+) sign. + </para> + + <refsect3> + <title>Values:</title> + <variablelist> + <varlistentry> + <term><parameter>tcp</parameter></term> + <listitem> + <para> + Audit TCP connections. + </para> + </listitem> + </varlistentry> + <varlistentry> + <term><parameter>rdma</parameter></term> + <listitem> + <para> + Audit RDMA connections. + </para> + </listitem> + </varlistentry> + <varlistentry> + <term><parameter>fc</parameter></term> + <listitem> + <para> + Audit Fibre Channel connections. + </para> + </listitem> + </varlistentry> + </variablelist> + </refsect3> + + <para> + Defaults to <parameter>tcp</parameter>. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><varname>connect-attempts-on-ncc=</varname></term> + <listitem> + <para> + The NCC bit (Not Connected to CDC) is a bit returned + by the CDC in the EFLAGS field of the DLPE. Only CDCs + will set the NCC bit. DDCs will always clear NCC to + 0. The NCC bit is a way for the CDC to let hosts + know that the subsystem is currently not reachable + by the CDC. This may indicate that the subsystem is + currently down or that there is an outage on the + section of the network connecting the CDC to the + subsystem. + </para> + + <para> + If a host is currently failing to connect to an I/O + controller and if the NCC bit associated with that + I/O controller is asserted, the host can decide to + stop trying to connect to that subsystem until + connectivity is restored. This will be indicated by + the CDC when it clears the NCC bit. + </para> + + <para> + The parameter <varname>connect-attempts-on-ncc=</varname> + controls whether <code>stacd</code> will take the + NCC bit into account when attempting to connect to + an I/O Controller. Setting <varname>connect-attempts-on-ncc=</varname> + to 0 means that <code>stacd</code> will ignore + the NCC bit and will keep trying to connect. Setting + <varname>connect-attempts-on-ncc=</varname> to a + non-zero value indicates the number of connection + attempts that will be made before <code>stacd</code> + gives up trying. Note that this value should be set + to a value greater than 1. In fact, when set to 1, + <code>stacd</code> will automatically use 2 instead. + The reason for this is simple. It is possible that a + first connect attempt may fail. + </para> + + + <para> + Defaults to <parameter>0</parameter>. + </para> + </listitem> + </varlistentry> + </variablelist> + </refsect2> + + <xi:include href="standard-conf.xml" xpointer="controller"/> + </refsect1> + + <refsect1> + <title>See Also</title> + <para> + <citerefentry> + <refentrytitle>stacd</refentrytitle> + <manvolnum>8</manvolnum> + </citerefentry> + </para> + </refsect1> +</refentry> diff --git a/doc/stacd.service.xml b/doc/stacd.service.xml new file mode 100644 index 0000000..17842af --- /dev/null +++ b/doc/stacd.service.xml @@ -0,0 +1,74 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!--*-nxml-*--> +<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" +"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"> +<!-- + SPDX-License-Identifier: Apache-2.0 + Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +--> +<refentry id="stacd.service"> + <refentryinfo> + <title>stacd.service</title> + <productname>nvme-stas</productname> + <author> + <personname> + <honorific>Mr</honorific> + <firstname>Martin</firstname> + <surname>Belanger</surname> + </personname> + <affiliation> + <orgname>Dell, Inc.</orgname> + </affiliation> + </author> + </refentryinfo> + + <refmeta> + <refentrytitle>stacd.service</refentrytitle> + <manvolnum>8</manvolnum> + </refmeta> + + <refnamediv> + <refname>stacd.service</refname> + <refpurpose>Systemd unit file for the stacd service</refpurpose> + </refnamediv> + + <refsynopsisdiv> + <para> + <filename>/usr/lib/systemd/system/stacd.service</filename> + </para> + </refsynopsisdiv> + + <refsect1> + <title>Description</title> + <para> + <citerefentry> + <refentrytitle>stacd</refentrytitle> + <manvolnum>8</manvolnum> + </citerefentry> + is a system service used to automatically connect to I/O controllers + discovered by <citerefentry> + <refentrytitle>stafd</refentrytitle> + <manvolnum>8</manvolnum> + </citerefentry>. + </para> + </refsect1> + + <refsect1> + <title>See Also</title> + + <para> + <citerefentry> + <refentrytitle>stacd</refentrytitle> + <manvolnum>8</manvolnum> + </citerefentry>, + <citerefentry> + <refentrytitle>stafd</refentrytitle> + <manvolnum>8</manvolnum> + </citerefentry>, + <citerefentry> + <refentrytitle>stas-config.target</refentrytitle> + <manvolnum>8</manvolnum> + </citerefentry> + </para> + </refsect1> +</refentry> diff --git a/doc/stacd.xml b/doc/stacd.xml new file mode 100644 index 0000000..493cfef --- /dev/null +++ b/doc/stacd.xml @@ -0,0 +1,229 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" +"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd" [ +<!ENTITY daemon "stacd"> +<!ENTITY deamondesc "STorage Appliance Connector"> +<!ENTITY control "stacctl"> +]> + +<!-- + SPDX-License-Identifier: Apache-2.0 + Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +--> + +<refentry id="&daemon;" xmlns:xi="http://www.w3.org/2001/XInclude"> + <refentryinfo> + <title>&daemon;</title> + <productname>nvme-stas</productname> + <author> + <personname> + <honorific>Mr</honorific> + <firstname>Martin</firstname> + <surname>Belanger</surname> + </personname> + <affiliation> + <orgname>Dell, Inc.</orgname> + </affiliation> + </author> + </refentryinfo> + + <refmeta> + <refentrytitle>&daemon;</refentrytitle> + <manvolnum>8</manvolnum> + </refmeta> + + <refnamediv> + <refname>&daemon;</refname> + <refpurpose>&deamondesc;</refpurpose> + </refnamediv> + + <refsynopsisdiv> + <cmdsynopsis> + <command>&daemon;</command> + <arg choice="opt" rep="repeat">OPTIONS</arg> + </cmdsynopsis> + </refsynopsisdiv> + + <refsect1> + <title>Description</title> + + <para> + <command>&daemon;</command> + is a system daemon that can be used to automatically connect to + NVMe-oF I/O Controllers using the discovery log pages collected by + <citerefentry> + <refentrytitle>stafd</refentrytitle> + <manvolnum>8</manvolnum> + </citerefentry>. It can also be manually configured with + <citerefentry> + <refentrytitle>&daemon;.conf</refentrytitle> + <manvolnum>5</manvolnum> + </citerefentry> + to connect to I/O Controllers that otherwise cannot be found + automatically. + </para> + </refsect1> + + <refsect1> + <title>Options</title> + <para>The following options are understood:</para> + + <variablelist> + <xi:include href="standard-options.xml" xpointer="help"/> + <xi:include href="standard-options.xml" xpointer="version"/> + </variablelist> + + <varlistentry> + <term><option>-fFILE</option></term> + <term><option>--conf-file=FILE</option></term> + <listitem> + <para> + Specify a different configuration file than + <citerefentry> + <refentrytitle>&daemon;.conf</refentrytitle> + <manvolnum>5</manvolnum> + </citerefentry> + (default: <filename>/etc/stas/&daemon;.conf</filename>). + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><option>-s</option></term> + <term><option>--syslog</option></term> + <listitem> + <para> + Send messages to syslog instead of stdout. Use this when + running &daemon; as a daemon. (default: <literal>false</literal>). + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><option>--tron</option></term> + <listitem> + <para>Trace ON. (default: <literal>false</literal>)</para> + </listitem> + </varlistentry> + + <varlistentry> + <term><option>--idl=FILE</option></term> + <listitem> + <para>Print D-Bus IDL to FILE and exit.</para> + </listitem> + </varlistentry> + </refsect1> + + <refsect1> + <title>Exit status</title> + <para> + On success, 0 is returned, a non-zero failure code otherwise. + </para> + </refsect1> + + <refsect1> + <title>Daemonization</title> + <para> + &daemon; is managed by <code>systemd</code>. The following + operations are supported: + </para> + + <table align='left' frame='all'> + <tgroup cols="2" align='left' colsep='1' rowsep='1'> + <thead> + <row> + <entry>Command</entry> + <entry>Description</entry> + </row> + </thead> + + <tbody> + <row> + <entry><programlisting>$ systemctl start &daemon; </programlisting></entry> + <entry>Start daemon.</entry> + </row> + + <row> + <entry><programlisting>$ systemctl stop &daemon; </programlisting></entry> + <entry>Stop daemon. The <code>SIGTERM</code> signal is used to tell the daemon to stop.</entry> + </row> + + <row> + <entry><programlisting>$ systemctl restart &daemon; </programlisting></entry> + <entry>Effectively a <code>stop</code> + <code>start</code>.</entry> + </row> + + <row> + <entry><programlisting>$ systemctl reload &daemon; </programlisting></entry> + <entry>Reload configuration. This is done in real time without restarting the daemon. The <code>SIGHUP</code> signal is used to tell the daemon to reload its configuration file. Note that configuration parameters that affect connections (e.g. <code>kato</code>), will not apply to existing connections. Only connections established after the configuration was changed will utilize the new configuration parameters.</entry> + </row> + </tbody> + </tgroup> + </table> + </refsect1> + + <refsect1> + <title>Design</title> + + <para> + <command>&daemon;</command> use the <code>GLib</code> main loop. + The <code>GLib</code> Python module provides several low-level + building blocks that <command>&daemon;</command> requires. In + addition, many Python modules "play nice" with <code>GLib</code> + such as <code>dasbus</code> (D-Bus package) and <code>pyudev</code> + (UDev package). <code>GLib</code> also provides additional components + such as timers, signal handlers, and much more. + </para> + </refsect1> + + <refsect1> + <title>Configuration</title> + <para> + <command>&daemon;</command> can automatically set up the I/O + connections to discovered storage subsystems. However, + <command>&daemon;</command> can also operate in a non-automatic + mode based on manually entered configuration. In other words, + storage subsystems can be entered in a configuration file named + <filename>/etc/stas/&daemon;.conf</filename>. + This configuration file also provides additional parameters, + as log-level attributes used for debugging purposes. + </para> + </refsect1> + + <refsect1> + <title>D-Bus API</title> + <para> + The interface to <command>&daemon;</command> is D-Bus. + This allows other programs, such as <command>&control;</command>, + to communicate with <command>&daemon;</command>. The D-Bus address + is <code>org.nvmexpress.stac</code>. + </para> + </refsect1> + + <refsect1> + <title>See Also</title> + + <para> + <citerefentry> + <refentrytitle>&daemon;.conf</refentrytitle> + <manvolnum>5</manvolnum> + </citerefentry>, + <citerefentry> + <refentrytitle>&daemon;.service</refentrytitle> + <manvolnum>8</manvolnum> + </citerefentry>, + <citerefentry> + <refentrytitle>stacctl</refentrytitle> + <manvolnum>1</manvolnum> + </citerefentry>, + <citerefentry> + <refentrytitle>stafd</refentrytitle> + <manvolnum>8</manvolnum> + </citerefentry>, + <citerefentry> + <refentrytitle>org.nvmexpress.stac</refentrytitle> + <manvolnum>5</manvolnum> + </citerefentry>. + </para> + </refsect1> +</refentry> diff --git a/doc/stafctl.xml b/doc/stafctl.xml new file mode 100644 index 0000000..beb8097 --- /dev/null +++ b/doc/stafctl.xml @@ -0,0 +1,205 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!--*-nxml-*--> +<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" +"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"> +<!-- + SPDX-License-Identifier: Apache-2.0 + Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +--> +<refentry id="stafctl" xmlns:xi="http://www.w3.org/2001/XInclude"> + <refentryinfo> + <title>stafctl</title> + <productname>nvme-stas</productname> + <author> + <personname> + <honorific>Mr</honorific> + <firstname>Martin</firstname> + <surname>Belanger</surname> + </personname> + <affiliation> + <orgname>Dell, Inc.</orgname> + </affiliation> + </author> + </refentryinfo> + + <refmeta> + <refentrytitle>stafctl</refentrytitle> + <manvolnum>1</manvolnum> + </refmeta> + + <refnamediv> + <refname>stafctl</refname> + <refpurpose>STorage Appliance Finder (STAF) utility program</refpurpose> + </refnamediv> + + <refsynopsisdiv> + <cmdsynopsis> + <command>stafctl</command> + <arg choice="opt" rep="repeat">OPTIONS</arg> + <arg choice="req">COMMAND</arg> + <arg choice="opt" rep="repeat">OPTIONS</arg> + </cmdsynopsis> + </refsynopsisdiv> + + <refsect1> + <title>Description</title> + + <para> + <command>stafctl</command> is a tool that can be used to communicate + with the <citerefentry><refentrytitle>stafd</refentrytitle> + <manvolnum>8</manvolnum></citerefentry> daemon to retrieve + operational data. + </para> + </refsect1> + + <refsect1> + <title>Commands</title> + <para>The following commands are understood:</para> + + <variablelist> + <xi:include href="standard-options.xml" xpointer="tron"/> + <xi:include href="standard-options.xml" xpointer="troff"/> + <xi:include href="standard-options.xml" xpointer="status"/> + + <varlistentry> + <term><command>ls</command></term> + <listitem> + <para> + Show the list of discovery controllers. This will list + all the controllers configured in + <citerefentry><refentrytitle>stafd.conf</refentrytitle> + <manvolnum>5</manvolnum></citerefentry> as well as those + discovered with mDNS service discovery. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><command>dlp</command></term> + <listitem> + <para>Show discovery log pages.</para> + </listitem> + </varlistentry> + + <varlistentry> + <term><command>adlp</command></term> + <listitem> + <para>Show all discovery log pages.</para> + </listitem> + </varlistentry> + </variablelist> + </refsect1> + + <refsect1> + <title>Options</title> + + <para>The following options are understood:</para> + + <variablelist> + <xi:include href="standard-options.xml" xpointer="help"/> + <xi:include href="standard-options.xml" xpointer="version"/> + <xi:include href="standard-options.xml" xpointer="detailed"/> + <xi:include href="standard-options.xml" xpointer="transport"/> + <xi:include href="standard-options.xml" xpointer="traddr"/> + <xi:include href="standard-options.xml" xpointer="trsvcid"/> + <xi:include href="standard-options.xml" xpointer="host-traddr"/> + <xi:include href="standard-options.xml" xpointer="host-iface"/> + <xi:include href="standard-options.xml" xpointer="nqn"/> + </variablelist> + </refsect1> + + <refsect1> + <title>Values</title> + + <variablelist> + <xi:include href="standard-options.xml" xpointer="TRTYPE-value"/> + <xi:include href="standard-options.xml" xpointer="TRADDR-value"/> + <xi:include href="standard-options.xml" xpointer="TRSVCID-value"/> + <xi:include href="standard-options.xml" xpointer="IFACE-value"/> + <xi:include href="standard-options.xml" xpointer="NQN-value"/> + </variablelist> + </refsect1> + + <refsect1> + <title>Exit status</title> + <para> + On success, 0 is returned; otherwise, a non-zero failure code is + returned. + </para> + </refsect1> + + <refsect1> + <title>Examples</title> + + <example> + <title>List all the discovery controllers</title> + <programlisting>$ stafctl ls +[{'device': 'nvme0', + 'host-iface': '', + 'host-traddr': '', + 'subsysnqn': 'nqn.2014-08.org.nvmexpress.discovery', + 'traddr': '::1', + 'transport': 'tcp', + 'trsvcid': '8009'}]</programlisting> + </example> + + <example> + <title>Enable tracing</title> + <programlisting>$ stafctl tron</programlisting> + </example> + + <example> + <title> + Show discovery log pages from a specific discovery controller + </title> + + <programlisting>$ stafctl dlp --transport tcp --traddr ::1 --trsvcid 8009 +[{'adrfam': 'ipv6', + 'asqsz': '32', + 'cntlid': '65535', + 'portid': '1', + 'subnqn': 'nqn.1988-11.com.dell:PowerSANxxx:01:20210225100113-454f73093ceb4847a7bdfc6e34ae8e28', + 'subtype': 'nvme', + 'traddr': '::1', + 'treq': 'disable sqflow', + 'trsvcid': '8009', + 'trtype': 'tcp'}, + {'adrfam': 'ipv6', + 'asqsz': '32', + 'cntlid': '65535', + 'portid': '1', + 'subnqn': 'nqn.1988-11.com.dell:PowerSANxxx:01:20210225100113-454f73093ceb4847a7bdfc6e34aedead', + 'subtype': 'nvme', + 'traddr': '::1', + 'treq': 'disable sqflow', + 'trsvcid': '8009', + 'trtype': 'tcp'}, + {'adrfam': 'ipv6', + 'asqsz': '32', + 'cntlid': '65535', + 'portid': '1', + 'subnqn': 'nqn.1988-11.com.dell:PowerSANxxx:01:20210225100113-454f73093ceb4847a7bdfc6e34aebeef', + 'subtype': 'nvme', + 'traddr': '::1', + 'treq': 'disable sqflow', + 'trsvcid': '8009', + 'trtype': 'tcp'}]</programlisting> + </example> + </refsect1> + + <refsect1> + <title>See Also</title> + + <para> + <citerefentry> + <refentrytitle>stafd.conf</refentrytitle> + <manvolnum>5</manvolnum> + </citerefentry>, + <citerefentry> + <refentrytitle>stafd</refentrytitle> + <manvolnum>8</manvolnum> + </citerefentry> + <citerefentry project="man-pages"/> + </para> + </refsect1> +</refentry> diff --git a/doc/stafd.conf.xml b/doc/stafd.conf.xml new file mode 100644 index 0000000..da0b842 --- /dev/null +++ b/doc/stafd.conf.xml @@ -0,0 +1,280 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!--*-nxml-*--> +<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" +"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"> +<!-- + SPDX-License-Identifier: Apache-2.0 + Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +--> +<refentry id="stafd.conf" xmlns:xi="http://www.w3.org/2001/XInclude"> + <refentryinfo> + <title>stafd.conf</title> + <productname>nvme-stas</productname> + + <author> + <personname> + <honorific>Mr</honorific> + <firstname>Martin</firstname> + <surname>Belanger</surname> + </personname> + + <affiliation> + <orgname>Dell, Inc.</orgname> + </affiliation> + </author> + </refentryinfo> + + <refmeta> + <refentrytitle>stafd.conf</refentrytitle> + <manvolnum>5</manvolnum> + </refmeta> + + <refnamediv> + <refname>stafd.conf</refname> + + <refpurpose> + <citerefentry project="man-pages"> + <refentrytitle>stafd</refentrytitle> + <manvolnum>8</manvolnum> + </citerefentry> + configuration file + </refpurpose> + </refnamediv> + + <refsynopsisdiv> + <para> + <filename>/etc/stas/stafd.conf</filename> + </para> + </refsynopsisdiv> + + <refsect1> + <title>Description</title> + + <para> + When <citerefentry project="man-pages"><refentrytitle>stafd</refentrytitle> + <manvolnum>8</manvolnum></citerefentry> starts up, it reads its + configuration from <filename>stafd.conf</filename>. + </para> + </refsect1> + + <refsect1> + <title>Configuration File Format</title> + <para> + <filename>stafd.conf</filename> is a plain text file divided into + sections, with configuration entries in the style + <replaceable>key</replaceable>=<replaceable>value</replaceable>. + Spaces immediately before or after the <literal>=</literal> are + ignored. Empty lines are ignored as well as lines starting with + <literal>#</literal>, which may be used for commenting. + </para> + </refsect1> + + <refsect1> + <title>Options</title> + + <refsect2> + <title>[Global] section</title> + <para> + The following options are available in the + <literal>[Global]</literal> section: + </para> + + <variablelist> + <xi:include href="standard-conf.xml" xpointer="tron"/> + <xi:include href="standard-conf.xml" xpointer="hdr-digest"/> + <xi:include href="standard-conf.xml" xpointer="data-digest"/> + <xi:include href="standard-conf.xml" xpointer="kato"/> + <xi:include href="standard-conf.xml" xpointer="ip-family"/> + <xi:include href="standard-conf.xml" xpointer="queue-size"/> + <xi:include href="standard-conf.xml" xpointer="reconnect-delay"/> + <xi:include href="standard-conf.xml" xpointer="ctrl-loss-tmo"/> + <xi:include href="standard-conf.xml" xpointer="disable-sqflow"/> + + <varlistentry> + <term><varname>ignore-iface=</varname></term> + <listitem> + <para> + Takes a boolean argument. This option controls how + connections with Discovery Controllers (DC) are made. + </para> + + <para> + DCs are automatically discovered using DNS-SD/mDNS. + mDNS provides the DC's IP address and the interface + on which the DC was discovered. + </para> + + <para> + There is no guarantee that there will be a route to + reach that DC. However, we can use the socket option + SO_BINDTODEVICE to force the connection to be made + on a specific interface instead of letting the + routing tables decide where to make the connection. + </para> + + <para> + This option determines whether <code>stafd</code> + will use SO_BINDTODEVICE to force connections on an + interface or just rely on the routing tables. The + default is to use SO_BINDTODEVICE, in other words, + <code>stafd</code> does not ignore the interface by + default. + </para> + <para> + Defaults to <parameter>false</parameter>. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><varname>pleo=</varname></term> + <listitem> + <para> + Port Local Entries Only. Takes a string argument + <parameter>enabled</parameter> or + <parameter>disabled</parameter>. This option is sent in + the LSP field (Log SPecific) of the Get Discovery Log + Page (DLP) command. It is used by <code>stafd</code> to + tell Discovery Controllers (DC) whether the response to + a Get DLP command should contain all the NVM subsystems + or only those reachable by the host on the interface + where the Get DLP command was issued by the host. + </para> + + <para> + This parameter was introduced in TP8010. When + <varname>pleo=</varname><parameter>enabled</parameter>, + then the DC shall return records for only NVM subsystem + ports that are presented through the same NVM subsystem + port that received the Get Log Page command. When + <varname>pleo=</varname><parameter>disabled</parameter>, + then the DC may return all the NVM subsystem ports + that it holds, even those that can only be reached + on NVM subsystem ports that did not receive the Get + Log Page command. In other words, the host may not + even be able to reach those subsystems. + </para> + + <para> + Defaults to <parameter>enabled</parameter>. + </para> + </listitem> + </varlistentry> + </variablelist> + </refsect2> + + <refsect2> + <title>[Service Discovery] section</title> + + <para> + The following options are available in the + <literal>[Service Discovery]</literal> section: + </para> + + <variablelist> + <varlistentry> + <term><varname>zeroconf=</varname></term> + + <listitem> + <para> + Enable zeroconf provisioning using DNS-SD/mDNS. + Takes a string argument <parameter>enabled</parameter> or + <parameter>disabled</parameter>. + </para> + <para> + When <parameter>enabled</parameter>, the default, + <code>stafd</code> makes a request with the + Avahi daemon to locate Discovery Controllers using + DNS-SD/mDNS. + </para> + <para> + Discovery Controllers that support zeroconf advertize + themselves over mDNS with the service type + <literal>_nvme-disc._tcp</literal>. + </para> + <para> + Defaults to <parameter>true</parameter>. + </para> + </listitem> + </varlistentry> + </variablelist> + </refsect2> + + <refsect2> + <title>[Discovery controller connection management] section</title> + + <para> + The following options are available in the + <literal>[Discovery controller connection management]</literal> section: + </para> + + <varlistentry> + <term><varname>persistent-connections=</varname></term> + <listitem> + <para> + Takes a boolean argument. Whether connections to + Discovery Controllers (DC) are persistent. When + true, connections initiated by stafd will persists + even when stafd is stopped. When + <parameter>false</parameter>, <code>stafd</code> + will disconnect from all DCs it is connected to on + exit. + </para> + <para> + Defaults to <parameter>false</parameter>. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><varname>zeroconf-connections-persistence=</varname></term> + <listitem> + <para> + Takes a unit-less value in seconds, or a time span value + such as "72hours" or "5days". A value of 0 means no + persistence. In other words, configuration acquired through + zeroconf (mDNS service discovery) will be removed + immediately when mDNS no longer reports the presence of + a Discovery Controller (DC) and connectivity to that DC + is lost. A value of -1 means that configuration acquired + through zeroconf will persist forever. + </para> + + <para> + This is used for the case where a DC that was discovered + through mDNS service discovery no longer advertises + itself through mDNS and can no longer be connected to. + For example, the DC had some catastrophic failure + (e.g. power surge) and needs to be replaced. In that + case, the connection to that DC can never be restored + and a replacement DC will be needed. The replacement + DC will likely have a different NQN (or IP address). + In that scenario, the host won't be able to determine + that the old DC is not coming back. It won't know either + that a newly discovered DC is really the replacement for + the old one. For that reason, the host needs a way to + "age" zeroconf-acquired configuration and remove it + automatically after a certain amount of time. This is + what this parameter is for. + </para> + + <para> + Defaults to <parameter>72hours</parameter>. + </para> + </listitem> + </varlistentry> + </refsect2> + + <xi:include href="standard-conf.xml" xpointer="controller"/> + </refsect1> + + <refsect1> + <title>See Also</title> + <para> + <citerefentry> + <refentrytitle>stafd</refentrytitle> + <manvolnum>8</manvolnum> + </citerefentry> + </para> + </refsect1> +</refentry> diff --git a/doc/stafd.service.xml b/doc/stafd.service.xml new file mode 100644 index 0000000..bc9ba82 --- /dev/null +++ b/doc/stafd.service.xml @@ -0,0 +1,67 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!--*-nxml-*--> +<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" +"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"> +<!-- + SPDX-License-Identifier: Apache-2.0 + Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +--> +<refentry id="stafd.service"> + <refentryinfo> + <title>stafd.service</title> + <productname>nvme-stas</productname> + <author> + <personname> + <honorific>Mr</honorific> + <firstname>Martin</firstname> + <surname>Belanger</surname> + </personname> + <affiliation> + <orgname>Dell, Inc.</orgname> + </affiliation> + </author> + </refentryinfo> + + <refmeta> + <refentrytitle>stafd.service</refentrytitle> + <manvolnum>8</manvolnum> + </refmeta> + + <refnamediv> + <refname>stafd.service</refname> + <refpurpose>Systemd unit file for the stafd service</refpurpose> + </refnamediv> + + <refsynopsisdiv> + <para> + <filename>/usr/lib/systemd/system/stafd.service</filename> + </para> + </refsynopsisdiv> + + <refsect1> + <title>Description</title> + <para> + <citerefentry> + <refentrytitle>stafd</refentrytitle> + <manvolnum>8</manvolnum> + </citerefentry> + is a system service used to automatically locate NVMe-oF Discovery + Controllers using mDNS service discovery. + </para> + </refsect1> + + <refsect1> + <title>See Also</title> + + <para> + <citerefentry> + <refentrytitle>stafd</refentrytitle> + <manvolnum>8</manvolnum> + </citerefentry>, + <citerefentry> + <refentrytitle>stas-config.target</refentrytitle> + <manvolnum>8</manvolnum> + </citerefentry> + </para> + </refsect1> +</refentry> diff --git a/doc/stafd.xml b/doc/stafd.xml new file mode 100644 index 0000000..10e454e --- /dev/null +++ b/doc/stafd.xml @@ -0,0 +1,237 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" +"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd" [ +<!ENTITY daemon "stafd"> +<!ENTITY deamondesc "STorage Appliance Finder"> +<!ENTITY control "stafctl"> +]> + +<!-- + SPDX-License-Identifier: Apache-2.0 + Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +--> + +<refentry id="&daemon;" xmlns:xi="http://www.w3.org/2001/XInclude"> + <refentryinfo> + <title>&daemon;</title> + <productname>nvme-stas</productname> + <author> + <personname> + <honorific>Mr</honorific> + <firstname>Martin</firstname> + <surname>Belanger</surname> + </personname> + <affiliation> + <orgname>Dell, Inc.</orgname> + </affiliation> + </author> + </refentryinfo> + + <refmeta> + <refentrytitle>&daemon;</refentrytitle> + <manvolnum>8</manvolnum> + </refmeta> + + <refnamediv> + <refname>&daemon;</refname> + <refpurpose>&deamondesc;</refpurpose> + </refnamediv> + + <refsynopsisdiv> + <cmdsynopsis> + <command>&daemon;</command> + <arg choice="opt" rep="repeat">OPTIONS</arg> + </cmdsynopsis> + </refsynopsisdiv> + + <refsect1> + <title>Description</title> + + <para> + <command>&daemon;</command> + is a system daemon that can be used to automatically locate and + connect to NVMe-oF Discovery Controllers using mDNS service discovery. + It can also be manually configured with + <citerefentry> + <refentrytitle>&daemon;.conf</refentrytitle> + <manvolnum>5</manvolnum> + </citerefentry> + to connect to Discovery Controllers that cannot be located using + mDNS. + </para> + </refsect1> + + <refsect1> + <title>Options</title> + <para>The following options are understood:</para> + + <variablelist> + <xi:include href="standard-options.xml" xpointer="help"/> + <xi:include href="standard-options.xml" xpointer="version"/> + </variablelist> + + <varlistentry> + <term><option>-fFILE</option></term> + <term><option>--conf-file=FILE</option></term> + <listitem> + <para> + Specify a different configuration file than + <citerefentry> + <refentrytitle>&daemon;.conf</refentrytitle> + <manvolnum>5</manvolnum> + </citerefentry> + (default: <filename>/etc/stas/&daemon;.conf</filename>). + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><option>-s</option></term> + <term><option>--syslog</option></term> + <listitem> + <para> + Send messages to syslog instead of stdout. Use this when + running &daemon; as a daemon. (default: <literal>false</literal>). + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><option>--tron</option></term> + <listitem> + <para>Trace ON. (default: <literal>false</literal>)</para> + </listitem> + </varlistentry> + + <varlistentry> + <term><option>--idl=FILE</option></term> + <listitem> + <para>Print D-Bus IDL to FILE and exit.</para> + </listitem> + </varlistentry> + </refsect1> + + <refsect1> + <title>Exit status</title> + <para> + On success, 0 is returned, a non-zero failure code otherwise. + </para> + </refsect1> + + <refsect1> + <title>Daemonization</title> + <para> + &daemon; is managed by <code>systemd</code>. The following + operations are supported: + </para> + + <table frame='all'> + <tgroup cols="2" align='left' colsep='1' rowsep='1'> + <thead> + <row> + <entry>Command</entry> + <entry>Description</entry> + </row> + </thead> + + <tbody> + <row> + <entry><programlisting>$ systemctl start &daemon; </programlisting></entry> + <entry>Start daemon.</entry> + </row> + + <row> + <entry><programlisting>$ systemctl stop &daemon; </programlisting></entry> + <entry>Stop daemon. The <code>SIGTERM</code> signal is used to tell the daemon to stop.</entry> + </row> + + <row> + <entry><programlisting>$ systemctl restart &daemon; </programlisting></entry> + <entry>Effectively a <code>stop</code> + <code>start</code>.</entry> + </row> + + <row> + <entry><programlisting>$ systemctl reload &daemon; </programlisting></entry> + <entry>Reload configuration. This is done in real time without restarting the daemon. The <code>SIGHUP</code> signal is used to tell the daemon to reload its configuration file. Note that configuration parameters that affect connections (e.g. <code>kato</code>), will not apply to existing connections. Only connections established after the configuration was changed will utilize the new configuration parameters.</entry> + </row> + </tbody> + </tgroup> + </table> + </refsect1> + + <refsect1> + <title>Design</title> + + <para> + <command>&daemon;</command> use the <code>GLib</code> main loop. + The <code>GLib</code> Python module provides several low-level + building blocks that <command>&daemon;</command> requires. In + addition, many Python modules "play nice" with <code>GLib</code> + such as <code>dasbus</code> (D-Bus package) and <code>pyudev</code> + (UDev package). <code>GLib</code> also provides additional components + such as timers, signal handlers, and much more. + </para> + + <para> + <command>&daemon;</command> connects to the <code>avahi-daemon</code> + using D-Bus. The <code>avahi-daemon</code>, or simply + <emphasis>Avahi</emphasis>, is an mDNS discovery service used for + zero-configuration networking (zeroconf). <command>&daemon;</command> + registers with Avahi to automatically locate Central Discovery + Controllers (CDC) and Direct Discovery Controllers (DDC). When Avahi + finds Discovery Controllers (DC), it notifies <command>&daemon;</command> + which connects to the DC with the help of the <code>libnvme</code> library. + Once a connection to a DC is established, <command>&daemon;</command> + can retrieve the <emphasis>discovery log pages</emphasis> from + that DC and cache them in memory. + </para> + </refsect1> + + + <refsect1> + <title>Configuration</title> + <para> + <command>&daemon;</command> can automatically locate discovery + controllers (DC) with the help of Avahi and connect to them. However, + <command>&daemon;</command> can also operate in a non-automatic + mode based on manually entered configuration. In other words, + DCs can be entered in a configuration named + <filename>/etc/stas/&daemon;.conf</filename>. + This configuration file also provides additional parameters, such + as log-level attributes used for debugging purposes. + </para> + </refsect1> + + <refsect1> + <title>D-Bus API</title> + <para> + The interface to <command>&daemon;</command> is D-Bus. + This allows other programs, such as <command>&control;</command>, + to communicate with <command>&daemon;</command>. The D-Bus address + is <code>org.nvmexpress.staf</code>. + </para> + </refsect1> + + <refsect1> + <title>See Also</title> + + <para> + <citerefentry> + <refentrytitle>&daemon;.conf</refentrytitle> + <manvolnum>5</manvolnum> + </citerefentry>, + <citerefentry> + <refentrytitle>&daemon;.service</refentrytitle> + <manvolnum>8</manvolnum> + </citerefentry>, + <citerefentry> + <refentrytitle>stafctl</refentrytitle> + <manvolnum>1</manvolnum> + </citerefentry>, + <citerefentry> + <refentrytitle>org.nvmexpress.staf</refentrytitle> + <manvolnum>5</manvolnum> + </citerefentry>. + </para> + </refsect1> +</refentry> diff --git a/doc/standard-conf.xml b/doc/standard-conf.xml new file mode 100644 index 0000000..50d4fe5 --- /dev/null +++ b/doc/standard-conf.xml @@ -0,0 +1,562 @@ +<?xml version="1.0"?> +<!DOCTYPE refsection PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" + "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd"> + +<!-- + SPDX-License-Identifier: Apache-2.0 + Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +--> + +<root> + + <variablelist> + <varlistentry id='tron'> + <term><varname>tron=</varname></term> + + <listitem id='tron-text'> + <para> + Trace ON. Takes a boolean argument. If <parameter>true</parameter>, + enables full code tracing. The trace will be displayed in + the system log such as systemd's journal. Defaults to + <parameter>false</parameter>. + </para> + </listitem> + </varlistentry> + + <varlistentry id='hdr-digest'> + <term><varname>hdr-digest=</varname></term> + + <listitem id='hdr-digest-text'> + <para> + Enable Protocol Data Unit (PDU) Header Digest. Takes a + boolean argument. NVMe/TCP facilitates an optional PDU + Header digest. Digests are calculated using the CRC32C + algorithm. If <parameter>true</parameter>, Header Digests + are inserted in PDUs and checked for errors. Defaults to + <parameter>false</parameter>. + </para> + </listitem> + </varlistentry> + + <varlistentry id='data-digest'> + <term><varname>data-digest=</varname></term> + + <listitem id='data-digest-text'> + <para> + Enable Protocol Data Unit (PDU) Data Digest. Takes a + boolean argument. NVMe/TCP facilitates an optional PDU + Data digest. Digests are calculated using the CRC32C + algorithm. If <parameter>true</parameter>, Data Digests + are inserted in PDUs and checked for errors. Defaults to + <parameter>false</parameter>. + </para> + </listitem> + </varlistentry> + + <varlistentry id='kato'> + <term><varname>kato=</varname></term> + + <listitem id='kato-text'> + <para> + Keep Alive Timeout (KATO) in seconds. Takes an unsigned + integer. This field specifies the timeout value for the Keep + Alive feature in seconds. Defaults to 30 seconds for + Discovery Controller connections and 120 seconds for I/O + Controller connections. + </para> + </listitem> + </varlistentry> + + <varlistentry id='ip-family'> + <term><varname>ip-family=</varname></term> + + <listitem id='ip-family-text'> + <para> + Takes a string argument. With this you can specify + whether IPv4, IPv6, or both are supported when + connecting to a Controller. Connections will not be + attempted to IP addresses (whether discovered or + manually configured with <varname>controller=</varname>) + disabled by this option. If an invalid value + is entered, then the default (see below) will apply. + </para> + + <para> + Choices are <parameter>ipv4</parameter>, <parameter>ipv6</parameter>, or <parameter>ipv4+ipv6</parameter>. + </para> + + <para> + Defaults to <parameter>ipv4+ipv6</parameter>. + </para> + </listitem> + </varlistentry> + + <varlistentry id='queue-size'> + <term><varname>queue-size=</varname></term> + + <listitem id='queue-size-text'> + <para> + Takes a value in the range 16...1024. + </para> + + <para> + Overrides the default number of elements in the I/O queues + created by the driver. This option will be ignored for + discovery, but will be passed on to the subsequent connect + call. + </para> + + <para>Note: This parameter is identical to that provided by nvme-cli.</para> + + <para> + Defaults to <parameter>128</parameter>. + </para> + </listitem> + </varlistentry> + + <varlistentry id='reconnect-delay'> + <term><varname>reconnect-delay=</varname></term> + + <listitem id='reconnect-delay-text'> + <para> + Takes a value in the range 1 to N seconds. + </para> + + <para> + Overrides the default delay before reconnect is attempted + after a connect loss. + </para> + + <para>Note: This parameter is identical to that provided by nvme-cli.</para> + + <para> + Defaults to <parameter>10</parameter>. Retry to connect every 10 seconds. + </para> + </listitem> + </varlistentry> + + <varlistentry id='ctrl-loss-tmo'> + <term><varname>ctrl-loss-tmo=</varname></term> + + <listitem id='ctrl-loss-tmo-text'> + <para> + Takes a value in the range -1, 0, ..., N seconds. -1 means + retry forever. 0 means do not retry. + </para> + + <para> + Overrides the default controller loss timeout period (in seconds). + </para> + + <para>Note: This parameter is identical to that provided by nvme-cli.</para> + + <para> + Defaults to <parameter>600</parameter> seconds (10 minutes). + </para> + </listitem> + </varlistentry> + + <varlistentry id='disable-sqflow'> + <term><varname>disable-sqflow=</varname></term> + + <listitem id='disable-sqflow-text'> + <para> + Takes a boolean argument. Disables SQ flow control to omit + head doorbell update for submission queues when sending nvme + completions. + </para> + + <para>Note: This parameter is identical to that provided by nvme-cli.</para> + + <para> + Defaults to <parameter>false</parameter>. + </para> + </listitem> + </varlistentry> + </variablelist> + + <refsect2 id='controller'> + <title>[Controllers] section</title> + + <para>The following options are available in the + <literal>[Controllers]</literal> section:</para> + + <varlistentry> + <term><varname>controller=</varname></term> + + <listitem id='controller-text'> + <para> + Controllers are specified with the <varname>controller</varname> + option. This option may be specified more than once to specify + more than one controller. The format is one line per Controller + composed of a series of fields separated by semi-colons as follows: + </para> + + <programlisting>controller=transport=[trtype];traddr=[traddr];trsvcid=[trsvcid];host-traddr=[traddr],host-iface=[iface];nqn=[nqn] + </programlisting> + + <refsect3> + <title>Fields</title> + <variablelist> + <varlistentry id='transport'> + <term><varname>transport=</varname></term> + + <listitem id='transport-text'> + <para> + This is a mandatory field that specifies the + network fabric being used for a + NVMe-over-Fabrics network. Current + <parameter>trtype</parameter> values understood + are: + </para> + + <table id='transport-types'> + <title>Transport type</title> + <tgroup cols="2"> + <thead> + <row> + <entry>trtype</entry> + <entry>Definition</entry> + </row> + </thead> + + <tbody> + <row> + <entry>rdma</entry> + <entry> + The network fabric is an rdma network (RoCE, iWARP, Infiniband, basic rdma, etc) + </entry> + </row> + + <row> + <entry>fc</entry> + <entry> + The network fabric is a Fibre Channel network. + </entry> + </row> + + <row> + <entry>tcp</entry> + <entry> + The network fabric is a TCP/IP network. + </entry> + </row> + + <row> + <entry>loop</entry> + <entry> + Connect to a NVMe over Fabrics target on the local host + </entry> + </row> + </tbody> + </tgroup> + </table> + </listitem> + </varlistentry> + + <varlistentry id='tradd'> + <term> + <varname>traddr=</varname> + </term> + + <listitem> + <para> + This is a mandatory field that specifies the + network address of the Controller. For + transports using IP addressing (e.g. rdma) + this should be an IP-based address (ex. + IPv4, IPv6). It could also be a resolvable + host name (e.g. localhost). + </para> + </listitem> + </varlistentry> + + <varlistentry id='trsvcid'> + <term> + <varname>trsvcid=</varname> + </term> + + <listitem> + <para> + This is an optional field that specifies the + transport service id. For transports using + IP addressing (e.g. rdma, tcp) this field is + the port number. + </para> + + <para> + Depending on the transport type, this field + will default to either 8009 or 4420 as + follows. + </para> + + <para> + UDP port 4420 and TCP port 4420 have been + assigned by IANA for use by NVMe over + Fabrics. NVMe/RoCEv2 controllers use UDP + port 4420 by default. NVMe/iWARP controllers + use TCP port 4420 by default. + </para> + + <para> + TCP port 4420 has been assigned for use by + NVMe over Fabrics and TCP port 8009 has been + assigned by IANA for use by NVMe over + Fabrics discovery. TCP port 8009 is the + default TCP port for NVMe/TCP discovery + controllers. There is no default TCP port + for NVMe/TCP I/O controllers, the Transport + Service Identifier (TRSVCID) field in the + Discovery Log Entry indicates the TCP port + to use. + </para> + + <para> + The TCP ports that may be used for NVMe/TCP + I/O controllers include TCP port 4420, and + the Dynamic and/or Private TCP ports (i.e., + ports in the TCP port number range from + 49152 to 65535). NVMe/TCP I/O controllers + should not use TCP port 8009. TCP port 4420 + shall not be used for both NVMe/iWARP and + NVMe/TCP at the same IP address on the same + network. + </para> + + <para> + Ref: + <ulink + url="https://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.xhtml?search=nvme"> + IANA Service names port numbers + </ulink> + </para> + </listitem> + </varlistentry> + + <varlistentry id='nqn'> + <term><varname>nqn=</varname></term> + <listitem> + <para> + This field specifies the Controller's NVMe + Qualified Name. + </para> + <para> + This field is mandatory for I/O Controllers, but is optional for + Discovery Controllers (DC). For the latter, the NQN will default + to the well-known DC NQN: <literal>nqn.2014-08.org.nvmexpress.discovery</literal> + if left undefined. + </para> + </listitem> + </varlistentry> + + <varlistentry id='host-traddr'> + <term><varname>host-traddr=</varname></term> + <listitem> + <para> + This is an optional field that specifies the + network address used on the host to connect + to the Controller. For TCP, this sets the + source address on the socket. + </para> + </listitem> + </varlistentry> + + <varlistentry id='host-iface'> + <term><varname>host-iface=</varname></term> + <listitem> + <para> + This is an optional field that specifies the + network interface used on the host to + connect to the Controller (e.g. IP eth1, + enp2s0, enx78e7d1ea46da). This forces the + connection to be made on a specific + interface instead of letting the system + decide. + </para> + </listitem> + </varlistentry> + + <varlistentry id='dhchap-ctrl-secret'> + <term><varname>dhchap-ctrl-secret=</varname></term> + <listitem> + <para> + This is an optional field that specifies the + NVMe In-band authentication controller secret + (i.e. key) for bi-directional authentication; + needs to be in ASCII format as specified in + NVMe 2.0 section 8.13.5.8 'Secret representation'. + Bi-directional authentication will be attempted + when present. + </para> + </listitem> + </varlistentry> + + <varlistentry id='hdr-digest-override'> + <term><varname>hdr-digest=</varname></term> + <listitem> + <para> + See definition in [Global] section. This is + an optional field used to override the value + specified in the [Global] section. + </para> + </listitem> + </varlistentry> + + <varlistentry id='data-digest-override'> + <term><varname>data-digest=</varname></term> + <listitem> + <para> + See definition in [Global] section. This is + an optional field used to override the value + specified in the [Global] section. + </para> + </listitem> + </varlistentry> + + <varlistentry id='nr-io-queues-override'> + <term><varname>nr-io-queues=</varname></term> + <listitem> + <para> + See definition in [Global] section. This is + an optional field used to override the value + specified in the [Global] section. + </para> + </listitem> + </varlistentry> + + <varlistentry id='nr-write-queues-override'> + <term><varname>nr-write-queues=</varname></term> + <listitem> + <para> + See definition in [Global] section. This is + an optional field used to override the value + specified in the [Global] section. + </para> + </listitem> + </varlistentry> + + <varlistentry id='nr-poll-queues-override'> + <term><varname>nr-poll-queues=</varname></term> + <listitem> + <para> + See definition in [Global] section. This is + an optional field used to override the value + specified in the [Global] section. + </para> + </listitem> + </varlistentry> + + <varlistentry id='queue-size-override'> + <term><varname>queue-size=</varname></term> + <listitem> + <para> + See definition in [Global] section. This is + an optional field used to override the value + specified in the [Global] section. + </para> + </listitem> + </varlistentry> + + <varlistentry id='kato-override'> + <term><varname>kato=</varname></term> + <listitem> + <para> + See definition in [Global] section. This is + an optional field used to override the value + specified in the [Global] section. + </para> + </listitem> + </varlistentry> + + <varlistentry id='reconnect-delay-override'> + <term><varname>reconnect-delay=</varname></term> + <listitem> + <para> + See definition in [Global] section. This is + an optional field used to override the value + specified in the [Global] section. + </para> + </listitem> + </varlistentry> + + <varlistentry id='ctrl-loss-tmo-override'> + <term><varname>ctrl-loss-tmo=</varname></term> + <listitem> + <para> + See definition in [Global] section. This is + an optional field used to override the value + specified in the [Global] section. + </para> + </listitem> + </varlistentry> + + <varlistentry id='disable-sqflow-override'> + <term><varname>disable-sqflow=</varname></term> + <listitem> + <para> + See definition in [Global] section. This is + an optional field used to override the value + specified in the [Global] section. + </para> + </listitem> + </varlistentry> + </variablelist> + </refsect3> + + <para> + Examples: + <programlisting>controller = transport=tcp;traddr=localhost;trsvcid=8009 +controller = transport=tcp;traddr=2001:db8::370:7334;host-iface=enp0s8 +controller = transport=fc;traddr=nn-0x204600a098cbcac6:pn-0x204700a098cbcac6 + </programlisting> + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><varname>exclude=</varname></term> + + <listitem id='exclude-text'> + <para> + Controllers that should be excluded can be specified with the + <varname>exclude=</varname> option. Using mDNS to + automatically discover and connect to controllers, can result + in unintentional connections being made. This keyword allows + configuring the controllers that should not be connected to. + </para> + + <para> + The syntax is the same as for "controller", except that only + <parameter>transport</parameter>, <parameter>traddr</parameter>, + <parameter>trsvcid</parameter>, <parameter>nqn</parameter>, and + <parameter>host-iface</parameter> apply. Multiple + <varname>exclude=</varname> keywords may appear in the config + file to specify more than 1 excluded controller. + </para> + + <para> + Note 1: A minimal match approach is used to eliminate unwanted + controllers. That is, you do not need to specify all the + parameters to identify a controller. Just specifying the + <parameter>host-iface</parameter>, for example, can be used to + exclude all controllers on an interface. + </para> + + <para> + Note 2: <varname>exclude=</varname> takes precedence over + <varname>controller</varname>. A controller specified by the + <varname>controller</varname> keyword, can be eliminated by + the <varname>exclude=</varname> keyword. + </para> + + <para> + Examples: + <programlisting>exclude = transport=tcp;traddr=fe80::2c6e:dee7:857:26bb # Eliminate a specific address +exclude = host-iface=enp0s8 # Eliminate everything on this interface + </programlisting> + </para> + </listitem> + </varlistentry> + + </refsect2> +</root> diff --git a/doc/standard-options.xml b/doc/standard-options.xml new file mode 100644 index 0000000..8ae3f8b --- /dev/null +++ b/doc/standard-options.xml @@ -0,0 +1,163 @@ +<?xml version="1.0"?> +<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" + "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd"> +<!-- + SPDX-License-Identifier: Apache-2.0 + Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +--> + +<variablelist> + <varlistentry id='help'> + <term><option>-h</option></term> + <term><option>--help</option></term> + + <listitem id='help-text'> + <para>Print the help text and exit. + </para></listitem> + </varlistentry> + + <varlistentry id='version'> + <term><option>--version</option></term> + + <listitem id='version-text'> + <para>Print the version string and exit.</para> + </listitem> + </varlistentry> + + <varlistentry id='tron'> + <term><command>tron</command></term> + <listitem> + <para>Trace ON. Enable code tracing, which is to say that lots of + debug information will be printed to the syslog + (e.g. systemd-journal).</para> + </listitem> + </varlistentry> + + <varlistentry id='troff'> + <term><command>troff</command></term> + <listitem> + <para>Trace OFF. Disable code tracing.</para> + </listitem> + </varlistentry> + + <varlistentry id='status'> + <term><command>status</command></term> + <listitem> + <para>Show runtime status information.</para> + </listitem> + </varlistentry> + + <varlistentry id='detailed'> + <term><option>-d</option></term> + <term><option>--detailed</option></term> + <listitem> + <para>Print additional details.</para> + </listitem> + </varlistentry> + + <varlistentry id='transport'> + <term><option>-tTRTYPE</option></term> + <term><option>--transport=TRTYPE</option></term> + + <listitem> + <para>NVMe-over-Fabrics fabric type (default: "tcp").</para> + </listitem> + </varlistentry> + + <varlistentry id='traddr'> + <term><option>-aTRADDR</option></term> + <term><option>--traddr=TRADDR</option></term> + <listitem> + <para>Discovery controller's network address.</para> + </listitem> + </varlistentry> + + <varlistentry id='trsvcid'> + <term><option>-sTRSVCID</option></term> + <term><option>--trsvcid=TRSVCID</option></term> + <listitem> + <para> + Transport service id (for IP addressing, e.g. tcp, rdma, + this field is the port number). + </para> + </listitem> + </varlistentry> + + <varlistentry id='host-traddr'> + <term><option>-wTRADDR</option></term> + <term><option>--host-traddr=TRADDR</option></term> + <listitem> + <para> + Network source address used on the host to connect to + the controller. + </para> + </listitem> + </varlistentry> + + <varlistentry id='host-iface'> + <term><option>-fIFACE</option></term> + <term><option>--host-iface=IFACE</option></term> + <listitem> + <para> + This field specifies the network interface used on the + host to connect to the controller. + </para> + </listitem> + </varlistentry> + + <varlistentry id='nqn'> + <term><option>-nNQN</option></term> + <term><option>--nqn=NQN</option></term> + <listitem> + <para> + This field specifies the Controller's NVMe Qualified Name. + </para> + <para> + This field is mandatory for I/O Controllers, but is optional for + Discovery Controllers (DC). For the latter, the NQN will default + to the well-known DC NQN: <literal>nqn.2014-08.org.nvmexpress.discovery</literal> + if left undefined. + </para> + </listitem> + </varlistentry> + + <varlistentry id='TRTYPE-value'> + <term><replaceable>TRTYPE</replaceable></term> + <listitem> + <para>rdma, fc, tcp, loop.</para> + </listitem> + </varlistentry> + + <varlistentry id='TRADDR-value'> + <term><replaceable>TRADDR</replaceable></term> + <listitem> + <para>IP or Fibre Channel address. E.g. 10.10.0.100.</para> + </listitem> + </varlistentry> + + <varlistentry id='TRSVCID-value'> + <term><replaceable>TRSVCID</replaceable></term> + <listitem> + <para>E.g., 8009.</para> + </listitem> + </varlistentry> + + <varlistentry id='IFACE-value'> + <term><replaceable>IFACE</replaceable></term> + <listitem> + <para> + Network interface name. E.g., eth1, enp0s8, wlp0s20f3. + </para> + </listitem> + </varlistentry> + + <varlistentry id='NQN-value'> + <term><replaceable>NQN</replaceable></term> + <listitem> + <para> + NVMe Qualified Name. + </para> + </listitem> + </varlistentry> + +</variablelist> diff --git a/doc/stas-config.target.xml b/doc/stas-config.target.xml new file mode 100644 index 0000000..1e81845 --- /dev/null +++ b/doc/stas-config.target.xml @@ -0,0 +1,73 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!--*-nxml-*--> +<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" +"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"> +<!-- + SPDX-License-Identifier: Apache-2.0 + Copyright (c) 2022, Dell Inc. or its subsidiaries. All rights reserved. +--> +<refentry id="stas-config.target"> + <refentryinfo> + <title>stas-config.target</title> + <productname>nvme-stas</productname> + <author> + <personname> + <honorific>Mr</honorific> + <firstname>Martin</firstname> + <surname>Belanger</surname> + </personname> + <affiliation> + <orgname>Dell, Inc.</orgname> + </affiliation> + </author> + </refentryinfo> + + <refmeta> + <refentrytitle>stas-config.target</refentrytitle> + <manvolnum>8</manvolnum> + </refmeta> + + <refnamediv> + <refname>stas-config.target</refname> + <refpurpose>Used to synchronize the start of nvme-stas processes</refpurpose> + </refnamediv> + + <refsynopsisdiv> + <para> + <filename>/usr/lib/systemd/system/stas-config.target</filename> + </para> + </refsynopsisdiv> + + <refsect1> + <title>Description</title> + <para> + This target is used as a synchronization point before starting + <citerefentry><refentrytitle>stacd.service</refentrytitle><manvolnum>8</manvolnum></citerefentry> and + <citerefentry><refentrytitle>stafd.service</refentrytitle><manvolnum>8</manvolnum></citerefentry>. + </para> + + <para> + It ensures that <filename>/etc/nvme/hostnqn</filename> and + <filename>/etc/nvme/hostid</filename> are present before starting + <citerefentry><refentrytitle>stacd.service</refentrytitle><manvolnum>8</manvolnum></citerefentry> and + <citerefentry><refentrytitle>stafd.service</refentrytitle><manvolnum>8</manvolnum></citerefentry>. + </para> + </refsect1> + + <refsect1> + <title>See Also</title> + + <para> + <citerefentry> + <refentrytitle>stacd</refentrytitle> + <manvolnum>8</manvolnum> + </citerefentry> + <citerefentry> + <refentrytitle>stafd</refentrytitle> + <manvolnum>8</manvolnum> + </citerefentry> + </para> + </refsect1> +</refentry> + + diff --git a/doc/stas-config@.service.xml b/doc/stas-config@.service.xml new file mode 100644 index 0000000..8d05d44 --- /dev/null +++ b/doc/stas-config@.service.xml @@ -0,0 +1,67 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!--*-nxml-*--> +<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" +"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"> +<!-- + SPDX-License-Identifier: Apache-2.0 + Copyright (c) 2022, Dell Inc. or its subsidiaries. All rights reserved. +--> +<refentry id="stas-config@.service"> + <refentryinfo> + <title>stas-config@.service</title> + <productname>nvme-stas</productname> + <author> + <personname> + <honorific>Mr</honorific> + <firstname>Martin</firstname> + <surname>Belanger</surname> + </personname> + <affiliation> + <orgname>Dell, Inc.</orgname> + </affiliation> + </author> + </refentryinfo> + + <refmeta> + <refentrytitle>stas-config@.service</refentrytitle> + <manvolnum>8</manvolnum> + </refmeta> + + <refnamediv> + <refname>stas-config@.service</refname> + <refpurpose>Used for auto-generation of nvme-stas configuration files.</refpurpose> + </refnamediv> + + <refsynopsisdiv> + <para> + <filename>/usr/lib/systemd/system/stas-config@.service</filename> + </para> + </refsynopsisdiv> + + <refsect1> + <title>Description</title> + <para> + This service is used for the automatic run-time generation of + NVMe configuration located in <filename>/etc/nvme</filename> + (e.g. <filename>/etc/nvme/hostnqn</filename>). This is needed by + <citerefentry><refentrytitle>stacd.service</refentrytitle><manvolnum>8</manvolnum></citerefentry> and + <citerefentry><refentrytitle>stafd.service</refentrytitle><manvolnum>8</manvolnum></citerefentry>. + </para> + </refsect1> + + <refsect1> + <title>See Also</title> + + <para> + <citerefentry> + <refentrytitle>stacd</refentrytitle> + <manvolnum>8</manvolnum> + </citerefentry> + <citerefentry> + <refentrytitle>stafd</refentrytitle> + <manvolnum>8</manvolnum> + </citerefentry> + </para> + </refsect1> +</refentry> + diff --git a/doc/stasadm.xml b/doc/stasadm.xml new file mode 100644 index 0000000..576328c --- /dev/null +++ b/doc/stasadm.xml @@ -0,0 +1,201 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!--*-nxml-*--> +<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" +"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"> +<!-- + SPDX-License-Identifier: Apache-2.0 + Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +--> +<refentry id="stasadm" xmlns:xi="http://www.w3.org/2001/XInclude"> + <refentryinfo> + <title>stasadm</title> + <productname>nvme-stas</productname> + <author> + <personname> + <honorific>Mr</honorific> + <firstname>Martin</firstname> + <surname>Belanger</surname> + </personname> + <affiliation> + <orgname>Dell, Inc.</orgname> + </affiliation> + </author> + </refentryinfo> + + <refmeta> + <refentrytitle>stasadm</refentrytitle> + <manvolnum>1</manvolnum> + </refmeta> + + <refnamediv> + <refname>stasadm</refname> + <refpurpose>STorage Appliance Services admin functions</refpurpose> + </refnamediv> + + <refsynopsisdiv> + <cmdsynopsis> + <command>stasadm</command> + <arg choice="opt" rep="repeat">OPTIONS</arg> + <arg choice="req">COMMAND</arg> + <arg choice="opt" rep="repeat">OPTIONS</arg> + </cmdsynopsis> + </refsynopsisdiv> + + <refsect1> + <title>Description</title> + + <para> + <command>stasadm</command> is used to configure <code>nvme-stas</code>. The + configuration is saved to <filename>/etc/stas/sys.conf</filename>. + </para> + + <para> + Although <code>nvme-stas</code>' configuration is saved to + <filename>/etc/stas/sys.conf</filename>, it's still possible to + interoperate with the configuration of <command>nvme-cli</command> + and <command>libnvme</command>. <code>nvme-stas</code> allows one to + save individual parameters such as the Host NQN and ID outside of + <filename>/etc/stas/sys.conf</filename>. This allows, for example, + using the same default Host NQN and ID defined by + <command>nvme-cli</command> and <command>libnvme</command> in + <filename>/etc/nvme/hostnqn</filename> and <filename>/etc/nvme/hostid</filename> + respectively. To tell <code>nvme-stas</code> that you want to use the + those files, simply use <command>stasadm</command>'s + <option>--file=FILE</option> option. + </para> + </refsect1> + + <refsect1> + <title>Commands</title> + <para>The following commands are understood:</para> + + <variablelist> + <varlistentry> + <term><command>hostnqn</command></term> + <listitem> + <para> + Generate the Host NQN. This is typically used as a post + installation step to generate <filename>/etc/nvme/hostnqn</filename>. + </para> + <para> + The NVMe base specifications says: <quote>An NQN is + permanent for the lifetime of the host</quote>. For + this reason, the host NQN should only be generated + if <filename>/etc/nvme/hostnqn</filename> does not exist + already. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><command>hostid</command></term> + <listitem> + <para> + Generate the Host ID. This is typically used as a post + installation step to generate <filename>/etc/nvme/hostid</filename>. + </para> + <para> + Although not explicitly specified in the NVMe + specifications, the Host ID, like the Host NQN, should + be permanent for the lifetime of the host. Only generate + the Host ID if <filename>/etc/nvme/hostid</filename> + does not exist. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><command>set-symname [SYMNAME]</command></term> + <listitem> + <para> + Set the host symbolic name. + </para> + + <para> + The symbolic name is an optional parameter that can be + used for explicit registration with a discovery controller. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><command>clear-symname</command></term> + <listitem> + <para> + Clear the host symbolic name. + </para> + </listitem> + </varlistentry> + </variablelist> + </refsect1> + + <refsect1> + <title>Options</title> + + <para>The following options are understood:</para> + + <variablelist> + <xi:include href="standard-options.xml" xpointer="help"/> + <xi:include href="standard-options.xml" xpointer="version"/> + + <varlistentry> + <term><option>-fFILE</option></term> + <term><option>--file=FILE</option></term> + <listitem> + <para> + By default, <command>hostnqn</command> and <command>hostid</command> + save the values to <filename>/etc/stas/sys.conf</filename>. + This option allows saving to a separate file. + </para> + <para> + Traditionally, <command>nvme-cli</command> and + <command>libnvme</command> retrieve the default Host NQN + and ID from <filename>/etc/nvme/hostnqn</filename> and + <filename>/etc/nvme/hostid</filename> respectively. The + <option>--file=FILE</option> option can be + used to tell <code>nvme-stas</code> that it should + use those same configuration files. + </para> + </listitem> + </varlistentry> + </variablelist> + </refsect1> + + <refsect1> + <title>Exit status</title> + <para> + On success, 0 is returned; otherwise, a non-zero failure code is + returned. + </para> + </refsect1> + + <refsect1> + <title>Examples</title> + + <example> + <title>Generate <filename>/etc/nvme/hostnqn</filename></title> + <programlisting>$ stasadm hostnqn --file /etc/nvme/hostnqn</programlisting> + </example> + + <example> + <title>Generate <filename>/etc/nvme/hostid</filename></title> + <programlisting>$ stasadm hostid -f /etc/nvme/hostid</programlisting> + </example> + + <example> + <title>Configure the host's symbolic name</title> + <programlisting>$ stasadm set-symname LukeSkywalker</programlisting> + </example> + </refsect1> + + <refsect1> + <title>See Also</title> + + <para> + <citerefentry> + <refentrytitle>nvme-stas</refentrytitle> + <manvolnum>7</manvolnum> + </citerefentry> + </para> + </refsect1> +</refentry> diff --git a/doc/sys.conf.xml b/doc/sys.conf.xml new file mode 100644 index 0000000..fc6838a --- /dev/null +++ b/doc/sys.conf.xml @@ -0,0 +1,141 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!--*-nxml-*--> +<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" +"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"> +<!-- + SPDX-License-Identifier: Apache-2.0 + Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +--> +<refentry id="sys.conf" xmlns:xi="http://www.w3.org/2001/XInclude"> + <refentryinfo> + <title>sys.conf</title> + <productname>nvme-stas</productname> + + <author> + <personname> + <honorific>Mr</honorific> + <firstname>Martin</firstname> + <surname>Belanger</surname> + </personname> + + <affiliation> + <orgname>Dell, Inc.</orgname> + </affiliation> + </author> + </refentryinfo> + + <refmeta> + <refentrytitle>sys.conf</refentrytitle> + <manvolnum>5</manvolnum> + </refmeta> + + <refnamediv> + <refname>sys.conf</refname> + + <refpurpose> + <citerefentry project="man-pages"> + <refentrytitle>nvme-stas</refentrytitle> + <manvolnum>7</manvolnum> + </citerefentry> + configuration file + </refpurpose> + </refnamediv> + + <refsynopsisdiv> + <para> + <filename>/etc/stas/sys.conf</filename> + </para> + </refsynopsisdiv> + + <refsect1> + <title>Description</title> + + <para> + When <citerefentry project="man-pages"><refentrytitle>stafd</refentrytitle> + <manvolnum>8</manvolnum></citerefentry> and + <citerefentry project="man-pages"><refentrytitle>stacd</refentrytitle> + <manvolnum>8</manvolnum></citerefentry> start up, they read the + system configuration from <filename>sys.conf</filename>. + </para> + </refsect1> + + <refsect1> + <title>Configuration File Format</title> + <para> + <filename>sys.conf</filename> is a plain text file divided into + sections, with configuration entries in the style + <replaceable>key</replaceable>=<replaceable>value</replaceable>. + Whitespace immediately before or after the <literal>=</literal> is + ignored. Empty lines and lines starting with <literal>#</literal> + are ignored, which may be used for commenting. + </para> + </refsect1> + + <refsect1> + <title>Options</title> + + <refsect2> + <title>[Host] section</title> + <para> + The following options are available in the + <literal>[Host]</literal> section: + </para> + + <variablelist> + <varlistentry> + <term><varname>nqn=</varname></term> + <listitem> + <para> + Takes a string argument identifying the Host NQN. + A value starting with <code>file://</code> + indicates that the Host NQN can be retrieved from + a separate file. This is a mandatory parameter. + Defaults to: <literal>file:///etc/nvme/hostnqn</literal>. + </para> + </listitem> + </varlistentry> + </variablelist> + + <variablelist> + <varlistentry> + <term><varname>id=</varname></term> + <listitem> + <para> + Takes a string argument identifying the Host ID. + A value starting with <code>file://</code> + indicates that the Host ID can be retrieved from + a separate file. This is a mandatory parameter. + Defaults to: <literal>file:///etc/nvme/hostid</literal>. + </para> + </listitem> + </varlistentry> + </variablelist> + + <variablelist> + <varlistentry> + <term><varname>symname=</varname></term> + <listitem> + <para> + Takes a string argument identifying the Host symbolic name. + A value starting with <code>file://</code> + indicates that the symbolic name can be retrieved from + a separate file. This is an optional parameter. + There is no default value. + </para> + </listitem> + </varlistentry> + </variablelist> + </refsect2> + + </refsect1> + + <refsect1> + <title>See Also</title> + <para> + <citerefentry> + <refentrytitle>nvme-stas</refentrytitle> + <manvolnum>7</manvolnum> + </citerefentry> + </para> + </refsect1> +</refentry> diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..4d5b384 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,27 @@ +version: '3.7' + +x-stas: &default-stas + image: ghcr.io/linux-nvme/nvme-stas:main + build: + context: . + volumes: + - /run/dbus:/run/dbus + - /etc/nvme:/etc/nvme + privileged: true + network_mode: host + +services: + stafd: + <<: *default-stas + environment: + RUNTIME_DIRECTORY: /run/stafd + XDG_CACHE_HOME: /var/cache/stafd + PYTHONUNBUFFERED: 1 + command: -u /usr/sbin/stafd + stacd: + <<: *default-stas + environment: + RUNTIME_DIRECTORY: /run/stacd + XDG_CACHE_HOME: /var/cache/stacd + PYTHONUNBUFFERED: 1 + command: -u /usr/sbin/stacd diff --git a/etc/dbus-1/system.d/meson.build b/etc/dbus-1/system.d/meson.build new file mode 100644 index 0000000..b9bc858 --- /dev/null +++ b/etc/dbus-1/system.d/meson.build @@ -0,0 +1,23 @@ +# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). +# +# Authors: Martin Belanger <Martin.Belanger@dell.com> +# +dbus_conf_dir = datadir / 'dbus-1' / 'system.d' + +configure_file( + input: 'org.nvmexpress.staf.in.conf', + output: 'org.nvmexpress.staf.conf', + configuration: conf, + install_dir: dbus_conf_dir, +) + +configure_file( + input: 'org.nvmexpress.stac.in.conf', + output: 'org.nvmexpress.stac.conf', + configuration: conf, + install_dir: dbus_conf_dir, +) diff --git a/etc/dbus-1/system.d/org.nvmexpress.stac.in.conf b/etc/dbus-1/system.d/org.nvmexpress.stac.in.conf new file mode 100644 index 0000000..56b4228 --- /dev/null +++ b/etc/dbus-1/system.d/org.nvmexpress.stac.in.conf @@ -0,0 +1,37 @@ +<!-- + Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. + SPDX-License-Identifier: Apache-2.0 + See the LICENSE file for details. + + This file is part of NVMe STorage Appliance Services (nvme-stas). + + Authors: Martin Belanger <Martin.Belanger@dell.com> +--> + +<!DOCTYPE busconfig PUBLIC + "-//freedesktop//DTD D-BUS Bus Configuration 1.0//EN" + "http://www.freedesktop.org/standards/dbus/1.0/busconfig.dtd"> +<busconfig> + + <!-- Only allow root to own the bus. --> + <policy user="root"> + <allow own_prefix="@STACD_DBUS_NAME@"/> + </policy> + + <!-- Allow anyone to invoke most methods on the bus, but deny setting properties. --> + <policy context="default"> + <allow send_destination="@STACD_DBUS_NAME@"/> + + <deny send_destination="@STACD_DBUS_NAME@" + send_interface="org.freedesktop.DBus.Properties" + send_member="Set" + send_type="method_call"/> + </policy> + + <!-- Allow root to invoke everything on the bus. --> + <policy user="root"> + <allow send_destination="@STACD_DBUS_NAME@"/> + </policy> + +</busconfig> + diff --git a/etc/dbus-1/system.d/org.nvmexpress.staf.in.conf b/etc/dbus-1/system.d/org.nvmexpress.staf.in.conf new file mode 100644 index 0000000..a04877a --- /dev/null +++ b/etc/dbus-1/system.d/org.nvmexpress.staf.in.conf @@ -0,0 +1,37 @@ +<!-- + Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. + SPDX-License-Identifier: Apache-2.0 + See the LICENSE file for details. + + This file is part of NVMe STorage Appliance Services (nvme-stas). + + Authors: Martin Belanger <Martin.Belanger@dell.com> +--> + +<!DOCTYPE busconfig PUBLIC + "-//freedesktop//DTD D-BUS Bus Configuration 1.0//EN" + "http://www.freedesktop.org/standards/dbus/1.0/busconfig.dtd"> +<busconfig> + + <!-- Only allow root to own the bus. --> + <policy user="root"> + <allow own_prefix="@STAFD_DBUS_NAME@"/> + </policy> + + <!-- Allow anyone to invoke most methods on the bus, but deny setting properties. --> + <policy context="default"> + <allow send_destination="@STAFD_DBUS_NAME@"/> + + <deny send_destination="@STAFD_DBUS_NAME@" + send_interface="org.freedesktop.DBus.Properties" + send_member="Set" + send_type="method_call"/> + </policy> + + <!-- Allow root to invoke everything on the bus. --> + <policy user="root"> + <allow send_destination="@STAFD_DBUS_NAME@"/> + </policy> + +</busconfig> + diff --git a/etc/stas/stacd.conf b/etc/stas/stacd.conf new file mode 100644 index 0000000..d03f0b1 --- /dev/null +++ b/etc/stas/stacd.conf @@ -0,0 +1,322 @@ +# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). + +# ============================================================================== +# STorage Appliance Connector Daemon (stacd) - configuration file +# +# In this file, options that are commented represent the default values used. +# Uncommented options override the default value. + + +[Global] +# tron: Trace-ON. Enable additional debug info +# Type: boolean +# Range: [false, true] +#tron=false + +# hdr-digest: Protocol Data Unit (PDU) Header Digest. NVMe/TCP facilitates an +# optional PDU Header digest. Digests are calculated using the +# CRC32C algorithm. +# Type: boolean +# Range: [false, true] +#hdr-digest=false + +# data-digest: Protocol Data Unit (PDU) Data Digest. NVMe/TCP facilitates an +# optional PDU Data digest. Digests are calculated using the +# CRC32C algorithm. +# Type: boolean +# Range: [false, true] +#data-digest=false + +# kato: Keep Alive Timeout (KATO): This field specifies the timeout value +# for the Keep Alive feature in seconds. The default value for this +# field is 30 seconds (2 minutes). +# Type: Unsigned integer +# Range: 0..N +# Unit: Seconds +#kato=30 + +# nr-io-queues: Overrides the default number of I/O queues create by the +# driver. +# Type: Unsigned integer +# Range: 1..N +# Default: Depends on kernel and other run time +# factors (e.g. number of CPUs). + +# nr-write-queues: Adds additional queues that will be used for write I/O. +# Type: Unsigned integer +# Range: 1..N +# Default: Depends on kernel and other run time +# factors (e.g. number of CPUs). + +# nr-poll-queues: Adds additional queues that will be used for polling +# latency sensitive I/O. +# Type: Unsigned integer +# Range: 1..N +# Default: Depends on kernel and other run time +# factors (e.g. number of CPUs). + +# queue-size: Overrides the default number of elements in the I/O queues +# created by the driver. +# Type: Unsigned integer +# Range: 16..1024 +# Default: 128 +#queue-size=128 + +# reconnect-delay: Overrides the default delay, in seconds, before reconnect +# is attempted after a connect loss. +# Type: Unsigned integer +# Range: 1..N +# Unit: Seconds +# Default: 10 (retry to connect every 10 seconds) +#reconnect-delay=10 + +# ctrl-loss-tmo: Overrides the default controller loss timeout period in +# seconds. +# Type: Unsigned integer +# Range: -1, 0..N where -1 means retry forever +# Unit: Seconds +# Default: 600 (retry to connect for up to 10 minutes) +#ctrl-loss-tmo=600 + +# disable-sqflow: Disables SQ flow control to omit head doorbell update for +# submission queues when sending nvme completions. +# Type: boolean +# Range: [false, true] +# Default: false +#disable-sqflow=false + +# ignore-iface: This option controls whether connections with I/O Controllers +# (IOC) will be forced on a specific interface or will rely on +# the routing tables to determine the interface. +# +# See the man pages for details: man stacd.conf +# +# Type: boolean +# Range: [false, true] +# Default: true +#ignore-iface=false + +# ip-family: With this you can specify whether stacd will support IPv4, IPv6, +# or both when connecting to I/O Controllers (IOC). +# +# See the man pages for details: man stacd.conf +# +# Type: String +# Range: [ipv4, ipv6, ipv4+ipv6] +# Default: ipv4+ipv6 +#ip-family=ipv4+ipv6 + + +# ============================================================================== +[I/O controller connection management] +# This section contains parameters to manage I/O controller connections. +# For example, parameters are provided to control disconnect policy. In other +# words, whether stacd will disconnect from IOCs on DLPE removal and which +# connections will be affected. +# +# Also, what should stacd do when a DLPE NCC bit (Not Connected to CDC) is +# asserted. Should stacd stop trying to connect to an I/O controller after a +# certain number of unsuccessful attempts. +# +# See the man pages for details: man stacd.conf + +# disconnect-scope: Determines which connections, if any, will be the target of +# a potential disconnect on DLPE removal. +# +# Type: String +# Range: [only-stas-connections | all-connections-matching-disconnect-trtypes | no-disconnect] +# Default: only-stas-connections +#disconnect-scope=only-stas-connections + +# disconnect-trtypes: Specify which connections should be audited based on the +# transport type. This parameter only applies when +# "disconnect-scope = all-connections-matching-disconnect-trtypes". +# +# Type: String +# Range: [tcp, rdma, fc, tcp+rdma, tcp+fc, rdma+fc, tcp+rdma+fc] +# Default: tcp +#disconnect-trtypes=tcp + +# connect-attempts-on-ncc: The NCC bit (Not Connected to CDC) returned in a +# DLPE indicates whether a connection is currently +# established between the CDC and the subsystem listed +# in the DLPE. +# +# When the NCC bit is asserted, it may mean that the +# subsystem is offline or that fabrics connectivity is +# momentarily lost. If the host is also unable to +# connect to the subsystem, then there is no point in +# continuing to try to connect. In fact, the CDC will +# monitor the situation an inform the host (AEN) when +# connectivity is restored. +# +# This field is used to tell stacd how to behave when +# the NCC bit is asserted. How many times should it try +# to connect before give-up, or whether to keep trying +# indefinitely. +# +# Type: Integer +# Range: [0, 2..N], 0 means "Never stop trying". A +# non-0 value indicates the number of attempts +# before giving up. This value should never be +# set to 1. A value of 1 will automatically be +# increased to 2. That's because a single +# failure may be normal and a mimimum of 2 +# attempts is required to conclude that a +# connection is not possible. +# Default: 0 +#connect-attempts-on-ncc=0 + +# ============================================================================== +[Controllers] +# controller: I/O Controllers (IOC) are specified with this keyword. +# +# Syntax: +# controller = transport=<trtype>;traddr=<traddr>;trsvcid=<trsvcid>;host-traddr=<traddr>;host-iface=<iface>,nqn=<subnqn> +# +# transport=<trtype> [MANDATORY] +# This field specifies the network fabric being used for a NVMe-over- +# Fabrics network. Current string values include: +# +# Value Definition +# ------- ----------------------------------------------------------- +# rdma The network fabric is an rdma network (RoCE, iWARP, Infiniband, basic rdma, etc) +# fc The network fabric is a Fibre Channel network. +# tcp The network fabric is a TCP/IP network. +# loop Connect to a NVMe over Fabrics target on the local host +# +# traddr=<traddr> [MANDATORY] +# This field specifies the network address of the Controller. For +# transports using IP addressing (e.g. rdma) this should be an IP- +# based address (ex. IPv4, IPv6). It could also be a resolvable host +# name (e.g. localhost). +# +# nqn=<subnqn> [MANDATORY] +# This field specifies the Subsystem's NVMe Qualified Name. +# +# trsvcid=<trsvcid> [OPTIONAL] +# This field specifies the transport service id. For transports using +# IP addressing (e.g. rdma) this field is the port number. +# +# Depending on the transport type, this field will default to either +# 8009 or 4420 as follows. +# +# UDP port 4420 and TCP port 4420 have been assigned by IANA +# for use by NVMe over Fabrics. NVMe/RoCEv2 controllers use UDP port +# 4420 by default. NVMe/iWARP controllers use TCP port 4420 by +# default. +# +# TCP port 4420 has been assigned for use by NVMe over Fabrics and TCP +# port 8009 has been assigned by IANA for use by NVMe over Fabrics +# discovery. TCP port 8009 is the default TCP port for NVMe/TCP +# discovery controllers. There is no default TCP port for NVMe/TCP I/O +# controllers, the Transport Service Identifier (TRSVCID) field in the +# Discovery Log Entry indicates the TCP port to use. +# +# The TCP ports that may be used for NVMe/TCP I/O controllers include +# TCP port 4420, and the Dynamic and/or Private TCP ports (i.e., ports +# in the TCP port number range from 49152 to 65535). NVMe/TCP I/O +# controllers should not use TCP port 8009. TCP port 4420 shall not be +# used for both NVMe/iWARP and NVMe/TCP at the same IP address on the +# same network. +# +# host-traddr=<traddr> [OPTIONAL] +# This field specifies the network address used on the host to connect +# to the Controller. For TCP, this sets the source address on the +# socket. +# +# host-iface=<iface> [OPTIONAL] +# This field specifies the network interface used on the host to +# connect to the Controller (e.g. IP eth1, enp2s0, enx78e7d1ea46da). +# This forces the connection to be made on a specific interface +# instead of letting the system decide. +# +# dhchap-ctrl-secret [OPTIONAL] +# NVMe In-band authentication controller secret (i.e. key) for +# bi-directional authentication; needs to be in ASCII format as +# specified in NVMe 2.0 section 8.13.5.8 'Secret representation'. +# Bi-directional authentication will be attempted when present. +# +# hdr-digest [OPTIONAL] +# See definition in [Global] section. This is used to override +# the value specified in the [Global] section. +# +# data-digest [OPTIONAL] +# See definition in [Global] section. This is used to override +# the value specified in the [Global] section. +# +# nr-io-queues [OPTIONAL] +# See definition in [Global] section. This is used to override +# the value specified in the [Global] section. +# +# nr-write-queues [OPTIONAL] +# See definition in [Global] section. This is used to override +# the value specified in the [Global] section. +# +# nr-poll-queues [OPTIONAL] +# See definition in [Global] section. This is used to override +# the value specified in the [Global] section. +# +# queue-size [OPTIONAL] +# See definition in [Global] section. This is used to override +# the value specified in the [Global] section. +# +# kato [OPTIONAL] +# See definition in [Global] section. This is used to override +# the value specified in the [Global] section. +# +# reconnect-delay [OPTIONAL] +# See definition in [Global] section. This is used to override +# the value specified in the [Global] section. +# +# ctrl-loss-tmo [OPTIONAL] +# See definition in [Global] section. This is used to override +# the value specified in the [Global] section. +# +# disable-sqflow [OPTIONAL] +# See definition in [Global] section. This is used to override +# the value specified in the [Global] section. +# +# Multiple DCs may be specified on separate lines like this (this is +# just an example and does not represent default values): +# +# controller = transport=tcp;traddr=localhost;nqn=nqn.1988-11.com.dell:PowerSANxxx:01:20210225100113-454f73093ceb4847a7bdfc6e34ae8e28 +# controller = transport=tcp;traddr=2001:db8::370:7334;host-iface=enp0s8;nqn=nqn.starship-enterprise +# controller = transport=fc;traddr=nn-0x204600a098cbcac6:pn-0x204700a098cbcac6;nqn=nqn.romulan-empire +# ... +# Type: String +# +# Default: There is no default controller. STAC will not try to +# connect to a default I/O Controller. +#controller= + + +# exclude: Excluded controllers. This keyword allows configuring I/O +# controllers that should not be connected to (whatever the +# reason may be). +# +# The syntax is the same as for "controller=", except that the key +# host-traddr does not apply. Multiple "exclude=" keywords may +# appear in the config file to define the exclusion list. +# +# Note 1: A minimal match approach is used to eliminate unwanted +# controllers. That is, you do not need to specify all the +# parameters to identify a controller. Just specifying the +# host-iface, for example, can be used to exclude all controllers +# on an interface. +# +# Note 2: "exclude=" takes precedence over "controller=". A +# controller specified by the "controller=" keyword, can be +# eliminated by the "exclude=" keyword. +# +# Syntax: Same as "controller=" above. +# Type: String +# +# Example: +# exclude = transport=tcp;traddr=fe80::2c6e:dee7:857:26bb # Eliminate a specific address +# exclude = host-iface=enp0s8 # Eliminate everything on this interface +#exclude= diff --git a/etc/stas/stafd.conf b/etc/stas/stafd.conf new file mode 100644 index 0000000..22b3f96 --- /dev/null +++ b/etc/stas/stafd.conf @@ -0,0 +1,277 @@ +# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). + +# ============================================================================== +# STorage Appliance Finder Daemon (stafd) - configuration file +# +# In this file, options that are commented represent the default values used. +# Uncommented options override the default value. + + +[Global] +# tron: Trace-ON. Enable additional debug info +# Type: boolean +# Range: [false, true] +#tron=false + +# hdr-digest: Protocol Data Unit (PDU) Header Digest. NVMe/TCP facilitates an +# optional PDU Header digest. Digests are calculated using the +# CRC32C algorithm. +# Type: boolean +# Range: [false, true] +#hdr-digest=false + +# data-digest: Protocol Data Unit (PDU) Data Digest. NVMe/TCP facilitates an +# optional PDU Data digest. Digests are calculated using the +# CRC32C algorithm. +# Type: boolean +# Range: [false, true] +#data-digest=false + +# kato: Keep Alive Timeout (KATO): This field specifies the timeout value +# for the Keep Alive feature in seconds. The default value for this +# field is 30 seconds. +# Type: Unsigned integer +# Range: 0..N +# Unit: Seconds +#kato=30 + +# queue-size: Overrides the default number of elements in the I/O queues +# created by the driver. +# Type: Unsigned integer +# Range: 16..1024 +# Default: 128 +#queue-size=128 + +# reconnect-delay: Overrides the default delay, in seconds, before reconnect +# is attempted after a connect loss. +# Type: Unsigned integer +# Range: 1..N +# Unit: Seconds +# Default: 10 (retry to connect every 10 seconds) +#reconnect-delay=10 + +# ctrl-loss-tmo: Overrides the default controller loss timeout period in +# seconds. +# Type: Unsigned integer +# Range: -1, 0..N where -1 means retry forever +# Unit: Seconds +# Default: 600 (retry to connect for up to 10 minutes) +#ctrl-loss-tmo=600 + +# disable-sqflow: Disables SQ flow control to omit head doorbell update for +# submission queues when sending nvme completions. +# Type: boolean +# Range: [false, true] +# Default: false +#disable-sqflow=false + +# ignore-iface: This option controls whether connections with Discovery +# Controllers (DC) will be forced on a specific interface or +# will rely on the routing tables to determine the interface. +# +# See the man pages for details: man stafd.conf +# +# Type: boolean +# Range: [false, true] +# Default: true +#ignore-iface=false + +# ip-family: With this you can specify whether stafd will support IPv4, IPv6, +# or both when connecting to Discovery Controllers (DC). +# +# See the man pages for details: man stafd.conf +# +# Type: String +# Range: [ipv4, ipv6, ipv4+ipv6] +# Default: ipv4+ipv6 +#ip-family=ipv4+ipv6 + +# pleo: Port Local Entries Only. If enabled and supported, when connected to a +# Direct Discovery Controller (DDC), stafd will ask the DDC to return +# records for only NVM subsystem ports that are presented through the same +# NVM subsystem port that received the Get Log Page command. When disabled +# or not supported by the DDC, the DDC may return NVM subsystems that are +# not even reachable by the host, including those using a transport +# different from the transport used for the Get Log Page command (e.g. Get +# Log Page using TCP and receiving FC subsystems). This configuration +# parameter has no effect if the DDC does not support PLEO (see PLEOS). +# +# Type: String +# Range: [disabled, enabled] +# Default: enabled +#pleo=enabled + +# ============================================================================== +[Service Discovery] +# zeroconf: Control whether DNS-SD/mDNS automatic discovery is enabled. This is +# used to enable or disable automatic discovery of Discovery +# Controllers using DNS-SD/mDNS. +# +# Type: String +# Range: [disabled, enabled] +# Default: enabled +#zeroconf=enabled + +# ============================================================================== +[Discovery controller connection management] +# persistent-connections: Whether connections to Discovery Controllers (DC) +# are persistent. If stafd is stopped, the connections +# will persists. When this is set to false, stafd will +# disconnect from all DCs it is connected to when stafd +# is stopped. +# Type: boolean +# Range: [false, true] +#persistent-connections=true + +# zeroconf-connections-persistence: DCs that are discovered with mDNS service +# discovery which are later lost (i.e. no mDNS +# and TCP connection fails), will be purged from +# the configuration after this amount of time. +# Type: Time specs. +# Unit: Takes a unit-less value in seconds, +# or a time span (TS) value such as +# "3 days 5 hours". +# Range: -1, 0, TS. +# With "-1" equal to "no timeout" and +# 0 equal to timeout immediately. +# Default: 72 hours (3 days) +#zeroconf-connections-persistence=72hours + +# ============================================================================== +[Controllers] +# controller: Discovery Controllers (DC) are specified with this keyword. +# +# Syntax: +# controller = transport=[trtype];traddr=[traddr];trsvcid=[trsvcid];host-traddr=[traddr];host-iface=[iface];nqn=<dcnqn> +# +# transport=<trtype> [MANDATORY] +# This field specifies the network fabric being used for a NVMe-over- +# Fabrics network. Current string values include: +# +# Value Definition +# ------- ----------------------------------------------------------- +# rdma The network fabric is an rdma network (RoCE, iWARP, +# Infiniband, basic rdma, etc) +# fc The network fabric is a Fibre Channel network. +# tcp The network fabric is a TCP/IP network. +# loop Connect to a NVMe over Fabrics target on the local host +# +# traddr=<traddr> [MANDATORY] +# This field specifies the network address of the Controller. For +# transports using IP addressing (e.g. rdma) this should be an IP- +# based address (ex. IPv4, IPv6). It could also be a resolvable host +# name (e.g. localhost). +# +# nqn=<dcnqn> [OPTIONAL] +# This field specifies the Discovery Controller's NVMe Qualified +# Name. If not specified, this will default to the well-known DC +# NQN: "nqn.2014-08.org.nvmexpress.discovery". +# +# trsvcid=<trsvcid> [OPTIONAL] +# This field specifies the transport service id. For transports using +# IP addressing (e.g. rdma) this field is the port number. +# +# Depending on the transport type, this field will default to either +# 8009 or 4420 as follows. +# +# UDP port 4420 and TCP port 4420 have been assigned by IANA +# for use by NVMe over Fabrics. NVMe/RoCEv2 controllers use UDP port +# 4420 by default. NVMe/iWARP controllers use TCP port 4420 by +# default. +# +# TCP port 4420 has been assigned for use by NVMe over Fabrics and TCP +# port 8009 has been assigned by IANA for use by NVMe over Fabrics +# discovery. TCP port 8009 is the default TCP port for NVMe/TCP +# discovery controllers. There is no default TCP port for NVMe/TCP I/O +# controllers, the Transport Service Identifier (TRSVCID) field in the +# Discovery Log Entry indicates the TCP port to use. +# +# The TCP ports that may be used for NVMe/TCP I/O controllers include +# TCP port 4420, and the Dynamic and/or Private TCP ports (i.e., ports +# in the TCP port number range from 49152 to 65535). NVMe/TCP I/O +# controllers should not use TCP port 8009. TCP port 4420 shall not be +# used for both NVMe/iWARP and NVMe/TCP at the same IP address on the +# same network. +# +# Ref: https://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.xhtml?search=nvme +# +# host-traddr=<traddr> [OPTIONAL] +# This field specifies the network address used on the host to connect +# to the Controller. For TCP, this sets the source address on the +# socket. +# +# host-iface=<iface> [OPTIONAL] +# This field specifies the network interface used on the host to +# connect to the Controller (e.g. IP eth1, enp2s0, enx78e7d1ea46da). +# This forces the connection to be made on a specific interface +# instead of letting the system decide. +# +# hdr-digest [OPTIONAL] +# See definition in [Global] section. This is used to override +# the value specified in the [Global] section. +# +# data-digest [OPTIONAL] +# See definition in [Global] section. This is used to override +# the value specified in the [Global] section. +# +# kato [OPTIONAL] +# See definition in [Global] section. This is used to override +# the value specified in the [Global] section. +# +# reconnect-delay [OPTIONAL] +# See definition in [Global] section. This is used to override +# the value specified in the [Global] section. +# +# ctrl-loss-tmo [OPTIONAL] +# See definition in [Global] section. This is used to override +# the value specified in the [Global] section. +# +# disable-sqflow [OPTIONAL] +# See definition in [Global] section. This is used to override +# the value specified in the [Global] section. +# +# Multiple DCs may be specified on separate lines like this (this is +# just an example and does not represent default values): +# +# controller = transport=tcp;traddr=localhost;trsvcid=8009 +# controller = transport=tcp;traddr=2001:db8::370:7334;host-iface=enp0s8 +# controller = transport=fc;traddr=nn-0x204600a098cbcac6:pn-0x204700a098cbcac6 +# ... +# +# Type: String +# +# Default: There is no default controller. STAF will not try to +# connect to a default Discovery Controller. +#controller= + + +# exclude: Excluded controllers. Using mDNS to automatically detect +# and connect controllers, can result in unintentional connections +# being made. This keyword allows configuring the controllers that +# should not be connected to (whatever the reason may be). +# +# The syntax is the same as for "controller=", except that the key +# host-traddr does not apply. Multiple "exclude=" keywords may +# appear in the config file to define the exclusion list. +# +# Note 1: A minimal match approach is used to eliminate unwanted +# controllers. That is, you do not need to specify all the +# parameters to identify a controller. Just specifying the +# host-iface, for example, can be used to exclude all controllers +# on an interface. +# +# Note 2: "exclude=" takes precedence over "controller=". A +# controller specified by the "controller=" keyword, can be +# eliminated by the "exclude=" keyword. +# +# Syntax: Same as "controller=" above. +# Type: String +# +# Example: +# exclude = transport=tcp;traddr=fe80::2c6e:dee7:857:26bb # Eliminate a specific address +# exclude = host-iface=enp0s8 # Eliminate everything on this interface +#exclude= diff --git a/etc/stas/sys.conf.doc b/etc/stas/sys.conf.doc new file mode 100644 index 0000000..5850028 --- /dev/null +++ b/etc/stas/sys.conf.doc @@ -0,0 +1,62 @@ +# Copyright (c) 2022, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). + +# ============================================================================== +# STorage Appliance Services (stas) - System configuration file +# +# In this file, options that are commented represent the default values used. +# Uncommented options override the default value. + + +[Host] +# nqn: The host's unique Non-Qualified Name. A value starting with "file://" +# indicates that the Host NQN can be retrieved from a separate file. +# Typically, nvme-cli saves the Host NQN in /etc/nvme/hostnqn. For +# compatibility with nvme-cli, nvme-stas defaults to looking for the +# existance of this file and will read the NQN from it. Otherwise, you +# can overwrite the default NQN by specifying its value here or +# specifying another file that contains the Host NQN to use. +# Type: string +# Default: file:///etc/nvme/hostnqn +#nqn=file:///etc/nvme/hostnqn + + +# id: The host's unique Identifier (ID). A value starting with "file://" +# indicates that the Host ID can be retrieved from a separate file. +# Typically, nvme-cli saves the Host ID in /etc/nvme/hostid. For +# compatibility with nvme-cli, nvme-stas defaults to looking for the +# existance of this file and will read the ID from it. Otherwise, you +# can overwrite the default ID by specifying its value here or +# specifying another file that contains the Host ID to use. +# Type: string +# Default: file:///etc/nvme/hostid +#id=file:///etc/nvme/hostid + + +# key: The host's DHCHAP key to be used for authentication. This is an +# optional parameter only required when authentication is needed. +# A value starting with "file://" indicates that the Host Key can +# be retrieved from a separate file. Typically, nvme-cli saves the +# Host Key in /etc/nvme/hostkey. For compatibility with nvme-cli, +# nvme-stas defaults to looking for the existance of this file and +# will read the Key from it. Otherwise, you can overwrite the default +# Key by specifying its value here or specifying another file that +# contains an alternate Host Key to use. +# Type: string +# Default: file:///etc/nvme/hostkey +#key=file:///etc/nvme/hostkey + + +# symname: The host's symbolic name. This can be a string or the name of a file +# containing the symbolic name. A value starting with "file://" +# indicates that the Symbolic Name can be retrieved from a separate +# file. +# Type: string +# Default: There is no default. The symbolic name is undefined by +# default. +#symname= + + diff --git a/meson.build b/meson.build new file mode 100644 index 0000000..b84e3d1 --- /dev/null +++ b/meson.build @@ -0,0 +1,182 @@ +# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). +# +# Authors: Martin Belanger <Martin.Belanger@dell.com> +# +project( + 'nvme-stas', + meson_version: '>= 0.53.0', + version: '2.2.1', + license: 'Apache-2.0', + default_options: [ + 'buildtype=release', + 'prefix=/usr', + 'sysconfdir=/etc', + ] +) + +fs = import('fs') + +#=============================================================================== +prefix = get_option('prefix') +datadir = prefix / get_option('datadir') +etcdir = prefix / get_option('sysconfdir') +bindir = prefix / get_option('bindir') +sbindir = prefix / get_option('sbindir') +mandir = prefix / get_option('mandir') +docdir = datadir / 'doc' / 'nvme-stas' +cnfdir = etcdir / 'stas' + +want_man = get_option('man') +want_html = get_option('html') +want_readthedocs = get_option('readthedocs') + +buildtime_modules = [] +if want_man or want_html or want_readthedocs + buildtime_modules += ['lxml'] +endif + +python3 = import('python').find_installation('python3', modules:buildtime_modules) +python_version = python3.language_version() +python_version_req = '>=3.6' +if not python_version.version_compare(python_version_req) + error('Python @0@ required. Found @1@ instead'.format(python_version_req, python_version)) +endif + +# Check if the runtime Python modules are present. These are not needed +# to build nvme-stas, but will be needed to run the tests. +missing_runtime_mods = false +py_modules_reqd = [ + ['libnvme', 'Install python3-libnvme (deb/rpm)'], + ['dasbus', 'Install python3-dasbus (deb/rpm) OR pip3 install dasbus'], + ['pyudev', 'Install python3-pyudev (deb/rpm)'], + ['systemd', 'Install python3-systemd (deb/rpm)'], + ['gi', 'Install python3-gi (deb) OR python3-gobject (rpm)'], +] +foreach p : py_modules_reqd + if run_command(python3, '-c', 'import @0@'.format(p[0]), check: false).returncode() != 0 + warning('Missing runtime module "@0@". @1@'.format(p[0], p[1])) + missing_runtime_mods = true + endif +endforeach + +if missing_runtime_mods and get_option('rt_pymods_reqd') + error('Please install missing runtime modules') +endif + + +#=============================================================================== +conf = configuration_data() + +conf.set('VERSION', meson.project_version()) +conf.set('LICENSE', meson.project_license()[0]) +conf.set('BUILD_DIR', meson.current_build_dir()) +conf.set('STAFD_DBUS_NAME', 'org.nvmexpress.staf') +conf.set('STAFD_DBUS_PATH', '/org/nvmexpress/staf') +conf.set('STACD_DBUS_NAME', 'org.nvmexpress.stac') +conf.set('STACD_DBUS_PATH', '/org/nvmexpress/stac') + +#=============================================================================== +stafd = configure_file( + input: 'stafd.py', + output: 'stafd', + install_dir: sbindir, + copy: true, +) +stacd = configure_file( + input: 'stacd.py', + output: 'stacd', + install_dir: sbindir, + copy: true, +) + +stafctl = configure_file( + input: 'stafctl.py', + output: 'stafctl', + install_dir: bindir, + copy: true, +) + +stacctl = configure_file( + input: 'stacctl.py', + output: 'stacctl', + install_dir: bindir, + copy: true, +) + +stasadm = configure_file( + input: 'stasadm.py', + output: 'stasadm', + install_dir: bindir, + copy: true, +) + +#=========================================================================== +install_subdir( + 'etc/stas', + install_dir: etcdir, +) + +#=========================================================================== +foreach component : [ 'nvme-stas.spec', '.coveragerc', 'coverage.sh', ] + configure_file( + input: component + '.in', + output: component, + configuration: conf, + ) +endforeach + +#=========================================================================== +# Make a list of modules to lint +modules_to_lint = [stafd, stafctl, stacd, stacctl, stasadm] + + +# Point Python Path to Current Build Dir. +# This is used by other meson.build files. +PYTHON_SEARCH_PATHS = [ + conf.get('BUILD_DIR'), + conf.get('BUILD_DIR') / 'subprojects' / 'libnvme', +] +PYTHONPATH = ':'.join(PYTHON_SEARCH_PATHS) + +#=========================================================================== +subdir('staslib') +subdir('etc/dbus-1/system.d') +subdir('usr/lib/systemd/system') +subdir('test') +subdir('doc') + + +#=========================================================================== +summary_dict = { + 'prefix ': prefix, + 'etcdir ': etcdir, + 'cnfdir ': cnfdir, + 'bindir ': bindir, + 'sbindir ': sbindir, + 'datadir ': datadir, + 'mandir ': mandir, + 'docdir ': docdir, + 'dbus_conf_dir ': dbus_conf_dir, + 'sd_unit_dir ': sd_unit_dir, + 'build location ': meson.current_build_dir(), + 'libnvme for tests ': libnvme_location, +} +summary(summary_dict, section: 'Directories') + +summary_dict = { + 'want_man ': want_man, + 'want_html ': want_html, + 'want_readthedocs ': want_readthedocs, +} +if meson.version().version_compare('>=0.57.0') # conf.keys() + foreach key : conf.keys() + if key not in ['BUILD_DIR', 'VERSION', 'LICENSE'] + summary_dict += { key + ' ': conf.get(key) } + endif + endforeach +endif +summary(summary_dict, section: 'Configuration', bool_yn: true) diff --git a/meson_options.txt b/meson_options.txt new file mode 100644 index 0000000..e5ae1c3 --- /dev/null +++ b/meson_options.txt @@ -0,0 +1,7 @@ +# -*- mode: meson -*- + +option('man', type: 'boolean', value: false, description: 'build and install man pages') +option('html', type: 'boolean', value: false, description: 'build and install html pages') +option('readthedocs', type: 'boolean', value: false, description: 'to be used by Read-The-Docs documentation builder') +option('libnvme-sel', type: 'combo', value: 'subproject', choices: ['subproject', 'pre-installed'], description: 'Select the libnvme to be used for testing. Either libnvme built as a "subproject", or libnvme already installed on the system.') +option('rt_pymods_reqd', type: 'boolean', value: false, description: 'Make sure all run-time python modules are installed') diff --git a/nvme-stas.spec.in b/nvme-stas.spec.in new file mode 100644 index 0000000..8b31fab --- /dev/null +++ b/nvme-stas.spec.in @@ -0,0 +1,100 @@ +Name: nvme-stas +Summary: NVMe STorage Appliance Services +Version: @VERSION@ +Release: 1%{?dist} +License: @LICENSE@ +URL: https://github.com/linux-nvme/nvme-stas + +BuildArch: noarch + +BuildRequires: meson +BuildRequires: glib2-devel +#BuildRequires: libnvme-devel +BuildRequires: libxslt +BuildRequires: docbook-style-xsl +#BuildRequires: systemd-devel +BuildRequires: systemd-rpm-macros + +BuildRequires: python3 +#BuildRequires: python3-devel +#BuildRequires: python3-pyflakes +#BuildRequires: python3-pylint +#BuildRequires: pylint + +#BuildRequires: python3-libnvme +#BuildRequires: python3-dasbus +#BuildRequires: python3-pyudev +#BuildRequires: python3-systemd +#BuildRequires: python3-gobject-devel +BuildRequires: python3-lxml + +Requires: avahi +Requires: python3-libnvme +Requires: python3-dasbus +Requires: python3-pyudev +Requires: python3-systemd +Requires: python3-gobject + +%description +nvme-stas is a Central Discovery Controller (CDC) client for Linux. It +handles Asynchronous Event Notifications (AEN), Automated NVMe subsystem +connection controls, Error handling and reporting, and Automatic (zeroconf) +and Manual configuration. nvme-stas is composed of two daemons: +stafd (STorage Appliance Finder) and stacd (STorage Appliance Connector). + +%prep +%autosetup -p1 -n %{name}-%{version} + +%build +%meson --wrap-mode=nodownload -Dman=true -Dhtml=true +%meson_build + +%install +%meson_install + +%check +%meson_test + +%define services stacd.service stafd.service + +%pre +%service_add_pre %services + +%post +%service_add_post %services + +%preun +%service_del_preun %services + +%postun +%service_del_postun %services + +%files +%license LICENSE +%doc README.md +%dir %{_sysconfdir}/stas +%config(noreplace) %{_sysconfdir}/stas/stacd.conf +%config(noreplace) %{_sysconfdir}/stas/stafd.conf +%{_sysconfdir}/stas/sys.conf.doc +%{_datadir}/dbus-1/system.d/org.nvmexpress.*.conf +%{_bindir}/stacctl +%{_bindir}/stafctl +%{_bindir}/stasadm +%{_sbindir}/stacd +%{_sbindir}/stafd +%{_unitdir}/stacd.service +%{_unitdir}/stafd.service +%{_unitdir}/stas-config.target +%{_unitdir}/stas-config@.service +%dir %{python3_sitelib}/staslib +%{python3_sitelib}/staslib/* +%doc %{_pkgdocdir} +%{_mandir}/man1/sta*.1* +%{_mandir}/man5/*.5* +%{_mandir}/man7/nvme*.7* +%{_mandir}/man8/sta*.8* + +%changelog +* Wed May 18 2022 Martin Belanger <martin.belanger@dell.com> - Release 1.1 +* Thu Mar 24 2022 Martin Belanger <martin.belanger@dell.com> - Release 1.0-rc4 +- diff --git a/stacctl.py b/stacctl.py new file mode 100755 index 0000000..29e3e9a --- /dev/null +++ b/stacctl.py @@ -0,0 +1,101 @@ +#!/usr/bin/python3 +# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). +# +# Authors: Martin Belanger <Martin.Belanger@dell.com> +# +''' STorage Appliance Connector Control Utility +''' +import sys +import json +import pprint +from argparse import ArgumentParser +import dasbus.error +from dasbus.connection import SystemMessageBus +from staslib import defs + + +def tron(args): # pylint: disable=unused-argument + '''@brief Trace ON''' + bus = SystemMessageBus() + iface = bus.get_proxy(defs.STACD_DBUS_NAME, defs.STACD_DBUS_PATH) + iface.tron = True # pylint: disable=assigning-non-slot + print(f'tron = {iface.tron}') # Read value back from stacd and print + + +def troff(args): # pylint: disable=unused-argument + '''@brief Trace OFF''' + bus = SystemMessageBus() + iface = bus.get_proxy(defs.STACD_DBUS_NAME, defs.STACD_DBUS_PATH) + iface.tron = False # pylint: disable=assigning-non-slot + print(f'tron = {iface.tron}') # Read value back from stacd and print + + +def _extract_cid(ctrl): + return ( + ctrl['transport'], + ctrl['traddr'], + ctrl['trsvcid'], + ctrl['host-traddr'], + ctrl['host-iface'], + ctrl['subsysnqn'], + ) + + +def status(args): # pylint: disable=unused-argument + '''@brief retrieve stacd's status information''' + bus = SystemMessageBus() + iface = bus.get_proxy(defs.STACD_DBUS_NAME, defs.STACD_DBUS_PATH) + info = json.loads(iface.process_info()) + info['controllers'] = iface.list_controllers(True) + for controller in info['controllers']: + transport, traddr, trsvcid, host_traddr, host_iface, subsysnqn = _extract_cid(controller) + controller.update( + json.loads(iface.controller_info(transport, traddr, trsvcid, host_traddr, host_iface, subsysnqn)) + ) + + print(pprint.pformat(info, width=120)) + + +def ls(args): + '''@brief list the I/O controller's that stacd is + connected (or trying to connect) to. + ''' + bus = SystemMessageBus() + iface = bus.get_proxy(defs.STACD_DBUS_NAME, defs.STACD_DBUS_PATH) + info = iface.list_controllers(args.detailed) + print(pprint.pformat(info, width=120)) + + +PARSER = ArgumentParser(description='STorage Appliance Connector (STAC)') +PARSER.add_argument('-v', '--version', action='store_true', help='Print version, then exit', default=False) + +SUBPARSER = PARSER.add_subparsers(title='Commands') + +PRSR = SUBPARSER.add_parser('tron', help='Trace ON') +PRSR.set_defaults(func=tron) + +PRSR = SUBPARSER.add_parser('troff', help='Trace OFF') +PRSR.set_defaults(func=troff) + +PRSR = SUBPARSER.add_parser('status', help='Show runtime status information about stacd') +PRSR.set_defaults(func=status) + +PRSR = SUBPARSER.add_parser('ls', help='List I/O controllers') +PRSR.add_argument( + '-d', '--detailed', action='store_true', help='Print detailed info (default: "%(default)s")', default=False +) +PRSR.set_defaults(func=ls) + +ARGS = PARSER.parse_args() +if ARGS.version: + print(f'nvme-stas {defs.VERSION}') + sys.exit(0) + +try: + ARGS.func(ARGS) +except dasbus.error.DBusError: + sys.exit('Unable to communicate with stacd over D-Bus. Is stacd running?') diff --git a/stacd.py b/stacd.py new file mode 100755 index 0000000..731b414 --- /dev/null +++ b/stacd.py @@ -0,0 +1,115 @@ +#!/usr/bin/python3 +# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). +# +# Authors: Martin Belanger <Martin.Belanger@dell.com> +# +''' STorage Appliance Connector Daemon +''' +import sys +from argparse import ArgumentParser +from staslib import defs + + +# ****************************************************************************** +def parse_args(conf_file: str): + '''Parse command line options''' + parser = ArgumentParser(description='STorage Appliance Connector (STAC). Must be root to run this program.') + parser.add_argument( + '-f', + '--conf-file', + action='store', + help='Configuration file (default: %(default)s)', + default=conf_file, + type=str, + metavar='FILE', + ) + parser.add_argument( + '-s', + '--syslog', + action='store_true', + help='Send messages to syslog instead of stdout. Use this when running %(prog)s as a daemon. (default: %(default)s)', + default=False, + ) + parser.add_argument('--tron', action='store_true', help='Trace ON. (default: %(default)s)', default=False) + parser.add_argument('-v', '--version', action='store_true', help='Print version, then exit', default=False) + return parser.parse_args() + + +ARGS = parse_args(defs.STACD_CONF_FILE) + +if ARGS.version: + print(f'nvme-stas {defs.VERSION}') + print(f'libnvme {defs.LIBNVME_VERSION}') + sys.exit(0) + + +# ****************************************************************************** +if __name__ == '__main__': + import json + import logging + from staslib import log, service, stas, udev # pylint: disable=ungrouped-imports + + # Before going any further, make sure the script is allowed to run. + stas.check_if_allowed_to_continue() + + class Dbus: + '''This is the DBus interface that external programs can use to + communicate with stacd. + ''' + + __dbus_xml__ = stas.load_idl('stacd.idl') + + @property + def tron(self): + '''@brief Get Trace ON property''' + return STAC.tron + + @tron.setter + def tron(self, value): # pylint: disable=no-self-use + '''@brief Set Trace ON property''' + STAC.tron = value + + @property + def log_level(self) -> str: + '''@brief Get Log Level property''' + return log.level() + + def process_info(self) -> str: + '''@brief Get status info (for debug) + @return A string representation of a json object. + ''' + info = { + 'tron': STAC.tron, + 'log-level': self.log_level, + } + info.update(STAC.info()) + return json.dumps(info) + + def controller_info( # pylint: disable=too-many-arguments,no-self-use + self, transport, traddr, trsvcid, host_traddr, host_iface, subsysnqn + ) -> str: + '''@brief D-Bus method used to return information about a controller''' + controller = STAC.get_controller(transport, traddr, trsvcid, host_traddr, host_iface, subsysnqn) + return json.dumps(controller.info()) if controller else '{}' + + def list_controllers(self, detailed) -> list: # pylint: disable=no-self-use + '''@brief Return the list of I/O controller IDs''' + return [ + controller.details() if detailed else controller.controller_id_dict() + for controller in STAC.get_controllers() + ] + + log.init(ARGS.syslog) + STAC = service.Stac(ARGS, Dbus()) + STAC.run() + + STAC = None + ARGS = None + + udev.shutdown() + + logging.shutdown() diff --git a/stafctl.py b/stafctl.py new file mode 100755 index 0000000..a05de3a --- /dev/null +++ b/stafctl.py @@ -0,0 +1,184 @@ +#!/usr/bin/python3 +# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). +# +# Authors: Martin Belanger <Martin.Belanger@dell.com> +# +''' STorage Appliance Finder Control Utility +''' +import sys +import json +import pprint +from argparse import ArgumentParser +import dasbus.error +from dasbus.connection import SystemMessageBus +from staslib import defs + + +def tron(args): # pylint: disable=unused-argument + '''@brief Trace ON''' + bus = SystemMessageBus() + iface = bus.get_proxy(defs.STAFD_DBUS_NAME, defs.STAFD_DBUS_PATH) + iface.tron = True # pylint: disable=assigning-non-slot + print(f'tron = {iface.tron}') # Read value back from stafd and print + + +def troff(args): # pylint: disable=unused-argument + '''@brief Trace OFF''' + bus = SystemMessageBus() + iface = bus.get_proxy(defs.STAFD_DBUS_NAME, defs.STAFD_DBUS_PATH) + iface.tron = False # pylint: disable=assigning-non-slot + print(f'tron = {iface.tron}') # Read value back from stafd and print + + +def _extract_cid(ctrl): + return ( + ctrl['transport'], + ctrl['traddr'], + ctrl['trsvcid'], + ctrl['host-traddr'], + ctrl['host-iface'], + ctrl['subsysnqn'], + ) + + +def status(args): # pylint: disable=unused-argument + '''@brief retrieve stafd's status information''' + bus = SystemMessageBus() + iface = bus.get_proxy(defs.STAFD_DBUS_NAME, defs.STAFD_DBUS_PATH) + info = json.loads(iface.process_info()) + info['controllers'] = iface.list_controllers(True) + for controller in info['controllers']: + transport, traddr, trsvcid, host_traddr, host_iface, subsysnqn = _extract_cid(controller) + controller['log_pages'] = iface.get_log_pages(transport, traddr, trsvcid, host_traddr, host_iface, subsysnqn) + controller.update( + json.loads(iface.controller_info(transport, traddr, trsvcid, host_traddr, host_iface, subsysnqn)) + ) + + print(pprint.pformat(info, width=120)) + + +def ls(args): + '''@brief list the discovery controller's that stafd is + connected (or trying to connect) to. + ''' + bus = SystemMessageBus() + iface = bus.get_proxy(defs.STAFD_DBUS_NAME, defs.STAFD_DBUS_PATH) + info = iface.list_controllers(args.detailed) + print(pprint.pformat(info, width=120)) + + +def dlp(args): + '''@brief retrieve a controller's discovery log pages from stafd''' + bus = SystemMessageBus() + iface = bus.get_proxy(defs.STAFD_DBUS_NAME, defs.STAFD_DBUS_PATH) + info = iface.get_log_pages(args.transport, args.traddr, args.trsvcid, args.host_traddr, args.host_iface, args.nqn) + print(pprint.pformat(info, width=120)) + + +def adlp(args): + '''@brief retrieve all of the controller's discovery log pages from stafd''' + bus = SystemMessageBus() + iface = bus.get_proxy(defs.STAFD_DBUS_NAME, defs.STAFD_DBUS_PATH) + info = json.loads(iface.get_all_log_pages(args.detailed)) + print(pprint.pformat(info, width=120)) + + +PARSER = ArgumentParser(description='STorage Appliance Finder (STAF)') +PARSER.add_argument('-v', '--version', action='store_true', help='Print version, then exit', default=False) + +SUBPARSER = PARSER.add_subparsers(title='Commands') + +PRSR = SUBPARSER.add_parser('tron', help='Trace ON') +PRSR.set_defaults(func=tron) + +PRSR = SUBPARSER.add_parser('troff', help='Trace OFF') +PRSR.set_defaults(func=troff) + +PRSR = SUBPARSER.add_parser('status', help='Show runtime status information about stafd') +PRSR.set_defaults(func=status) + +PRSR = SUBPARSER.add_parser('ls', help='List discovery controllers') +PRSR.add_argument( + '-d', + '--detailed', + action='store_true', + help='Print detailed info (default: "%(default)s")', + default=False, +) +PRSR.set_defaults(func=ls) + +PRSR = SUBPARSER.add_parser('dlp', help='Show discovery log pages') +PRSR.add_argument( + '-t', + '--transport', + metavar='<trtype>', + action='store', + help='NVMe-over-Fabrics fabric type (default: "%(default)s")', + choices=['tcp', 'rdma', 'fc', 'loop'], + default='tcp', +) +PRSR.add_argument( + '-a', + '--traddr', + metavar='<traddr>', + action='store', + help='Discovery Controller\'s network address', + required=True, +) +PRSR.add_argument( + '-s', + '--trsvcid', + metavar='<trsvcid>', + action='store', + help='Transport service id (for IP addressing, e.g. tcp, rdma, this field is the port number)', + required=True, +) +PRSR.add_argument( + '-w', + '--host-traddr', + metavar='<traddr>', + action='store', + help='Network address used on the host to connect to the Controller (default: "%(default)s")', + default='', +) +PRSR.add_argument( + '-f', + '--host-iface', + metavar='<iface>', + action='store', + help='This field specifies the network interface used on the host to connect to the Controller (default: "%(default)s")', + default='', +) +PRSR.add_argument( + '-n', + '--nqn', + metavar='<nqn>', + action='store', + help='This field specifies the discovery controller\'s NQN. When not specified this option defaults to "%(default)s"', + default=defs.WELL_KNOWN_DISC_NQN, +) +PRSR.set_defaults(func=dlp) + +PRSR = SUBPARSER.add_parser('adlp', help='Show all discovery log pages') +PRSR.add_argument( + '-d', + '--detailed', + action='store_true', + help='Print detailed info (default: "%(default)s")', + default=False, +) +PRSR.set_defaults(func=adlp) + +ARGS = PARSER.parse_args() +if ARGS.version: + print(f'nvme-stas {defs.VERSION}') + sys.exit(0) + +try: + ARGS.func(ARGS) +except dasbus.error.DBusError: + sys.exit('Unable to communicate with stafd over D-Bus. Is stafd running?') diff --git a/stafd.py b/stafd.py new file mode 100755 index 0000000..afa21f7 --- /dev/null +++ b/stafd.py @@ -0,0 +1,152 @@ +#!/usr/bin/python3 +# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). +# +# Authors: Martin Belanger <Martin.Belanger@dell.com> +# +''' STorage Appliance Finder Daemon +''' +import sys +from argparse import ArgumentParser +from staslib import defs + + +# ****************************************************************************** +def parse_args(conf_file: str): + '''Parse command line options''' + parser = ArgumentParser(description='STorage Appliance Finder (STAF). Must be root to run this program.') + parser.add_argument( + '-f', + '--conf-file', + action='store', + help='Configuration file (default: %(default)s)', + default=conf_file, + type=str, + metavar='FILE', + ) + parser.add_argument( + '-s', + '--syslog', + action='store_true', + help='Send messages to syslog instead of stdout. Use this when running %(prog)s as a daemon. (default: %(default)s)', + default=False, + ) + parser.add_argument('--tron', action='store_true', help='Trace ON. (default: %(default)s)', default=False) + parser.add_argument('-v', '--version', action='store_true', help='Print version, then exit', default=False) + return parser.parse_args() + + +ARGS = parse_args(defs.STAFD_CONF_FILE) + +if ARGS.version: + print(f'nvme-stas {defs.VERSION}') + print(f'libnvme {defs.LIBNVME_VERSION}') + sys.exit(0) + + +# ****************************************************************************** +if __name__ == '__main__': + import json + import logging + import dasbus.server.interface + from staslib import log, service, stas, udev # pylint: disable=ungrouped-imports + + # Before going any further, make sure the script is allowed to run. + stas.check_if_allowed_to_continue() + + class Dbus: + '''This is the DBus interface that external programs can use to + communicate with stafd. + ''' + + __dbus_xml__ = stas.load_idl('stafd.idl') + + @dasbus.server.interface.dbus_signal + def log_pages_changed( # pylint: disable=too-many-arguments + self, + transport: str, + traddr: str, + trsvcid: str, + host_traddr: str, + host_iface: str, + subsysnqn: str, + device: str, + ): + '''@brief Signal sent when log pages have changed.''' + + @dasbus.server.interface.dbus_signal + def dc_removed(self): + '''@brief Signal sent when log pages have changed.''' + + @property + def tron(self): + '''@brief Get Trace ON property''' + return STAF.tron + + @tron.setter + def tron(self, value): # pylint: disable=no-self-use + '''@brief Set Trace ON property''' + STAF.tron = value + + @property + def log_level(self) -> str: + '''@brief Get Log Level property''' + return log.level() + + def process_info(self) -> str: + '''@brief Get status info (for debug) + @return A string representation of a json object. + ''' + info = { + 'tron': STAF.tron, + 'log-level': self.log_level, + } + info.update(STAF.info()) + return json.dumps(info) + + def controller_info( # pylint: disable=no-self-use,too-many-arguments + self, transport, traddr, trsvcid, host_traddr, host_iface, subsysnqn + ) -> str: + '''@brief D-Bus method used to return information about a controller''' + controller = STAF.get_controller(transport, traddr, trsvcid, host_traddr, host_iface, subsysnqn) + return json.dumps(controller.info()) if controller else '{}' + + def get_log_pages( # pylint: disable=no-self-use,too-many-arguments + self, transport, traddr, trsvcid, host_traddr, host_iface, subsysnqn + ) -> list: + '''@brief D-Bus method used to retrieve the discovery log pages from one controller''' + controller = STAF.get_controller(transport, traddr, trsvcid, host_traddr, host_iface, subsysnqn) + return controller.log_pages() if controller else list() + + def get_all_log_pages(self, detailed) -> str: # pylint: disable=no-self-use + '''@brief D-Bus method used to retrieve the discovery log pages from all controllers''' + log_pages = list() + for controller in STAF.get_controllers(): + log_pages.append( + { + 'discovery-controller': controller.details() if detailed else controller.controller_id_dict(), + 'log-pages': controller.log_pages(), + } + ) + return json.dumps(log_pages) + + def list_controllers(self, detailed) -> list: # pylint: disable=no-self-use + '''@brief Return the list of discovery controller IDs''' + return [ + controller.details() if detailed else controller.controller_id_dict() + for controller in STAF.get_controllers() + ] + + log.init(ARGS.syslog) + STAF = service.Staf(ARGS, Dbus()) + STAF.run() + + STAF = None + ARGS = None + + udev.shutdown() + + logging.shutdown() diff --git a/stasadm.py b/stasadm.py new file mode 100755 index 0000000..294fdde --- /dev/null +++ b/stasadm.py @@ -0,0 +1,202 @@ +#!/usr/bin/python3 +# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). +# +# Authors: Martin Belanger <Martin.Belanger@dell.com> +# +''' STorage Appliance Services Admin Tool ''' +import os +import sys +import uuid +import configparser +from argparse import ArgumentParser +from staslib import defs + +try: + import hmac + import hashlib +except (ImportError, ModuleNotFoundError): + hmac = None + hashlib = None + + +def read_from_file(fname, size): # pylint: disable=missing-function-docstring + try: + with open(fname) as f: # pylint: disable=unspecified-encoding + data = f.read(size) + if len(data) == size: + return data + except FileNotFoundError: + pass + + return None + + +def get_machine_app_specific(app_id): + '''@brief Get a machine ID specific to an application. We use the + value retrieved from /etc/machine-id. The documentation states that + /etc/machine-id: + "should be considered "confidential", and must not be exposed in + untrusted environments, in particular on the network. If a stable + unique identifier that is tied to the machine is needed for some + application, the machine ID or any part of it must not be used + directly. Instead the machine ID should be hashed with a crypto- + graphic, keyed hash function, using a fixed, application-specific + key. That way the ID will be properly unique, and derived in a + constant way from the machine ID but there will be no way to + retrieve the original machine ID from the application-specific one" + + @note systemd's C function sd_id128_get_machine_app_specific() was the + inspiration for this code. + + @ref https://www.freedesktop.org/software/systemd/man/machine-id.html + ''' + if not hmac: + return None + + data = read_from_file('/etc/machine-id', 32) + if not data: + return None + + hmac_obj = hmac.new(app_id, uuid.UUID(data).bytes, hashlib.sha256) + id128_bytes = hmac_obj.digest()[0:16] + return str(uuid.UUID(bytes=id128_bytes, version=4)) + + +def get_uuid_from_system(): + '''@brief Try to find system UUID in the following order: + 1) /etc/machine-id + 2) /sys/class/dmi/id/product_uuid + 3) /proc/device-tree/ibm,partition-uuid + ''' + uuid_str = get_machine_app_specific(b'$nvmexpress.org$') + if uuid_str: + return uuid_str + + # The following files are only readable by root + if os.geteuid() != 0: + sys.exit('Permission denied. Root privileges required.') + + id128 = read_from_file('/sys/class/dmi/id/product_uuid', 36) + if id128: + # Swap little-endian to network order per + # DMTF SMBIOS 3.0 Section 7.2.1 System UUID. + swapped = ''.join([id128[x] for x in (6, 7, 4, 5, 2, 3, 0, 1, 8, 11, 12, 9, 10, 13, 16, 17, 14, 15)]) + return swapped + id128[18:] + + return read_from_file('/proc/device-tree/ibm,partition-uuid', 36) + + +def save(section, option, string, conf_file, fname): + '''@brief Save configuration + + @param section: section in @conf_file where @option will be added + @param option: option to be added under @section in @conf_file + @param string: Text to be saved to @fname + @param conf_file: Configuration file name + @param fname: Optional file where @string will be saved + ''' + if fname and string is not None: + with open(fname, 'w') as f: # pylint: disable=unspecified-encoding + print(string, file=f) + + if conf_file: + config = configparser.ConfigParser( + default_section=None, allow_no_value=True, delimiters=('='), interpolation=None, strict=False + ) + if os.path.isfile(conf_file): + config.read(conf_file) + + try: + config.add_section(section) + except configparser.DuplicateSectionError: + pass + + if fname: + string = 'file://' + fname + + if string is not None: + config.set(section, option, string) + else: + config.remove_option(section, option) + + with open(conf_file, 'w') as f: # pylint: disable=unspecified-encoding + config.write(f) + + +def hostnqn(args): + '''@brief Configure the host NQN''' + uuid_str = get_uuid_from_system() or str(uuid.uuid4()) + uuid_str = f'nqn.2014-08.org.nvmexpress:uuid:{uuid_str}' + save('Host', 'nqn', uuid_str, args.conf_file, args.file) + + +def hostid(args): + '''@brief Configure the host ID''' + save('Host', 'id', str(uuid.uuid4()), args.conf_file, args.file) + + +def set_symname(args): + '''@brief Define the host Symbolic Name''' + save('Host', 'symname', args.symname, args.conf_file, args.file) + + +def clr_symname(args): + '''@brief Undefine the host NQN''' + save('Host', 'symname', None, args.conf_file, None) + + +def get_parser(): # pylint: disable=missing-function-docstring + parser = ArgumentParser(description='Configuration utility for STAS.') + parser.add_argument('-v', '--version', action='store_true', help='Print version, then exit', default=False) + parser.add_argument( + '-c', + '--conf-file', + action='store', + help='Configuration file. Default %(default)s.', + default=defs.SYS_CONF_FILE, + type=str, + metavar='FILE', + ) + + subparser = parser.add_subparsers(title='Commands') + + prsr = subparser.add_parser('hostnqn', help='Configure the host NQN. The NQN is auto-generated.') + prsr.add_argument( + '-f', '--file', action='store', help='Optional file where to save the NQN.', type=str, metavar='FILE' + ) + prsr.set_defaults(cmd=hostnqn) + + prsr = subparser.add_parser('hostid', help='Configure the host ID. The ID is auto-generated.') + prsr.add_argument( + '-f', '--file', action='store', help='Optional file where to save the ID.', type=str, metavar='FILE' + ) + prsr.set_defaults(cmd=hostid) + + prsr = subparser.add_parser('set-symname', help='Set the host symbolic') + prsr.add_argument( + '-f', '--file', action='store', help='Optional file where to save the symbolic name.', type=str, metavar='FILE' + ) + prsr.add_argument('symname', action='store', help='Symbolic name', default=None, metavar='SYMNAME') + prsr.set_defaults(cmd=set_symname) + + prsr = subparser.add_parser('clear-symname', help='Clear the host symbolic') + prsr.set_defaults(cmd=clr_symname) + + return parser + + +PARSER = get_parser() +ARGS = PARSER.parse_args() +if ARGS.version: + print(f'nvme-stas {defs.VERSION}') + sys.exit(0) + +try: + ARGS.cmd(ARGS) +except AttributeError as ex: + print(str(ex)) + PARSER.print_usage() diff --git a/staslib/.gitignore b/staslib/.gitignore new file mode 100644 index 0000000..bee8a64 --- /dev/null +++ b/staslib/.gitignore @@ -0,0 +1 @@ +__pycache__ diff --git a/staslib/__init__.py b/staslib/__init__.py new file mode 100644 index 0000000..27673d1 --- /dev/null +++ b/staslib/__init__.py @@ -0,0 +1,11 @@ +# Copyright (c) 2022, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). +# +# Authors: Martin Belanger <Martin.Belanger@dell.com> +# +'''STorage Appliance Services''' + +__version__ = '@VERSION@' diff --git a/staslib/avahi.py b/staslib/avahi.py new file mode 100644 index 0000000..c8a3a0b --- /dev/null +++ b/staslib/avahi.py @@ -0,0 +1,456 @@ +# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). +# +# Authors: Martin Belanger <Martin.Belanger@dell.com> +# +''' Module that provides a way to retrieve discovered + services from the Avahi daemon over D-Bus. +''' +import socket +import typing +import logging +import functools +import dasbus.error +import dasbus.connection +import dasbus.client.proxy +import dasbus.client.observer +from gi.repository import GLib +from staslib import defs, conf, gutil + + +def _txt2dict(txt: list): + '''@param txt: A list of list of integers. The integers are the ASCII value + of printable text characters. + ''' + the_dict = dict() + for list_of_chars in txt: + try: + string = functools.reduce(lambda accumulator, c: accumulator + chr(c), list_of_chars, '') + key, val = string.split("=") + the_dict[key.lower()] = val + except Exception: # pylint: disable=broad-except + pass + + return the_dict + + +def _proto2trans(protocol): + '''Return the matching transport for the given protocol.''' + if protocol is None: + return None + + protocol = protocol.strip().lower() + if protocol == 'tcp': + return 'tcp' + + if protocol in ('roce', 'iwarp', 'rdma'): + return 'rdma' + + return None + + +# ****************************************************************************** +class Avahi: # pylint: disable=too-many-instance-attributes + '''@brief Avahi Server proxy. Set up the D-Bus connection to the Avahi + daemon and register to be notified when services of a certain + type (stype) are discovered or lost. + ''' + + DBUS_NAME = 'org.freedesktop.Avahi' + DBUS_INTERFACE_SERVICE_BROWSER = DBUS_NAME + '.ServiceBrowser' + DBUS_INTERFACE_SERVICE_RESOLVER = DBUS_NAME + '.ServiceResolver' + LOOKUP_USE_MULTICAST = 2 + + IF_UNSPEC = -1 + PROTO_INET = 0 + PROTO_INET6 = 1 + PROTO_UNSPEC = -1 + + LOOKUP_RESULT_LOCAL = 8 # This record/service resides on and was announced by the local host + LOOKUP_RESULT_CACHED = 1 # This response originates from the cache + LOOKUP_RESULT_STATIC = 32 # The returned data has been defined statically by some configuration option + LOOKUP_RESULT_OUR_OWN = 16 # This service belongs to the same local client as the browser object + LOOKUP_RESULT_WIDE_AREA = 2 # This response originates from wide area DNS + LOOKUP_RESULT_MULTICAST = 4 # This response originates from multicast DNS + + result_flags = { + LOOKUP_RESULT_LOCAL: 'local', + LOOKUP_RESULT_CACHED: 'cache', + LOOKUP_RESULT_STATIC: 'static', + LOOKUP_RESULT_OUR_OWN: 'own', + LOOKUP_RESULT_WIDE_AREA: 'wan', + LOOKUP_RESULT_MULTICAST: 'mcast', + } + + protos = {PROTO_INET: 'IPv4', PROTO_INET6: 'IPv6', PROTO_UNSPEC: 'uspecified'} + + @classmethod + def result_flags_as_string(cls, flags): + '''Convert flags to human-readable string''' + return '+'.join((value for flag, value in Avahi.result_flags.items() if (flags & flag) != 0)) + + @classmethod + def protocol_as_string(cls, proto): + '''Convert protocol codes to human-readable strings''' + return Avahi.protos.get(proto, 'unknown') + + # ========================================================================== + def __init__(self, sysbus, change_cb): + self._change_cb = change_cb + self._services = dict() + self._sysbus = sysbus + self._stypes = set() + self._service_browsers = dict() + + # Avahi is an on-demand service. If, for some reason, the avahi-daemon + # were to stop, we need to try to contact it for it to restart. For + # example, when installing the avahi-daemon package on a running system, + # the daemon doesn't get started right away. It needs another process to + # access it over D-Bus to wake it up. The following timer is used to + # periodically query the avahi-daemon until we successfully establish + # first contact. + self._kick_avahi_tmr = gutil.GTimer(60, self._on_kick_avahi) + + # Subscribe for Avahi signals (i.e. events). This must be done before + # any Browser or Resolver is created to avoid race conditions and + # missed events. + self._subscriptions = [ + self._sysbus.connection.signal_subscribe( + Avahi.DBUS_NAME, + Avahi.DBUS_INTERFACE_SERVICE_BROWSER, + 'ItemNew', + None, + None, + 0, + self._service_discovered, + ), + self._sysbus.connection.signal_subscribe( + Avahi.DBUS_NAME, + Avahi.DBUS_INTERFACE_SERVICE_BROWSER, + 'ItemRemove', + None, + None, + 0, + self._service_removed, + ), + self._sysbus.connection.signal_subscribe( + Avahi.DBUS_NAME, Avahi.DBUS_INTERFACE_SERVICE_BROWSER, 'Failure', None, None, 0, self._failure_handler + ), + self._sysbus.connection.signal_subscribe( + Avahi.DBUS_NAME, Avahi.DBUS_INTERFACE_SERVICE_RESOLVER, 'Found', None, None, 0, self._service_identified + ), + self._sysbus.connection.signal_subscribe( + Avahi.DBUS_NAME, Avahi.DBUS_INTERFACE_SERVICE_RESOLVER, 'Failure', None, None, 0, self._failure_handler + ), + ] + + self._avahi = self._sysbus.get_proxy(Avahi.DBUS_NAME, '/') + + self._avahi_watcher = dasbus.client.observer.DBusObserver(self._sysbus, Avahi.DBUS_NAME) + self._avahi_watcher.service_available.connect(self._avahi_available) + self._avahi_watcher.service_unavailable.connect(self._avahi_unavailable) + self._avahi_watcher.connect_once_available() + + def kill(self): + '''@brief Clean up object''' + logging.debug('Avahi.kill()') + + self._kick_avahi_tmr.kill() + self._kick_avahi_tmr = None + + for subscription in self._subscriptions: + self._sysbus.connection.signal_unsubscribe(subscription) + self._subscriptions = list() + + self._disconnect() + + self._avahi_watcher.service_available.disconnect() + self._avahi_watcher.service_unavailable.disconnect() + self._avahi_watcher.disconnect() + self._avahi_watcher = None + + dasbus.client.proxy.disconnect_proxy(self._avahi) + self._avahi = None + + self._change_cb = None + self._sysbus = None + + def info(self) -> dict: + '''@brief return debug info about this object''' + services = dict() + for service, obj in self._services.items(): + interface, protocol, name, stype, domain = service + key = f'({socket.if_indextoname(interface)}, {Avahi.protos.get(protocol, "unknown")}, {name}.{domain}, {stype})' + services[key] = obj.get('data', {}) + + info = { + 'avahi wake up timer': str(self._kick_avahi_tmr), + 'service types': list(self._stypes), + 'services': services, + } + + return info + + def get_controllers(self) -> list: + '''@brief Get the discovery controllers as a list of dict() + as follows: + [ + { + 'transport': tcp, + 'traddr': str(), + 'trsvcid': str(), + 'host-iface': str(), + 'subsysnqn': 'nqn.2014-08.org.nvmexpress.discovery', + }, + { + 'transport': tcp, + 'traddr': str(), + 'trsvcid': str(), + 'host-iface': str(), + 'subsysnqn': 'nqn.2014-08.org.nvmexpress.discovery', + }, + [...] + ] + ''' + return [service['data'] for service in self._services.values() if len(service['data'])] + + def config_stypes(self, stypes: list): + '''@brief Configure the service types that we want to discover. + @param stypes: A list of services types, e.g. ['_nvme-disc._tcp'] + ''' + self._stypes = set(stypes) + success = self._configure_browsers() + if not success: + self._kick_avahi_tmr.start() + + def kick_start(self): + '''@brief We use this to kick start the Avahi + daemon (i.e. socket activation). + ''' + self._kick_avahi_tmr.clear() + + def _disconnect(self): + logging.debug('Avahi._disconnect()') + for service in self._services.values(): + resolver = service.pop('resolver', None) + if resolver is not None: + try: + resolver.Free() + dasbus.client.proxy.disconnect_proxy(resolver) + except (AttributeError, dasbus.error.DBusError) as ex: + logging.debug('Avahi._disconnect() - Failed to Free() resolver. %s', ex) + + self._services = dict() + + for browser in self._service_browsers.values(): + try: + browser.Free() + dasbus.client.proxy.disconnect_proxy(browser) + except (AttributeError, dasbus.error.DBusError) as ex: + logging.debug('Avahi._disconnect() - Failed to Free() browser. %s', ex) + + self._service_browsers = dict() + + def _on_kick_avahi(self): + try: + # try to contact avahi-daemon. This is just a wake + # up call in case the avahi-daemon was sleeping. + self._avahi.GetVersionString() + except dasbus.error.DBusError: + return GLib.SOURCE_CONTINUE + + return GLib.SOURCE_REMOVE + + def _avahi_available(self, _avahi_watcher): + '''@brief Hook up DBus signal handlers for signals from stafd.''' + logging.info('avahi-daemon service available, zeroconf supported.') + success = self._configure_browsers() + if not success: + self._kick_avahi_tmr.start() + + def _avahi_unavailable(self, _avahi_watcher): + self._disconnect() + logging.warning('avahi-daemon not available, zeroconf not supported.') + self._kick_avahi_tmr.start() + + def _configure_browsers(self): + stypes_cur = set(self._service_browsers.keys()) + stypes_to_add = self._stypes - stypes_cur + stypes_to_rm = stypes_cur - self._stypes + + logging.debug('Avahi._configure_browsers() - stypes_to_rm = %s', list(stypes_to_rm)) + logging.debug('Avahi._configure_browsers() - stypes_to_add = %s', list(stypes_to_add)) + + for stype_to_rm in stypes_to_rm: + browser = self._service_browsers.pop(stype_to_rm, None) + if browser is not None: + try: + browser.Free() + dasbus.client.proxy.disconnect_proxy(browser) + except (AttributeError, dasbus.error.DBusError) as ex: + logging.debug('Avahi._configure_browsers() - Failed to Free() browser. %s', ex) + + # Find the cached services corresponding to stype_to_rm and remove them + services_to_rm = [service for service in self._services if service[3] == stype_to_rm] + for service in services_to_rm: + resolver = self._services.pop(service, {}).pop('resolver', None) + if resolver is not None: + try: + resolver.Free() + dasbus.client.proxy.disconnect_proxy(resolver) + except (AttributeError, dasbus.error.DBusError) as ex: + logging.debug('Avahi._configure_browsers() - Failed to Free() resolver. %s', ex) + + for stype in stypes_to_add: + try: + obj_path = self._avahi.ServiceBrowserNew( + Avahi.IF_UNSPEC, Avahi.PROTO_UNSPEC, stype, 'local', Avahi.LOOKUP_USE_MULTICAST + ) + self._service_browsers[stype] = self._sysbus.get_proxy(Avahi.DBUS_NAME, obj_path) + except dasbus.error.DBusError as ex: + logging.debug('Avahi._configure_browsers() - Failed to contact avahi-daemon. %s', ex) + logging.warning('avahi-daemon not available, operating w/o mDNS discovery.') + return False + + return True + + def _service_discovered( + self, + _connection, + _sender_name: str, + _object_path: str, + _interface_name: str, + _signal_name: str, + args: typing.Tuple[int, int, str, str, str, int], + *_user_data, + ): + (interface, protocol, name, stype, domain, flags) = args + logging.debug( + 'Avahi._service_discovered() - interface=%s (%s), protocol=%s, stype=%s, domain=%s, flags=%s %-14s name=%s', + interface, + socket.if_indextoname(interface), + Avahi.protocol_as_string(protocol), + stype, + domain, + flags, + '(' + Avahi.result_flags_as_string(flags) + '),', + name, + ) + + service = (interface, protocol, name, stype, domain) + if service not in self._services: + try: + obj_path = self._avahi.ServiceResolverNew( + interface, protocol, name, stype, domain, Avahi.PROTO_UNSPEC, Avahi.LOOKUP_USE_MULTICAST + ) + self._services[service] = { + 'resolver': self._sysbus.get_proxy(Avahi.DBUS_NAME, obj_path), + 'data': {}, + } + except dasbus.error.DBusError as ex: + logging.warning('Failed to create resolver: "%s", "%s", "%s". %s', interface, name, stype, ex) + + def _service_removed( + self, + _connection, + _sender_name: str, + _object_path: str, + _interface_name: str, + _signal_name: str, + args: typing.Tuple[int, int, str, str, str, int], + *_user_data, + ): + (interface, protocol, name, stype, domain, flags) = args + logging.debug( + 'Avahi._service_removed() - interface=%s (%s), protocol=%s, stype=%s, domain=%s, flags=%s %-14s name=%s', + interface, + socket.if_indextoname(interface), + Avahi.protocol_as_string(protocol), + stype, + domain, + flags, + '(' + Avahi.result_flags_as_string(flags) + '),', + name, + ) + + service = (interface, protocol, name, stype, domain) + resolver = self._services.pop(service, {}).pop('resolver', None) + if resolver is not None: + try: + resolver.Free() + dasbus.client.proxy.disconnect_proxy(resolver) + except (AttributeError, dasbus.error.DBusError) as ex: + logging.debug('Avahi._service_removed() - Failed to Free() resolver. %s', ex) + + self._change_cb() + + def _service_identified( # pylint: disable=too-many-locals + self, + _connection, + _sender_name: str, + _object_path: str, + _interface_name: str, + _signal_name: str, + args: typing.Tuple[int, int, str, str, str, str, int, str, int, list, int], + *_user_data, + ): + (interface, protocol, name, stype, domain, host, aprotocol, address, port, txt, flags) = args + txt = _txt2dict(txt) + logging.debug( + 'Avahi._service_identified() - interface=%s (%s), protocol=%s, stype=%s, domain=%s, flags=%s %-14s name=%s, host=%s, aprotocol=%s, address=%s, port=%s, txt=%s', + interface, + socket.if_indextoname(interface), + Avahi.protocol_as_string(protocol), + stype, + domain, + flags, + '(' + Avahi.result_flags_as_string(flags) + '),', + name, + host, + Avahi.protocol_as_string(aprotocol), + address, + port, + txt, + ) + + service = (interface, protocol, name, stype, domain) + if service in self._services: + transport = _proto2trans(txt.get('p')) + if transport is not None: + self._services[service]['data'] = { + 'transport': transport, + 'traddr': address.strip(), + 'trsvcid': str(port).strip(), + # host-iface permitted for tcp alone and not rdma + 'host-iface': socket.if_indextoname(interface).strip() if transport == 'tcp' else '', + 'subsysnqn': txt.get('nqn', defs.WELL_KNOWN_DISC_NQN).strip() + if conf.NvmeOptions().discovery_supp + else defs.WELL_KNOWN_DISC_NQN, + } + + self._change_cb() + else: + logging.error( + 'Received invalid/undefined protocol in mDNS TXT field: address=%s, iface=%s, TXT=%s', + address, + socket.if_indextoname(interface).strip(), + txt, + ) + + def _failure_handler( # pylint: disable=no-self-use + self, + _connection, + _sender_name: str, + _object_path: str, + interface_name: str, + _signal_name: str, + args: typing.Tuple[str], + *_user_data, + ): + (error,) = args + if 'ServiceResolver' not in interface_name or 'TimeoutError' not in error: + # ServiceResolver may fire a timeout event after being Free'd(). This seems to be normal. + logging.error('Avahi._failure_handler() - name=%s, error=%s', interface_name, error) diff --git a/staslib/conf.py b/staslib/conf.py new file mode 100644 index 0000000..a54da98 --- /dev/null +++ b/staslib/conf.py @@ -0,0 +1,703 @@ +# Copyright (c) 2022, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). +# +# Authors: Martin Belanger <Martin.Belanger@dell.com> +# +'''nvme-stas configuration module''' + +import re +import os +import sys +import logging +import functools +import configparser +from staslib import defs, singleton, timeparse + +__TOKEN_RE = re.compile(r'\s*;\s*') +__OPTION_RE = re.compile(r'\s*=\s*') + + +class InvalidOption(Exception): + '''Exception raised when an invalid option value is detected''' + + +def _parse_controller(controller): + '''@brief Parse a "controller" entry. Controller entries are strings + composed of several configuration parameters delimited by + semi-colons. Each configuration parameter is specified as a + "key=value" pair. + @return A dictionary of key-value pairs. + ''' + options = dict() + tokens = __TOKEN_RE.split(controller) + for token in tokens: + if token: + try: + option, val = __OPTION_RE.split(token) + options[option.strip()] = val.strip() + except ValueError: + pass + + return options + + +def _parse_single_val(text): + if isinstance(text, str): + return text + if not isinstance(text, list) or len(text) == 0: + return None + + return text[-1] + + +def _parse_list(text): + return text if isinstance(text, list) else [text] + + +def _to_int(text): + try: + return int(_parse_single_val(text)) + except (ValueError, TypeError): + raise InvalidOption # pylint: disable=raise-missing-from + + +def _to_bool(text, positive='true'): + return _parse_single_val(text).lower() == positive + + +def _to_ncc(text): + value = _to_int(text) + if value == 1: # 1 is invalid. A minimum of 2 is required (with the exception of 0, which is valid). + value = 2 + return value + + +def _to_ip_family(text): + return tuple((4 if text == 'ipv4' else 6 for text in _parse_single_val(text).split('+'))) + + +# ****************************************************************************** +class OrderedMultisetDict(dict): + '''This class is used to change the behavior of configparser.ConfigParser + and allow multiple configuration parameters with the same key. The + result is a list of values. + ''' + + def __setitem__(self, key, value): + if key in self and isinstance(value, list): + self[key].extend(value) + else: + super().__setitem__(key, value) + + def __getitem__(self, key): + value = super().__getitem__(key) + + if isinstance(value, str): + return value.split('\n') + + return value + + +class SvcConf(metaclass=singleton.Singleton): # pylint: disable=too-many-public-methods + '''Read and cache configuration file.''' + + OPTION_CHECKER = { + 'Global': { + 'tron': { + 'convert': _to_bool, + 'default': False, + 'txt-chk': lambda text: _parse_single_val(text).lower() in ('false', 'true'), + }, + 'kato': { + 'convert': _to_int, + }, + 'pleo': { + 'convert': functools.partial(_to_bool, positive='enabled'), + 'default': True, + 'txt-chk': lambda text: _parse_single_val(text).lower() in ('disabled', 'enabled'), + }, + 'ip-family': { + 'convert': _to_ip_family, + 'default': (4, 6), + 'txt-chk': lambda text: _parse_single_val(text) in ('ipv4', 'ipv6', 'ipv4+ipv6', 'ipv6+ipv4'), + }, + 'queue-size': { + 'convert': _to_int, + 'rng-chk': lambda value: None if value in range(16, 1025) else range(16, 1025), + }, + 'hdr-digest': { + 'convert': _to_bool, + 'default': False, + 'txt-chk': lambda text: _parse_single_val(text).lower() in ('false', 'true'), + }, + 'data-digest': { + 'convert': _to_bool, + 'default': False, + 'txt-chk': lambda text: _parse_single_val(text).lower() in ('false', 'true'), + }, + 'ignore-iface': { + 'convert': _to_bool, + 'default': False, + 'txt-chk': lambda text: _parse_single_val(text).lower() in ('false', 'true'), + }, + 'nr-io-queues': { + 'convert': _to_int, + }, + 'ctrl-loss-tmo': { + 'convert': _to_int, + }, + 'disable-sqflow': { + 'convert': _to_bool, + 'txt-chk': lambda text: _parse_single_val(text).lower() in ('false', 'true'), + }, + 'nr-poll-queues': { + 'convert': _to_int, + }, + 'nr-write-queues': { + 'convert': _to_int, + }, + 'reconnect-delay': { + 'convert': _to_int, + }, + ### BEGIN: LEGACY SECTION TO BE REMOVED ### + 'persistent-connections': { + 'convert': _to_bool, + 'default': False, + 'txt-chk': lambda text: _parse_single_val(text).lower() in ('false', 'true'), + }, + ### END: LEGACY SECTION TO BE REMOVED ### + }, + 'Service Discovery': { + 'zeroconf': { + 'convert': functools.partial(_to_bool, positive='enabled'), + 'default': True, + 'txt-chk': lambda text: _parse_single_val(text).lower() in ('disabled', 'enabled'), + }, + }, + 'Discovery controller connection management': { + 'persistent-connections': { + 'convert': _to_bool, + 'default': True, + 'txt-chk': lambda text: _parse_single_val(text).lower() in ('false', 'true'), + }, + 'zeroconf-connections-persistence': { + 'convert': lambda text: timeparse.timeparse(_parse_single_val(text)), + 'default': timeparse.timeparse('72hours'), + }, + }, + 'I/O controller connection management': { + 'disconnect-scope': { + 'convert': _parse_single_val, + 'default': 'only-stas-connections', + 'txt-chk': lambda text: _parse_single_val(text) + in ('only-stas-connections', 'all-connections-matching-disconnect-trtypes', 'no-disconnect'), + }, + 'disconnect-trtypes': { + # Use set() to eliminate potential duplicates + 'convert': lambda text: set(_parse_single_val(text).split('+')), + 'default': [ + 'tcp', + ], + 'lst-chk': ('tcp', 'rdma', 'fc'), + }, + 'connect-attempts-on-ncc': { + 'convert': _to_ncc, + 'default': 0, + }, + }, + 'Controllers': { + 'controller': { + 'convert': _parse_list, + 'default': [], + }, + 'exclude': { + 'convert': _parse_list, + 'default': [], + }, + ### BEGIN: LEGACY SECTION TO BE REMOVED ### + 'blacklist': { + 'convert': _parse_list, + 'default': [], + }, + ### END: LEGACY SECTION TO BE REMOVED ### + }, + } + + def __init__(self, default_conf=None, conf_file='/dev/null'): + self._config = None + self._defaults = default_conf if default_conf else {} + + if self._defaults is not None and len(self._defaults) != 0: + self._valid_conf = {} + for section, option in self._defaults: + self._valid_conf.setdefault(section, set()).add(option) + else: + self._valid_conf = None + + self._conf_file = conf_file + self.reload() + + def reload(self): + '''@brief Reload the configuration file.''' + self._config = self._read_conf_file() + + @property + def conf_file(self): + '''Return the configuration file name''' + return self._conf_file + + def set_conf_file(self, fname): + '''Set the configuration file name and reload config''' + self._conf_file = fname + self.reload() + + def get_option(self, section, option, ignore_default=False): # pylint: disable=too-many-locals + '''Retrieve @option from @section, convert raw text to + appropriate object type, and validate.''' + try: + checker = self.OPTION_CHECKER[section][option] + except KeyError: + logging.error('Requesting invalid section=%s and/or option=%s', section, option) + raise + + default = checker.get('default', None) + + try: + text = self._config.get(section=section, option=option) + except (configparser.NoSectionError, configparser.NoOptionError, KeyError): + return None if ignore_default else self._defaults.get((section, option), default) + + return self._check(text, section, option, default) + + tron = property(functools.partial(get_option, section='Global', option='tron')) + kato = property(functools.partial(get_option, section='Global', option='kato')) + ip_family = property(functools.partial(get_option, section='Global', option='ip-family')) + queue_size = property(functools.partial(get_option, section='Global', option='queue-size')) + hdr_digest = property(functools.partial(get_option, section='Global', option='hdr-digest')) + data_digest = property(functools.partial(get_option, section='Global', option='data-digest')) + ignore_iface = property(functools.partial(get_option, section='Global', option='ignore-iface')) + pleo_enabled = property(functools.partial(get_option, section='Global', option='pleo')) + nr_io_queues = property(functools.partial(get_option, section='Global', option='nr-io-queues')) + ctrl_loss_tmo = property(functools.partial(get_option, section='Global', option='ctrl-loss-tmo')) + disable_sqflow = property(functools.partial(get_option, section='Global', option='disable-sqflow')) + nr_poll_queues = property(functools.partial(get_option, section='Global', option='nr-poll-queues')) + nr_write_queues = property(functools.partial(get_option, section='Global', option='nr-write-queues')) + reconnect_delay = property(functools.partial(get_option, section='Global', option='reconnect-delay')) + + zeroconf_enabled = property(functools.partial(get_option, section='Service Discovery', option='zeroconf')) + + zeroconf_persistence_sec = property( + functools.partial( + get_option, section='Discovery controller connection management', option='zeroconf-connections-persistence' + ) + ) + + disconnect_scope = property( + functools.partial(get_option, section='I/O controller connection management', option='disconnect-scope') + ) + disconnect_trtypes = property( + functools.partial(get_option, section='I/O controller connection management', option='disconnect-trtypes') + ) + connect_attempts_on_ncc = property( + functools.partial(get_option, section='I/O controller connection management', option='connect-attempts-on-ncc') + ) + + @property + def stypes(self): + '''@brief Get the DNS-SD/mDNS service types.''' + return ['_nvme-disc._tcp', '_nvme-disc._udp'] if self.zeroconf_enabled else list() + + @property + def persistent_connections(self): + '''@brief return the "persistent-connections" config parameter''' + section = 'Discovery controller connection management' + option = 'persistent-connections' + + value = self.get_option(section, option, ignore_default=True) + legacy = self.get_option('Global', 'persistent-connections', ignore_default=True) + + if value is None and legacy is None: + return self._defaults.get((section, option), True) + + return value or legacy + + def get_controllers(self): + '''@brief Return the list of controllers in the config file. + Each controller is in the form of a dictionary as follows. + Note that some of the keys are optional. + { + 'transport': [TRANSPORT], + 'traddr': [TRADDR], + 'trsvcid': [TRSVCID], + 'host-traddr': [TRADDR], + 'host-iface': [IFACE], + 'subsysnqn': [NQN], + 'dhchap-ctrl-secret': [KEY], + 'hdr-digest': [BOOL] + 'data-digest': [BOOL] + 'nr-io-queues': [NUMBER] + 'nr-write-queues': [NUMBER] + 'nr-poll-queues': [NUMBER] + 'queue-size': [SIZE] + 'kato': [KATO] + 'reconnect-delay': [SECONDS] + 'ctrl-loss-tmo': [SECONDS] + 'disable-sqflow': [BOOL] + } + ''' + controller_list = self.get_option('Controllers', 'controller') + cids = [_parse_controller(controller) for controller in controller_list] + for cid in cids: + try: + # replace 'nqn' key by 'subsysnqn', if present. + cid['subsysnqn'] = cid.pop('nqn') + except KeyError: + pass + + # Verify values of the options used to overload the matching [Global] options + for option in cid: + if option in self.OPTION_CHECKER['Global']: + value = self._check(cid[option], 'Global', option, None) + if value is not None: + cid[option] = value + + return cids + + def get_excluded(self): + '''@brief Return the list of excluded controllers in the config file. + Each excluded controller is in the form of a dictionary + as follows. All the keys are optional. + { + 'transport': [TRANSPORT], + 'traddr': [TRADDR], + 'trsvcid': [TRSVCID], + 'host-iface': [IFACE], + 'subsysnqn': [NQN], + } + ''' + controller_list = self.get_option('Controllers', 'exclude') + + # 2022-09-20: Look for "blacklist". This is for backwards compatibility + # with releases 1.0 to 1.1.6. This is to be phased out (i.e. remove by 2024) + controller_list += self.get_option('Controllers', 'blacklist') + + excluded = [_parse_controller(controller) for controller in controller_list] + for controller in excluded: + controller.pop('host-traddr', None) # remove host-traddr + try: + # replace 'nqn' key by 'subsysnqn', if present. + controller['subsysnqn'] = controller.pop('nqn') + except KeyError: + pass + return excluded + + def _check(self, text, section, option, default): + checker = self.OPTION_CHECKER[section][option] + text_checker = checker.get('txt-chk', None) + if text_checker is not None and not text_checker(text): + logging.warning( + 'File:%s [%s]: %s - Text check found invalid value "%s". Default will be used', + self.conf_file, + section, + option, + text, + ) + return self._defaults.get((section, option), default) + + converter = checker.get('convert', None) + try: + value = converter(text) + except InvalidOption: + logging.warning( + 'File:%s [%s]: %s - Data converter found invalid value "%s". Default will be used', + self.conf_file, + section, + option, + text, + ) + return self._defaults.get((section, option), default) + + value_in_range = checker.get('rng-chk', None) + if value_in_range is not None: + expected_range = value_in_range(value) + if expected_range is not None: + logging.warning( + 'File:%s [%s]: %s - "%s" is not within range %s..%s. Default will be used', + self.conf_file, + section, + option, + value, + min(expected_range), + max(expected_range), + ) + return self._defaults.get((section, option), default) + + list_checker = checker.get('lst-chk', None) + if list_checker: + values = set() + for item in value: + if item not in list_checker: + logging.warning( + 'File:%s [%s]: %s - List checker found invalid item "%s" will be ignored.', + self.conf_file, + section, + option, + item, + ) + else: + values.add(item) + + if len(values) == 0: + return self._defaults.get((section, option), default) + + value = list(values) + + return value + + def _read_conf_file(self): + '''@brief Read the configuration file if the file exists.''' + config = configparser.ConfigParser( + default_section=None, + allow_no_value=True, + delimiters=('='), + interpolation=None, + strict=False, + dict_type=OrderedMultisetDict, + ) + if self._conf_file and os.path.isfile(self._conf_file): + config.read(self._conf_file) + + # Parse Configuration and validate. + if self._valid_conf is not None: + invalid_sections = set() + for section in config.sections(): + if section not in self._valid_conf: + invalid_sections.add(section) + else: + invalid_options = set() + for option in config.options(section): + if option not in self._valid_conf.get(section, []): + invalid_options.add(option) + + if len(invalid_options) != 0: + logging.error( + 'File:%s [%s] contains invalid options: %s', + self.conf_file, + section, + invalid_options, + ) + + if len(invalid_sections) != 0: + logging.error( + 'File:%s contains invalid sections: %s', + self.conf_file, + invalid_sections, + ) + + return config + + +# ****************************************************************************** +class SysConf(metaclass=singleton.Singleton): + '''Read and cache the host configuration file.''' + + def __init__(self, conf_file=defs.SYS_CONF_FILE): + self._config = None + self._conf_file = conf_file + self.reload() + + def reload(self): + '''@brief Reload the configuration file.''' + self._config = self._read_conf_file() + + @property + def conf_file(self): + '''Return the configuration file name''' + return self._conf_file + + def set_conf_file(self, fname): + '''Set the configuration file name and reload config''' + self._conf_file = fname + self.reload() + + def as_dict(self): + '''Return configuration as a dictionary''' + return { + 'hostnqn': self.hostnqn, + 'hostid': self.hostid, + 'hostkey': self.hostkey, + 'symname': self.hostsymname, + } + + @property + def hostnqn(self): + '''@brief return the host NQN + @return: Host NQN + @raise: Host NQN is mandatory. The program will terminate if a + Host NQN cannot be determined. + ''' + try: + value = self.__get_value('Host', 'nqn', defs.NVME_HOSTNQN) + except FileNotFoundError as ex: + sys.exit(f'Error reading mandatory Host NQN (see stasadm --help): {ex}') + + if value is not None and not value.startswith('nqn.'): + sys.exit(f'Error Host NQN "{value}" should start with "nqn."') + + return value + + @property + def hostid(self): + '''@brief return the host ID + @return: Host ID + @raise: Host ID is mandatory. The program will terminate if a + Host ID cannot be determined. + ''' + try: + value = self.__get_value('Host', 'id', defs.NVME_HOSTID) + except FileNotFoundError as ex: + sys.exit(f'Error reading mandatory Host ID (see stasadm --help): {ex}') + + return value + + @property + def hostkey(self): + '''@brief return the host key + @return: Host key + @raise: Host key is optional, but mandatory if authorization will be performed. + ''' + try: + value = self.__get_value('Host', 'key', defs.NVME_HOSTKEY) + except FileNotFoundError as ex: + logging.info('Host key undefined: %s', ex) + value = None + + return value + + @property + def hostsymname(self): + '''@brief return the host symbolic name (or None) + @return: symbolic name or None + ''' + try: + value = self.__get_value('Host', 'symname') + except FileNotFoundError as ex: + logging.warning('Error reading host symbolic name (will remain undefined): %s', ex) + value = None + + return value + + def _read_conf_file(self): + '''@brief Read the configuration file if the file exists.''' + config = configparser.ConfigParser( + default_section=None, allow_no_value=True, delimiters=('='), interpolation=None, strict=False + ) + if os.path.isfile(self._conf_file): + config.read(self._conf_file) + return config + + def __get_value(self, section, option, default_file=None): + '''@brief A configuration file consists of sections, each led by a + [section] header, followed by key/value entries separated + by a equal sign (=). This method retrieves the value + associated with the key @option from the section @section. + If the value starts with the string "file://", then the value + will be retrieved from that file. + + @param section: Configuration section + @param option: The key to look for + @param default_file: A file that contains the default value + + @return: On success, the value associated with the key. On failure, + this method will return None is a default_file is not + specified, or will raise an exception if a file is not + found. + + @raise: This method will raise the FileNotFoundError exception if + the value retrieved is a file that does not exist. + ''' + try: + value = self._config.get(section=section, option=option) + if not value.startswith('file://'): + return value + file = value[7:] + except (configparser.NoSectionError, configparser.NoOptionError, KeyError): + if default_file is None: + return None + file = default_file + + try: + with open(file) as f: # pylint: disable=unspecified-encoding + return f.readline().split()[0] + except IndexError: + return None + + +# ****************************************************************************** +class NvmeOptions(metaclass=singleton.Singleton): + '''Object used to read and cache contents of file /dev/nvme-fabrics. + Note that this file was not readable prior to Linux 5.16. + ''' + + def __init__(self): + # Supported options can be determined by looking at the kernel version + # or by reading '/dev/nvme-fabrics'. The ability to read the options + # from '/dev/nvme-fabrics' was only introduced in kernel 5.17, but may + # have been backported to older kernels. In any case, if the kernel + # version meets the minimum version for that option, then we don't + # even need to read '/dev/nvme-fabrics'. + self._supported_options = { + 'discovery': defs.KERNEL_VERSION >= defs.KERNEL_TP8013_MIN_VERSION, + 'host_iface': defs.KERNEL_VERSION >= defs.KERNEL_IFACE_MIN_VERSION, + 'dhchap_secret': defs.KERNEL_VERSION >= defs.KERNEL_HOSTKEY_MIN_VERSION, + 'dhchap_ctrl_secret': defs.KERNEL_VERSION >= defs.KERNEL_CTRLKEY_MIN_VERSION, + } + + # If some of the options are False, we need to check wether they can be + # read from '/dev/nvme-fabrics'. This method allows us to determine that + # an older kernel actually supports a specific option because it was + # backported to that kernel. + if not all(self._supported_options.values()): # At least one option is False. + try: + with open('/dev/nvme-fabrics') as f: # pylint: disable=unspecified-encoding + options = [option.split('=')[0].strip() for option in f.readline().rstrip('\n').split(',')] + except PermissionError: # Must be root to read this file + raise + except (OSError, FileNotFoundError): + logging.warning('Cannot determine which NVMe options the kernel supports') + else: + for option, supported in self._supported_options.items(): + if not supported: + self._supported_options[option] = option in options + + def __str__(self): + return f'supported options: {self._supported_options}' + + def get(self): + '''get the supported options as a dict''' + return self._supported_options + + @property + def discovery_supp(self): + '''This option adds support for TP8013''' + return self._supported_options['discovery'] + + @property + def host_iface_supp(self): + '''This option allows forcing connections to go over + a specific interface regardless of the routing tables. + ''' + return self._supported_options['host_iface'] + + @property + def dhchap_hostkey_supp(self): + '''This option allows specifying the host DHCHAP key used for authentication.''' + return self._supported_options['dhchap_secret'] + + @property + def dhchap_ctrlkey_supp(self): + '''This option allows specifying the controller DHCHAP key used for authentication.''' + return self._supported_options['dhchap_ctrl_secret'] diff --git a/staslib/ctrl.py b/staslib/ctrl.py new file mode 100644 index 0000000..97a1c7b --- /dev/null +++ b/staslib/ctrl.py @@ -0,0 +1,850 @@ +# Copyright (c) 2022, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). +# +# Authors: Martin Belanger <Martin.Belanger@dell.com> +# +'''This module defines the base Controller object from which the +Dc (Discovery Controller) and Ioc (I/O Controller) objects are derived.''' + +import time +import inspect +import logging +from gi.repository import GLib +from libnvme import nvme +from staslib import conf, defs, gutil, trid, udev, stas + + +DLP_CHANGED = ( + (nvme.NVME_LOG_LID_DISCOVER << 16) | (nvme.NVME_AER_NOTICE_DISC_CHANGED << 8) | nvme.NVME_AER_NOTICE +) # 0x70f002 + + +def get_eflags(dlpe): + '''@brief Return eflags field of dlpe''' + return int(dlpe.get('eflags', 0)) if dlpe else 0 + + +def get_ncc(eflags: int): + '''@brief Return True if Not Connected to CDC bit is asserted, False otherwise''' + return eflags & nvme.NVMF_DISC_EFLAGS_NCC != 0 + + +def dlp_supp_opts_as_string(dlp_supp_opts: int): + '''@brief Return the list of options supported by the Get + discovery log page command. + ''' + data = { + nvme.NVMF_LOG_DISC_LID_EXTDLPES: "EXTDLPES", + nvme.NVMF_LOG_DISC_LID_PLEOS: "PLEOS", + nvme.NVMF_LOG_DISC_LID_ALLSUBES: "ALLSUBES", + } + return [txt for msk, txt in data.items() if dlp_supp_opts & msk] + + +# ****************************************************************************** +class Controller(stas.ControllerABC): # pylint: disable=too-many-instance-attributes + '''@brief Base class used to manage the connection to a controller.''' + + def __init__(self, tid: trid.TID, service, discovery_ctrl: bool = False): + sysconf = conf.SysConf() + self._nvme_options = conf.NvmeOptions() + self._root = nvme.root() + self._host = nvme.host( + self._root, hostnqn=sysconf.hostnqn, hostid=sysconf.hostid, hostsymname=sysconf.hostsymname + ) + self._host.dhchap_key = sysconf.hostkey if self._nvme_options.dhchap_hostkey_supp else None + self._udev = udev.UDEV + self._device = None # Refers to the nvme device (e.g. /dev/nvme[n]) + self._ctrl = None # libnvme's nvme.ctrl object + self._connect_op = None + + super().__init__(tid, service, discovery_ctrl) + + def _release_resources(self): + logging.debug('Controller._release_resources() - %s | %s', self.id, self.device) + + if self._udev: + self._udev.unregister_for_device_events(self._on_udev_notification) + + self._kill_ops() + + super()._release_resources() + + self._ctrl = None + self._udev = None + self._host = None + self._root = None + self._nvme_options = None + + @property + def device(self) -> str: + '''@brief return the Linux nvme device id (e.g. nvme3) or empty + string if no device is associated with this controller''' + if not self._device and self._ctrl and self._ctrl.name: + self._device = self._ctrl.name + + return self._device or 'nvme?' + + def all_ops_completed(self) -> bool: + '''@brief Returns True if all operations have completed. False otherwise.''' + return self._connect_op is None or self._connect_op.completed() + + def connected(self): + '''@brief Return whether a connection is established''' + return self._ctrl and self._ctrl.connected() + + def controller_id_dict(self) -> dict: + '''@brief return the controller ID as a dict.''' + cid = super().controller_id_dict() + cid['device'] = self.device + return cid + + def details(self) -> dict: + '''@brief return detailed debug info about this controller''' + details = super().details() + details.update( + self._udev.get_attributes(self.device, ('hostid', 'hostnqn', 'model', 'serial', 'dctype', 'cntrltype')) + ) + details['connected'] = str(self.connected()) + return details + + def info(self) -> dict: + '''@brief Get the controller info for this object''' + info = super().info() + if self._connect_op: + info['connect operation'] = str(self._connect_op.as_dict()) + return info + + def cancel(self): + '''@brief Used to cancel pending operations.''' + super().cancel() + if self._connect_op: + self._connect_op.cancel() + + def _kill_ops(self): + if self._connect_op: + self._connect_op.kill() + self._connect_op = None + + def set_level_from_tron(self, tron): + '''Set log level based on TRON''' + if self._root: + self._root.log_level("debug" if tron else "err") + + def _on_udev_notification(self, udev_obj): + if self._alive(): + if udev_obj.action == 'change': + nvme_aen = udev_obj.get('NVME_AEN') + nvme_event = udev_obj.get('NVME_EVENT') + if isinstance(nvme_aen, str): + logging.info('%s | %s - Received AEN: %s', self.id, udev_obj.sys_name, nvme_aen) + self._on_aen(int(nvme_aen, 16)) + if isinstance(nvme_event, str): + self._on_nvme_event(nvme_event) + elif udev_obj.action == 'remove': + logging.info('%s | %s - Received "remove" event', self.id, udev_obj.sys_name) + self._on_ctrl_removed(udev_obj) + else: + logging.debug( + 'Controller._on_udev_notification() - %s | %s: Received "%s" event', + self.id, + udev_obj.sys_name, + udev_obj.action, + ) + else: + logging.debug( + 'Controller._on_udev_notification() - %s | %s: Received event on dead object. udev_obj %s: %s', + self.id, + self.device, + udev_obj.action, + udev_obj.sys_name, + ) + + def _on_ctrl_removed(self, udev_obj): # pylint: disable=unused-argument + if self._udev: + self._udev.unregister_for_device_events(self._on_udev_notification) + self._kill_ops() # Kill all pending operations + self._ctrl = None + + # Defer removal of this object to the next main loop's idle period. + GLib.idle_add(self._serv.remove_controller, self, True) + + def _get_cfg(self): + '''Get configuration parameters. These may either come from the [Global] + section or from a "controller" entry in the configuration file. A + definition found in a "controller" entry overrides the same definition + found in the [Global] section. + ''' + cfg = {} + service_conf = conf.SvcConf() + for option, keyword in ( + ('kato', 'keep_alive_tmo'), + ('queue-size', 'queue_size'), + ('hdr-digest', 'hdr_digest'), + ('data-digest', 'data_digest'), + ('nr-io-queues', 'nr_io_queues'), + ('ctrl-loss-tmo', 'ctrl_loss_tmo'), + ('disable-sqflow', 'disable_sqflow'), + ('nr-poll-queues', 'nr_poll_queues'), + ('nr-write-queues', 'nr_write_queues'), + ('reconnect-delay', 'reconnect_delay'), + ): + # Check if the value is defined as a "controller" entry (i.e. override) + ovrd_val = self.tid.cfg.get(option, None) + if ovrd_val is not None: + cfg[keyword] = ovrd_val + else: + # Check if the value is found in the [Global] section. + glob_val = service_conf.get_option('Global', option) + if glob_val is not None: + cfg[keyword] = glob_val + + return cfg + + def _do_connect(self): + service_conf = conf.SvcConf() + host_iface = ( + self.tid.host_iface + if (self.tid.host_iface and not service_conf.ignore_iface and self._nvme_options.host_iface_supp) + else None + ) + self._ctrl = nvme.ctrl( + self._root, + subsysnqn=self.tid.subsysnqn, + transport=self.tid.transport, + traddr=self.tid.traddr, + trsvcid=self.tid.trsvcid if self.tid.trsvcid else None, + host_traddr=self.tid.host_traddr if self.tid.host_traddr else None, + host_iface=host_iface, + ) + self._ctrl.discovery_ctrl_set(self._discovery_ctrl) + + # Set the DHCHAP key on the controller + # NOTE that this will eventually have to + # change once we have support for AVE (TP8019) + ctrl_dhchap_key = self.tid.cfg.get('dhchap-ctrl-secret') + if ctrl_dhchap_key and self._nvme_options.dhchap_ctrlkey_supp: + has_dhchap_key = hasattr(self._ctrl, 'dhchap_key') + if not has_dhchap_key: + logging.warning( + '%s | %s - libnvme-%s does not allow setting the controller DHCHAP key. Please upgrade libnvme.', + self.id, + self.device, + defs.LIBNVME_VERSION, + ) + else: + self._ctrl.dhchap_key = ctrl_dhchap_key + + # Audit existing nvme devices. If we find a match, then + # we'll just borrow that device instead of creating a new one. + udev_obj = self._find_existing_connection() + if udev_obj is not None: + # A device already exists. + self._device = udev_obj.sys_name + logging.debug( + 'Controller._do_connect() - %s Found existing control device: %s', self.id, udev_obj.sys_name + ) + self._connect_op = gutil.AsyncTask( + self._on_connect_success, self._on_connect_fail, self._ctrl.init, self._host, int(udev_obj.sys_number) + ) + else: + cfg = self._get_cfg() + logging.debug( + 'Controller._do_connect() - %s Connecting to nvme control with cfg=%s', self.id, cfg + ) + self._connect_op = gutil.AsyncTask( + self._on_connect_success, self._on_connect_fail, self._ctrl.connect, self._host, cfg + ) + + self._connect_op.run_async() + + # -------------------------------------------------------------------------- + def _on_connect_success(self, op_obj: gutil.AsyncTask, data): + '''@brief Function called when we successfully connect to the + Controller. + ''' + op_obj.kill() + self._connect_op = None + + if self._alive(): + self._device = self._ctrl.name + logging.info('%s | %s - Connection established!', self.id, self.device) + self._connect_attempts = 0 + self._udev.register_for_device_events(self._device, self._on_udev_notification) + else: + logging.debug( + 'Controller._on_connect_success() - %s | %s: Received event on dead object. data=%s', + self.id, + self.device, + data, + ) + + def _on_connect_fail(self, op_obj: gutil.AsyncTask, err, fail_cnt): # pylint: disable=unused-argument + '''@brief Function called when we fail to connect to the Controller.''' + op_obj.kill() + self._connect_op = None + if self._alive(): + if self._connect_attempts == 1: + # Do a fast re-try on the first failure. + self._retry_connect_tmr.set_timeout(self.FAST_CONNECT_RETRY_PERIOD_SEC) + elif self._connect_attempts == 2: + # If the fast connect re-try fails, then we can print a message to + # indicate the failure, and start a slow re-try period. + self._retry_connect_tmr.set_timeout(self.CONNECT_RETRY_PERIOD_SEC) + logging.error('%s Failed to connect to controller. %s %s', self.id, err.domain, err.message) + + if self._should_try_to_reconnect(): + logging.debug( + 'Controller._on_connect_fail() - %s %s. Retry in %s sec.', + self.id, + err, + self._retry_connect_tmr.get_timeout(), + ) + self._retry_connect_tmr.start() + else: + logging.debug( + 'Controller._on_connect_fail() - %s Received event on dead object. %s %s', + self.id, + err.domain, + err.message, + ) + + def disconnect(self, disconnected_cb, keep_connection): + '''@brief Issue an asynchronous disconnect command to a Controller. + Once the async command has completed, the callback 'disconnected_cb' + will be invoked. If a controller is already disconnected, then the + callback will be added to the main loop's next idle slot to be executed + ASAP. + + @param disconnected_cb: Callback to be called when disconnect has + completed. the callback must have this signature: + def cback(controller: Controller, success: bool) + @param keep_connection: Whether the underlying connection should remain + in the kernel. + ''' + logging.debug( + 'Controller.disconnect() - %s | %s: keep_connection=%s', self.id, self.device, keep_connection + ) + if self._ctrl and self._ctrl.connected() and not keep_connection: + logging.info('%s | %s - Disconnect initiated', self.id, self.device) + op = gutil.AsyncTask(self._on_disconn_success, self._on_disconn_fail, self._ctrl.disconnect) + op.run_async(disconnected_cb) + else: + # Defer callback to the next main loop's idle period. The callback + # cannot be called directly as the current Controller object is in the + # process of being disconnected and the callback will in fact delete + # the object. This would invariably lead to unpredictable outcome. + GLib.idle_add(disconnected_cb, self, True) + + def _on_disconn_success(self, op_obj: gutil.AsyncTask, data, disconnected_cb): # pylint: disable=unused-argument + logging.debug('Controller._on_disconn_success() - %s | %s', self.id, self.device) + op_obj.kill() + # Defer callback to the next main loop's idle period. The callback + # cannot be called directly as the current Controller object is in the + # process of being disconnected and the callback will in fact delete + # the object. This would invariably lead to unpredictable outcome. + GLib.idle_add(disconnected_cb, self, True) + + def _on_disconn_fail( + self, op_obj: gutil.AsyncTask, err, fail_cnt, disconnected_cb + ): # pylint: disable=unused-argument + logging.debug('Controller._on_disconn_fail() - %s | %s: %s', self.id, self.device, err) + op_obj.kill() + # Defer callback to the next main loop's idle period. The callback + # cannot be called directly as the current Controller object is in the + # process of being disconnected and the callback will in fact delete + # the object. This would invariably lead to unpredictable outcome. + GLib.idle_add(disconnected_cb, self, False) + + +# ****************************************************************************** +class Dc(Controller): + '''@brief This object establishes a connection to one Discover Controller (DC). + It retrieves the discovery log pages and caches them. + It also monitors udev events associated with that DC and updates + the cached discovery log pages accordingly. + ''' + + GET_LOG_PAGE_RETRY_RERIOD_SEC = 20 + REGISTRATION_RETRY_RERIOD_SEC = 5 + GET_SUPPORTED_RETRY_RERIOD_SEC = 5 + + def __init__(self, staf, tid: trid.TID, log_pages=None, origin=None): + super().__init__(tid, staf, discovery_ctrl=True) + self._register_op = None + self._get_supported_op = None + self._get_log_op = None + self._origin = origin + self._log_pages = log_pages if log_pages else list() # Log pages cache + + # For Avahi-discovered DCs that later become unresponsive, monitor how + # long the controller remains unresponsive and if it does not return for + # a configurable soak period (_ctrl_unresponsive_tmr), remove that + # controller. Only Avahi-discovered controllers need this timeout-based + # cleanup. + self._ctrl_unresponsive_time = None # The time at which connectivity was lost + self._ctrl_unresponsive_tmr = gutil.GTimer(0, self._serv.controller_unresponsive, self.tid) + + def _release_resources(self): + logging.debug('Dc._release_resources() - %s | %s', self.id, self.device) + super()._release_resources() + + if self._ctrl_unresponsive_tmr is not None: + self._ctrl_unresponsive_tmr.kill() + + self._log_pages = list() + self._ctrl_unresponsive_tmr = None + + def _kill_ops(self): + super()._kill_ops() + if self._get_log_op: + self._get_log_op.kill() + self._get_log_op = None + if self._register_op: + self._register_op.kill() + self._register_op = None + if self._get_supported_op: + self._get_supported_op.kill() + self._get_supported_op = None + + def all_ops_completed(self) -> bool: + '''@brief Returns True if all operations have completed. False otherwise.''' + return ( + super().all_ops_completed() + and (self._get_log_op is None or self._get_log_op.completed()) + and (self._register_op is None or self._register_op.completed()) + and (self._get_supported_op is None or self._get_supported_op.completed()) + ) + + @property + def origin(self): + '''@brief Return how this controller came into existance. Was it + "discovered" through mDNS service discovery (TP8009), was it manually + "configured" in stafd.conf, or was it a "referral". + ''' + return self._origin + + @origin.setter + def origin(self, value): + '''@brief Set the origin of this controller.''' + if value in ('discovered', 'configured', 'referral'): + self._origin = value + self._handle_lost_controller() + else: + logging.error('%s | %s - Trying to set invalid origin to %s', self.id, self.device, value) + + def reload_hdlr(self): + '''@brief This is called when a "reload" signal is received.''' + logging.debug('Dc.reload_hdlr() - %s | %s', self.id, self.device) + + self._handle_lost_controller() + self._resync_with_controller() + + def info(self) -> dict: + '''@brief Get the controller info for this object''' + timeout = conf.SvcConf().zeroconf_persistence_sec + unresponsive_time = ( + time.asctime(self._ctrl_unresponsive_time) if self._ctrl_unresponsive_time is not None else '---' + ) + info = super().info() + info['origin'] = self.origin + if self.origin == 'discovered': + # The code that handles "unresponsive" DCs only applies to + # discovered DCs. So, let's only print that info when it's relevant. + info['unresponsive timer'] = str(self._ctrl_unresponsive_tmr) + info['unresponsive timeout'] = f'{timeout} sec' if timeout >= 0 else 'forever' + info['unresponsive time'] = unresponsive_time + if self._get_log_op: + info['get log page operation'] = str(self._get_log_op.as_dict()) + if self._register_op: + info['register operation'] = str(self._register_op.as_dict()) + if self._get_supported_op: + info['get supported log page operation'] = str(self._get_supported_op.as_dict()) + return info + + def cancel(self): + '''@brief Used to cancel pending operations.''' + super().cancel() + if self._get_log_op: + self._get_log_op.cancel() + if self._register_op: + self._register_op.cancel() + if self._get_supported_op: + self._get_supported_op.cancel() + + def log_pages(self) -> list: + '''@brief Get the cached log pages for this object''' + return self._log_pages + + def referrals(self) -> list: + '''@brief Return the list of referrals''' + return [page for page in self._log_pages if page['subtype'] == 'referral'] + + def _is_ddc(self): + return self._ctrl and self._ctrl.dctype != 'cdc' + + def _on_aen(self, aen: int): + if aen == DLP_CHANGED and self._get_log_op: + self._get_log_op.run_async() + + def _handle_lost_controller(self): + if self.origin == 'discovered': # Only apply to mDNS-discovered DCs + if not self._serv.is_avahi_reported(self.tid) and not self.connected(): + timeout = conf.SvcConf().zeroconf_persistence_sec + if timeout >= 0: + if self._ctrl_unresponsive_time is None: + self._ctrl_unresponsive_time = time.localtime() + self._ctrl_unresponsive_tmr.start(timeout) + logging.info( + '%s | %s - Controller is not responding. Will be removed by %s unless restored', + self.id, + self.device, + time.ctime(time.mktime(self._ctrl_unresponsive_time) + timeout), + ) + + return + + logging.info( + '%s | %s - Controller not responding. Retrying...', + self.id, + self.device, + ) + + self._ctrl_unresponsive_time = None + self._ctrl_unresponsive_tmr.stop() + self._ctrl_unresponsive_tmr.set_timeout(0) + + def is_unresponsive(self): + '''@brief For "discovered" DC, return True if DC is unresponsive, + False otherwise. + ''' + return ( + self.origin == 'discovered' + and not self._serv.is_avahi_reported(self.tid) + and not self.connected() + and self._ctrl_unresponsive_time is not None + and self._ctrl_unresponsive_tmr.time_remaining() <= 0 + ) + + def _resync_with_controller(self): + '''Communicate with DC to resync the states''' + if self._register_op: + self._register_op.run_async() + elif self._get_supported_op: + self._get_supported_op.run_async() + elif self._get_log_op: + self._get_log_op.run_async() + + def _on_nvme_event(self, nvme_event: str): + if nvme_event in ('connected', 'rediscover'): + # This event indicates that the kernel + # driver re-connected to the DC. + logging.debug( + 'Dc._on_nvme_event() - %s | %s: Received "%s" event', + self.id, + self.device, + nvme_event, + ) + self._resync_with_controller() + + def _find_existing_connection(self): + return self._udev.find_nvme_dc_device(self.tid) + + def _post_registration_actions(self): + # Need to check that supported_log_pages() is available (introduced in libnvme 1.2) + has_supported_log_pages = hasattr(self._ctrl, 'supported_log_pages') + if not has_supported_log_pages: + logging.warning( + '%s | %s - libnvme-%s does not support "Get supported log pages". Please upgrade libnvme.', + self.id, + self.device, + defs.LIBNVME_VERSION, + ) + + if conf.SvcConf().pleo_enabled and self._is_ddc() and has_supported_log_pages: + self._get_supported_op = gutil.AsyncTask( + self._on_get_supported_success, self._on_get_supported_fail, self._ctrl.supported_log_pages + ) + self._get_supported_op.run_async() + else: + self._get_log_op = gutil.AsyncTask(self._on_get_log_success, self._on_get_log_fail, self._ctrl.discover) + self._get_log_op.run_async() + + # -------------------------------------------------------------------------- + def _on_connect_success(self, op_obj: gutil.AsyncTask, data): + '''@brief Function called when we successfully connect to the + Discovery Controller. + ''' + super()._on_connect_success(op_obj, data) + + if self._alive(): + self._ctrl_unresponsive_time = None + self._ctrl_unresponsive_tmr.stop() + self._ctrl_unresponsive_tmr.set_timeout(0) + + if self._ctrl.is_registration_supported(): + self._register_op = gutil.AsyncTask( + self._on_registration_success, + self._on_registration_fail, + self._ctrl.registration_ctlr, + nvme.NVMF_DIM_TAS_REGISTER, + ) + self._register_op.run_async() + else: + self._post_registration_actions() + + def _on_connect_fail(self, op_obj: gutil.AsyncTask, err, fail_cnt): + '''@brief Function called when we fail to connect to the Controller.''' + super()._on_connect_fail(op_obj, err, fail_cnt) + + if self._alive(): + self._handle_lost_controller() + + # -------------------------------------------------------------------------- + def _on_registration_success(self, op_obj: gutil.AsyncTask, data): # pylint: disable=unused-argument + '''@brief Function called when we successfully register with the + Discovery Controller. See self._register_op object + for details. + + NOTE: The name _on_registration_success() may be misleading. "success" + refers to the fact that a successful exchange was made with the DC. + It doesn't mean that the registration itself succeeded. + ''' + if self._alive(): + if data is not None: + logging.warning('%s | %s - Registration error. %s.', self.id, self.device, data) + else: + logging.debug('Dc._on_registration_success() - %s | %s', self.id, self.device) + + self._post_registration_actions() + else: + logging.debug( + 'Dc._on_registration_success() - %s | %s: Received event on dead object.', self.id, self.device + ) + + def _on_registration_fail(self, op_obj: gutil.AsyncTask, err, fail_cnt): + '''@brief Function called when we fail to register with the + Discovery Controller. See self._register_op object + for details. + ''' + if self._alive(): + logging.debug( + 'Dc._on_registration_fail() - %s | %s: %s. Retry in %s sec', + self.id, + self.device, + err, + Dc.REGISTRATION_RETRY_RERIOD_SEC, + ) + if fail_cnt == 1: # Throttle the logs. Only print the first time the command fails + logging.error('%s | %s - Failed to register with Discovery Controller. %s', self.id, self.device, err) + op_obj.retry(Dc.REGISTRATION_RETRY_RERIOD_SEC) + else: + logging.debug( + 'Dc._on_registration_fail() - %s | %s: Received event on dead object. %s', + self.id, + self.device, + err, + ) + op_obj.kill() + + # -------------------------------------------------------------------------- + def _on_get_supported_success(self, op_obj: gutil.AsyncTask, data): # pylint: disable=unused-argument + '''@brief Function called when we successfully retrieved the supported + log pages from the Discovery Controller. See self._get_supported_op object + for details. + + NOTE: The name _on_get_supported_success() may be misleading. "success" + refers to the fact that a successful exchange was made with the DC. + It doesn't mean that the Get Supported Log Page itself succeeded. + ''' + if self._alive(): + try: + dlp_supp_opts = data[nvme.NVME_LOG_LID_DISCOVER] >> 16 + except (TypeError, IndexError): + dlp_supp_opts = 0 + + logging.debug( + 'Dc._on_get_supported_success() - %s | %s: supported options = 0x%04X = %s', + self.id, + self.device, + dlp_supp_opts, + dlp_supp_opts_as_string(dlp_supp_opts), + ) + + if 'lsp' in inspect.signature(self._ctrl.discover).parameters: + lsp = nvme.NVMF_LOG_DISC_LSP_PLEO if dlp_supp_opts & nvme.NVMF_LOG_DISC_LID_PLEOS else 0 + self._get_log_op = gutil.AsyncTask( + self._on_get_log_success, self._on_get_log_fail, self._ctrl.discover, lsp + ) + else: + logging.warning( + '%s | %s - libnvme-%s does not support setting PLEO bit. Please upgrade.', + self.id, + self.device, + defs.LIBNVME_VERSION, + ) + self._get_log_op = gutil.AsyncTask(self._on_get_log_success, self._on_get_log_fail, self._ctrl.discover) + self._get_log_op.run_async() + else: + logging.debug( + 'Dc._on_get_supported_success() - %s | %s: Received event on dead object.', self.id, self.device + ) + + def _on_get_supported_fail(self, op_obj: gutil.AsyncTask, err, fail_cnt): + '''@brief Function called when we fail to retrieve the supported log + page from the Discovery Controller. See self._get_supported_op object + for details. + ''' + if self._alive(): + logging.debug( + 'Dc._on_get_supported_fail() - %s | %s: %s. Retry in %s sec', + self.id, + self.device, + err, + Dc.GET_SUPPORTED_RETRY_RERIOD_SEC, + ) + if fail_cnt == 1: # Throttle the logs. Only print the first time the command fails + logging.error( + '%s | %s - Failed to Get supported log pages from Discovery Controller. %s', + self.id, + self.device, + err, + ) + op_obj.retry(Dc.GET_SUPPORTED_RETRY_RERIOD_SEC) + else: + logging.debug( + 'Dc._on_get_supported_fail() - %s | %s: Received event on dead object. %s', + self.id, + self.device, + err, + ) + op_obj.kill() + + # -------------------------------------------------------------------------- + def _on_get_log_success(self, op_obj: gutil.AsyncTask, data): # pylint: disable=unused-argument + '''@brief Function called when we successfully retrieve the log pages + from the Discovery Controller. See self._get_log_op object + for details. + ''' + if self._alive(): + # Note that for historical reasons too long to explain, the CDC may + # return invalid addresses ('0.0.0.0', '::', or ''). Those need to + # be filtered out. + referrals_before = self.referrals() + self._log_pages = ( + [ + {k.strip(): str(v).strip() for k, v in dictionary.items()} + for dictionary in data + if dictionary.get('traddr', '').strip() not in ('0.0.0.0', '::', '') + ] + if data + else list() + ) + logging.info( + '%s | %s - Received discovery log pages (num records=%s).', self.id, self.device, len(self._log_pages) + ) + referrals_after = self.referrals() + self._serv.log_pages_changed(self, self.device) + if referrals_after != referrals_before: + logging.debug( + 'Dc._on_get_log_success() - %s | %s: Referrals before = %s', + self.id, + self.device, + referrals_before, + ) + logging.debug( + 'Dc._on_get_log_success() - %s | %s: Referrals after = %s', + self.id, + self.device, + referrals_after, + ) + self._serv.referrals_changed() + else: + logging.debug( + 'Dc._on_get_log_success() - %s | %s: Received event on dead object.', self.id, self.device + ) + + def _on_get_log_fail(self, op_obj: gutil.AsyncTask, err, fail_cnt): + '''@brief Function called when we fail to retrieve the log pages + from the Discovery Controller. See self._get_log_op object + for details. + ''' + if self._alive(): + logging.debug( + 'Dc._on_get_log_fail() - %s | %s: %s. Retry in %s sec', + self.id, + self.device, + err, + Dc.GET_LOG_PAGE_RETRY_RERIOD_SEC, + ) + if fail_cnt == 1: # Throttle the logs. Only print the first time the command fails + logging.error('%s | %s - Failed to retrieve log pages. %s', self.id, self.device, err) + op_obj.retry(Dc.GET_LOG_PAGE_RETRY_RERIOD_SEC) + else: + logging.debug( + 'Dc._on_get_log_fail() - %s | %s: Received event on dead object. %s', + self.id, + self.device, + err, + ) + op_obj.kill() + + +# ****************************************************************************** +class Ioc(Controller): + '''@brief This object establishes a connection to one I/O Controller.''' + + def __init__(self, stac, tid: trid.TID): + self._dlpe = None + super().__init__(tid, stac) + + def _find_existing_connection(self): + return self._udev.find_nvme_ioc_device(self.tid) + + def _on_aen(self, aen: int): + pass + + def _on_nvme_event(self, nvme_event): + pass + + def reload_hdlr(self): + '''@brief This is called when a "reload" signal is received.''' + if not self.connected() and self._retry_connect_tmr.time_remaining() == 0: + self._try_to_connect_deferred.schedule() + + @property + def eflags(self): + '''@brief Return the eflag field of the DLPE''' + return get_eflags(self._dlpe) + + @property + def ncc(self): + '''@brief Return Not Connected to CDC status''' + return get_ncc(self.eflags) + + def details(self) -> dict: + '''@brief return detailed debug info about this controller''' + details = super().details() + details['dlpe'] = str(self._dlpe) + details['dlpe.eflags.ncc'] = str(self.ncc) + return details + + def update_dlpe(self, dlpe): + '''@brief This method is called when a new DLPE associated + with this controller is received.''' + new_ncc = get_ncc(get_eflags(dlpe)) + old_ncc = self.ncc + self._dlpe = dlpe + + if old_ncc and not new_ncc: # NCC bit cleared? + if not self.connected(): + self._connect_attempts = 0 + self._try_to_connect_deferred.schedule() + + def _should_try_to_reconnect(self): + '''@brief This is used to determine when it's time to stop trying toi connect''' + max_connect_attempts = conf.SvcConf().connect_attempts_on_ncc if self.ncc else 0 + return max_connect_attempts == 0 or self._connect_attempts < max_connect_attempts diff --git a/staslib/defs.py b/staslib/defs.py new file mode 100644 index 0000000..5a50371 --- /dev/null +++ b/staslib/defs.py @@ -0,0 +1,51 @@ +# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). +# +# Authors: Martin Belanger <Martin.Belanger@dell.com> + +''' @brief This file gets automagically configured by meson at build time. +''' +import os +import sys +import shutil +import platform +from staslib.version import KernelVersion + +try: + import libnvme + + LIBNVME_VERSION = libnvme.__version__ +except (AttributeError, ModuleNotFoundError): + LIBNVME_VERSION = '?.?' + +VERSION = '@VERSION@' +LICENSE = '@LICENSE@' + +STACD_DBUS_NAME = '@STACD_DBUS_NAME@' +STACD_DBUS_PATH = '@STACD_DBUS_PATH@' + +STAFD_DBUS_NAME = '@STAFD_DBUS_NAME@' +STAFD_DBUS_PATH = '@STAFD_DBUS_PATH@' + +KERNEL_VERSION = KernelVersion(platform.release()) +KERNEL_IFACE_MIN_VERSION = KernelVersion('5.14') +KERNEL_TP8013_MIN_VERSION = KernelVersion('5.16') +KERNEL_HOSTKEY_MIN_VERSION = KernelVersion('5.20') +KERNEL_CTRLKEY_MIN_VERSION = KernelVersion('5.20') + +WELL_KNOWN_DISC_NQN = 'nqn.2014-08.org.nvmexpress.discovery' + +PROG_NAME = os.path.basename(sys.argv[0]) + +NVME_HOSTID = '/etc/nvme/hostid' +NVME_HOSTNQN = '/etc/nvme/hostnqn' +NVME_HOSTKEY = '/etc/nvme/hostkey' + +SYS_CONF_FILE = '/etc/stas/sys.conf' +STAFD_CONF_FILE = '/etc/stas/stafd.conf' +STACD_CONF_FILE = '/etc/stas/stacd.conf' + +SYSTEMCTL = shutil.which('systemctl') diff --git a/staslib/gutil.py b/staslib/gutil.py new file mode 100644 index 0000000..c40b80e --- /dev/null +++ b/staslib/gutil.py @@ -0,0 +1,418 @@ +# Copyright (c) 2022, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). +# +# Authors: Martin Belanger <Martin.Belanger@dell.com> +# +'''This module provides utility functions/classes to provide easier to use +access to GLib/Gio/Gobject resources. +''' + +import logging +from gi.repository import Gio, GLib, GObject +from staslib import conf, iputil, trid + + +# ****************************************************************************** +class GTimer: + '''@brief Convenience class to wrap GLib timers''' + + def __init__( + self, interval_sec: float = 0, user_cback=lambda: GLib.SOURCE_REMOVE, *user_data, priority=GLib.PRIORITY_DEFAULT + ): # pylint: disable=keyword-arg-before-vararg + self._source = None + self._interval_sec = float(interval_sec) + self._user_cback = user_cback + self._user_data = user_data + self._priority = priority if priority is not None else GLib.PRIORITY_DEFAULT + + def _release_resources(self): + self.stop() + self._user_cback = None + self._user_data = None + + def kill(self): + '''@brief Used to release all resources associated with a timer.''' + self._release_resources() + + def __str__(self): + if self._source is not None: + return f'{self._interval_sec}s [{self.time_remaining()}s]' + + return f'{self._interval_sec}s [off]' + + def _callback(self, *_): + retval = self._user_cback(*self._user_data) + if retval == GLib.SOURCE_REMOVE: + self._source = None + return retval + + def stop(self): + '''@brief Stop timer''' + if self._source is not None: + self._source.destroy() + self._source = None + + def start(self, new_interval_sec: float = -1.0): + '''@brief Start (or restart) timer''' + if new_interval_sec >= 0: + self._interval_sec = float(new_interval_sec) + + if self._source is not None: + self._source.set_ready_time( + self._source.get_time() + (self._interval_sec * 1000000) + ) # ready time is in micro-seconds (monotonic time) + else: + if self._interval_sec.is_integer(): + self._source = GLib.timeout_source_new_seconds(int(self._interval_sec)) # seconds resolution + else: + self._source = GLib.timeout_source_new(self._interval_sec * 1000.0) # mili-seconds resolution + + self._source.set_priority(self._priority) + self._source.set_callback(self._callback) + self._source.attach() + + def clear(self): + '''@brief Make timer expire now. The callback function + will be invoked immediately by the main loop. + ''' + if self._source is not None: + self._source.set_ready_time(0) # Expire now! + + def set_callback(self, user_cback, *user_data): + '''@brief set the callback function to invoke when timer expires''' + self._user_cback = user_cback + self._user_data = user_data + + def set_timeout(self, new_interval_sec: float): + '''@brief set the timer's duration''' + if new_interval_sec >= 0: + self._interval_sec = float(new_interval_sec) + + def get_timeout(self): + '''@brief get the timer's duration''' + return self._interval_sec + + def time_remaining(self) -> float: + '''@brief Get how much time remains on a timer before it fires.''' + if self._source is not None: + delta_us = self._source.get_ready_time() - self._source.get_time() # monotonic time in micro-seconds + if delta_us > 0: + return delta_us / 1000000.0 + + return 0 + + +# ****************************************************************************** +class NameResolver: # pylint: disable=too-few-public-methods + '''@brief DNS resolver to convert host names to IP addresses.''' + + def __init__(self): + self._resolver = Gio.Resolver.get_default() + + def resolve_ctrl_async(self, cancellable, controllers_in: list, callback): + '''@brief The traddr fields may specify a hostname instead of an IP + address. We need to resolve all the host names to addresses. + Resolving hostnames may take a while as a DNS server may need + to be contacted. For that reason, we're using async APIs with + callbacks to resolve all the hostnames. + + The callback @callback will be called once all hostnames have + been resolved. + + @param controllers: List of trid.TID + ''' + pending_resolution_count = 0 + controllers_out = [] + service_conf = conf.SvcConf() + + def addr_resolved(resolver, result, controller): + try: + addresses = resolver.lookup_by_name_finish(result) # List of Gio.InetAddress objects + + except GLib.GError as err: + # We don't need to report "cancellation" errors. + if err.matches(Gio.io_error_quark(), Gio.IOErrorEnum.CANCELLED): + # pylint: disable=no-member + logging.debug('NameResolver.resolve_ctrl_async() - %s %s', err.message, controller) + else: + logging.error('%s', err.message) # pylint: disable=no-member + + # if err.matches(Gio.resolver_error_quark(), Gio.ResolverError.TEMPORARY_FAILURE): + # elif err.matches(Gio.resolver_error_quark(), Gio.ResolverError.NOT_FOUND): + # elif err.matches(Gio.resolver_error_quark(), Gio.ResolverError.INTERNAL): + + else: + traddr = None + + # If multiple addresses are returned (which is often the case), + # prefer IPv4 addresses over IPv6. + if 4 in service_conf.ip_family: + for address in addresses: + # There may be multiple IPv4 addresses. Pick 1st one. + if address.get_family() == Gio.SocketFamily.IPV4: + traddr = address.to_string() + break + + if traddr is None and 6 in service_conf.ip_family: + for address in addresses: + # There may be multiple IPv6 addresses. Pick 1st one. + if address.get_family() == Gio.SocketFamily.IPV6: + traddr = address.to_string() + break + + if traddr is not None: + logging.debug( + 'NameResolver.resolve_ctrl_async() - resolved \'%s\' -> %s', controller.traddr, traddr + ) + cid = controller.as_dict() + cid['traddr'] = traddr + nonlocal controllers_out + controllers_out.append(trid.TID(cid)) + + # Invoke callback after all hostnames have been resolved + nonlocal pending_resolution_count + pending_resolution_count -= 1 + if pending_resolution_count == 0: + callback(controllers_out) + + for controller in controllers_in: + if controller.transport in ('tcp', 'rdma'): + hostname_or_addr = controller.traddr + if not hostname_or_addr: + logging.error('Invalid traddr: %s', controller) + else: + # Try to convert to an ipaddress object. If this + # succeeds, then we don't need to call the resolver. + ip = iputil.get_ipaddress_obj(hostname_or_addr) + if ip is None: + logging.debug('NameResolver.resolve_ctrl_async() - resolving \'%s\'', hostname_or_addr) + pending_resolution_count += 1 + self._resolver.lookup_by_name_async(hostname_or_addr, cancellable, addr_resolved, controller) + elif ip.version in service_conf.ip_family: + controllers_out.append(controller) + else: + logging.warning( + 'Excluding configured IP address %s based on "ip-family" setting', hostname_or_addr + ) + else: + controllers_out.append(controller) + + if pending_resolution_count == 0: # No names are pending asynchronous resolution + callback(controllers_out) + + +# ****************************************************************************** +class _TaskRunner(GObject.Object): + '''@brief This class allows running methods asynchronously in a thread.''' + + def __init__(self, user_function, *user_args): + '''@param user_function: function to run inside a thread + @param user_args: arguments passed to @user_function + ''' + super().__init__() + self._user_function = user_function + self._user_args = user_args + + def communicate(self, cancellable, cb_function, *cb_args): + '''@param cancellable: A Gio.Cancellable object that can be used to + cancel an in-flight async command. + @param cb_function: User callback function to call when the async + command has completed. The callback function + will be passed these arguments: + + (runner, result, *cb_args) + + Where: + runner: This _TaskRunner object instance + result: A GObject.Object instance that contains the result + cb_args: The cb_args arguments passed to communicate() + + @param cb_args: User arguments to pass to @cb_function + ''' + + def in_thread_exec(task, self, task_data, cancellable): # pylint: disable=unused-argument + if task.return_error_if_cancelled(): + return # Bail out if task has been cancelled + + try: + value = GObject.Object() + value.result = self._user_function(*self._user_args) + task.return_value(value) + except Exception as ex: # pylint: disable=broad-except + task.return_error(GLib.Error(message=str(ex), domain=type(ex).__name__)) + + task = Gio.Task.new(self, cancellable, cb_function, *cb_args) + task.set_return_on_cancel(False) + task.run_in_thread(in_thread_exec) + return task + + def communicate_finish(self, result): # pylint: disable=no-self-use + '''@brief Use this function in your callback (see @cb_function) to + extract data from the result object. + + @return On success (True, data, None), + On failure (False, None, err: GLib.Error) + ''' + try: + success, value = result.propagate_value() + return success, value.result, None + except GLib.Error as err: + return False, None, err + + +# ****************************************************************************** +class AsyncTask: # pylint: disable=too-many-instance-attributes + '''Object used to manage an asynchronous GLib operation. The operation + can be cancelled or retried. + ''' + + def __init__(self, on_success_callback, on_failure_callback, operation, *op_args): + '''@param on_success_callback: Callback method invoked when @operation completes successfully + @param on_failure_callback: Callback method invoked when @operation fails + @param operation: Operation (i.e. a function) to execute asynchronously + @param op_args: Arguments passed to operation + ''' + self._cancellable = Gio.Cancellable() + self._operation = operation + self._op_args = op_args + self._success_cb = on_success_callback + self._fail_cb = on_failure_callback + self._retry_tmr = None + self._errmsg = None + self._task = None + self._fail_cnt = 0 + + def _release_resources(self): + if self._alive(): + self._cancellable.cancel() + + if self._retry_tmr is not None: + self._retry_tmr.kill() + + self._operation = None + self._op_args = None + self._success_cb = None + self._fail_cb = None + self._retry_tmr = None + self._errmsg = None + self._task = None + self._fail_cnt = None + self._cancellable = None + + def __str__(self): + return str(self.as_dict()) + + def as_dict(self): + '''Return object members as a dictionary''' + info = { + 'fail count': self._fail_cnt, + 'completed': self._task.get_completed(), + 'alive': self._alive(), + } + + if self._retry_tmr: + info['retry timer'] = str(self._retry_tmr) + + if self._errmsg: + info['error'] = self._errmsg + + return info + + def _alive(self): + return self._cancellable and not self._cancellable.is_cancelled() + + def completed(self): + '''@brief Returns True if the task has completed, False otherwise.''' + return self._task is not None and self._task.get_completed() + + def cancel(self): + '''@brief cancel async operation''' + if self._alive(): + self._cancellable.cancel() + + def kill(self): + '''@brief kill and clean up this object''' + self._release_resources() + + def run_async(self, *args): + '''@brief + Method used to initiate an asynchronous operation with the + Controller. When the operation completes (or fails) the + callback method @_on_operation_complete() will be invoked. + ''' + runner = _TaskRunner(self._operation, *self._op_args) + self._task = runner.communicate(self._cancellable, self._on_operation_complete, *args) + + def retry(self, interval_sec, *args): + '''@brief Tell this object that the async operation is to be retried + in @interval_sec seconds. + + ''' + if self._retry_tmr is None: + self._retry_tmr = GTimer() + self._retry_tmr.set_callback(self._on_retry_timeout, *args) + self._retry_tmr.start(interval_sec) + + def _on_retry_timeout(self, *args): + '''@brief + When an operation fails, the application has the option to + retry at a later time by calling the retry() method. The + retry() method starts a timer at the end of which the operation + will be executed again. This is the method that is called when + the timer expires. + ''' + if self._alive(): + self.run_async(*args) + return GLib.SOURCE_REMOVE + + def _on_operation_complete(self, runner, result, *args): + '''@brief + This callback method is invoked when the operation with the + Controller has completed (be it successful or not). + ''' + # The operation might have been cancelled. + # Only proceed if it hasn't been cancelled. + if self._operation is None or not self._alive(): + return + + success, data, err = runner.communicate_finish(result) + + if success: + self._errmsg = None + self._fail_cnt = 0 + self._success_cb(self, data, *args) + else: + self._errmsg = str(err) + self._fail_cnt += 1 + self._fail_cb(self, err, self._fail_cnt, *args) + + +# ****************************************************************************** +class Deferred: + '''Implement a deferred function call. A deferred is a function that gets + added to the main loop to be executed during the next idle slot.''' + + def __init__(self, func, *user_data): + self._source = None + self._func = func + self._user_data = user_data + + def schedule(self): + '''Schedule the function to be called by the main loop. If the + function is already scheduled, then do nothing''' + if not self.is_scheduled(): + srce_id = GLib.idle_add(self._func, *self._user_data) + self._source = GLib.main_context_default().find_source_by_id(srce_id) + + def is_scheduled(self): + '''Check if deferred is currently schedules to run''' + return self._source and not self._source.is_destroyed() + + def cancel(self): + '''Remove deferred from main loop''' + if self.is_scheduled(): + self._source.destroy() + self._source = None diff --git a/staslib/iputil.py b/staslib/iputil.py new file mode 100644 index 0000000..9199a49 --- /dev/null +++ b/staslib/iputil.py @@ -0,0 +1,169 @@ +# Copyright (c) 2022, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). +# +# Authors: Martin Belanger <Martin.Belanger@dell.com> + +'''A collection of IP address and network interface utilities''' + +import socket +import logging +import ipaddress +from staslib import conf + +RTM_NEWADDR = 20 +RTM_GETADDR = 22 +NLM_F_REQUEST = 0x01 +NLM_F_ROOT = 0x100 +NLMSG_DONE = 3 +IFLA_ADDRESS = 1 +NLMSGHDR_SZ = 16 +IFADDRMSG_SZ = 8 +RTATTR_SZ = 4 + +# Netlink request (Get address command) +GETADDRCMD = ( + # BEGIN: struct nlmsghdr + b'\0' * 4 # nlmsg_len (placeholder - actual length calculated below) + + (RTM_GETADDR).to_bytes(2, byteorder='little', signed=False) # nlmsg_type + + (NLM_F_REQUEST | NLM_F_ROOT).to_bytes(2, byteorder='little', signed=False) # nlmsg_flags + + b'\0' * 2 # nlmsg_seq + + b'\0' * 2 # nlmsg_pid + # END: struct nlmsghdr + + b'\0' * 8 # struct ifaddrmsg +) +GETADDRCMD = len(GETADDRCMD).to_bytes(4, byteorder='little') + GETADDRCMD[4:] # nlmsg_len + + +# ****************************************************************************** +def get_ipaddress_obj(ipaddr): + '''@brief Return a IPv4Address or IPv6Address depending on whether @ipaddr + is a valid IPv4 or IPv6 address. Return None otherwise.''' + try: + ip = ipaddress.ip_address(ipaddr) + except ValueError: + return None + + return ip + + +# ****************************************************************************** +def _data_matches_ip(data_family, data, ip): + if data_family == socket.AF_INET: + try: + other_ip = ipaddress.IPv4Address(data) + except ValueError: + return False + if ip.version == 6: + ip = ip.ipv4_mapped + elif data_family == socket.AF_INET6: + try: + other_ip = ipaddress.IPv6Address(data) + except ValueError: + return False + if ip.version == 4: + other_ip = other_ip.ipv4_mapped + else: + return False + + return other_ip == ip + + +# ****************************************************************************** +def iface_of(src_addr): + '''@brief Find the interface that has src_addr as one of its assigned IP addresses. + @param src_addr: The IP address to match + @type src_addr: Instance of ipaddress.IPv4Address or ipaddress.IPv6Address + ''' + with socket.socket(socket.AF_NETLINK, socket.SOCK_RAW) as sock: + sock.sendall(GETADDRCMD) + nlmsg = sock.recv(8192) + nlmsg_idx = 0 + while True: + if nlmsg_idx >= len(nlmsg): + nlmsg += sock.recv(8192) + + nlmsg_type = int.from_bytes(nlmsg[nlmsg_idx + 4 : nlmsg_idx + 6], byteorder='little', signed=False) + if nlmsg_type == NLMSG_DONE: + break + + if nlmsg_type != RTM_NEWADDR: + break + + nlmsg_len = int.from_bytes(nlmsg[nlmsg_idx : nlmsg_idx + 4], byteorder='little', signed=False) + if nlmsg_len % 4: # Is msg length not a multiple of 4? + break + + ifaddrmsg_indx = nlmsg_idx + NLMSGHDR_SZ + ifa_family = nlmsg[ifaddrmsg_indx] + ifa_index = int.from_bytes(nlmsg[ifaddrmsg_indx + 4 : ifaddrmsg_indx + 8], byteorder='little', signed=False) + + rtattr_indx = ifaddrmsg_indx + IFADDRMSG_SZ + while rtattr_indx < (nlmsg_idx + nlmsg_len): + rta_len = int.from_bytes(nlmsg[rtattr_indx : rtattr_indx + 2], byteorder='little', signed=False) + rta_type = int.from_bytes(nlmsg[rtattr_indx + 2 : rtattr_indx + 4], byteorder='little', signed=False) + if rta_type == IFLA_ADDRESS: + data = nlmsg[rtattr_indx + RTATTR_SZ : rtattr_indx + rta_len] + if _data_matches_ip(ifa_family, data, src_addr): + return socket.if_indextoname(ifa_index) + + rta_len = (rta_len + 3) & ~3 # Round up to multiple of 4 + rtattr_indx += rta_len # Move to next rtattr + + nlmsg_idx += nlmsg_len # Move to next Netlink message + + return '' + + +# ****************************************************************************** +def get_interface(src_addr): + '''Get interface for given source address + @param src_addr: The source address + @type src_addr: str + ''' + if not src_addr: + return '' + + src_addr = src_addr.split('%')[0] # remove scope-id (if any) + src_addr = get_ipaddress_obj(src_addr) + return '' if src_addr is None else iface_of(src_addr) + + +# ****************************************************************************** +def remove_invalid_addresses(controllers: list): + '''@brief Remove controllers with invalid addresses from the list of controllers. + @param controllers: List of TIDs + ''' + service_conf = conf.SvcConf() + valid_controllers = list() + for controller in controllers: + if controller.transport in ('tcp', 'rdma'): + # Let's make sure that traddr is + # syntactically a valid IPv4 or IPv6 address. + ip = get_ipaddress_obj(controller.traddr) + if ip is None: + logging.warning('%s IP address is not valid', controller) + continue + + # Let's make sure the address family is enabled. + if ip.version not in service_conf.ip_family: + logging.debug( + '%s ignored because IPv%s is disabled in %s', + controller, + ip.version, + service_conf.conf_file, + ) + continue + + valid_controllers.append(controller) + + elif controller.transport in ('fc', 'loop'): + # At some point, need to validate FC addresses as well... + valid_controllers.append(controller) + + else: + logging.warning('Invalid transport %s', controller.transport) + + return valid_controllers diff --git a/staslib/log.py b/staslib/log.py new file mode 100644 index 0000000..9622e98 --- /dev/null +++ b/staslib/log.py @@ -0,0 +1,53 @@ +# Copyright (c) 2022, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). +# +# Authors: Martin Belanger <Martin.Belanger@dell.com> +# +'''nvme-stas logging module''' + +import sys +import logging +from staslib import defs + + +def init(syslog: bool): + '''Init log module + @param syslog: True to send messages to the syslog, + False to send messages to stdout. + ''' + log = logging.getLogger() + log.propagate = False + + if syslog: + try: + # Try journal logger first + import systemd.journal # pylint: disable=import-outside-toplevel + + handler = systemd.journal.JournalHandler(SYSLOG_IDENTIFIER=defs.PROG_NAME) + except ModuleNotFoundError: + # Go back to standard syslog handler + from logging.handlers import SysLogHandler # pylint: disable=import-outside-toplevel + + handler = SysLogHandler(address="/dev/log") + handler.setFormatter(logging.Formatter(f'{defs.PROG_NAME}: %(message)s')) + else: + # Log to stdout + handler = logging.StreamHandler(stream=sys.stdout) + + log.addHandler(handler) + log.setLevel(logging.INFO if syslog else logging.DEBUG) + + +def level() -> str: + '''@brief return current log level''' + logger = logging.getLogger() + return str(logging.getLevelName(logger.getEffectiveLevel())) + + +def set_level_from_tron(tron): + '''Set log level based on TRON''' + logger = logging.getLogger() + logger.setLevel(logging.DEBUG if tron else logging.INFO) diff --git a/staslib/meson.build b/staslib/meson.build new file mode 100644 index 0000000..eb006f0 --- /dev/null +++ b/staslib/meson.build @@ -0,0 +1,60 @@ +# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). +# +# Authors: Martin Belanger <Martin.Belanger@dell.com> +# + +files_to_configure = [ 'defs.py', '__init__.py', 'stafd.idl', 'stacd.idl' ] +configured_files = [] +foreach file : files_to_configure + configured_files += configure_file( + input: file, + output: file, + configuration: conf + ) +endforeach + +files_to_copy = [ + 'avahi.py', + 'conf.py', + 'ctrl.py', + 'gutil.py', + 'iputil.py', + 'log.py', + 'service.py', + 'singleton.py', + 'stas.py', + 'timeparse.py', + 'trid.py', + 'udev.py', + 'version.py' +] +copied_files = [] +foreach file : files_to_copy + copied_files += configure_file( + input: file, + output: file, + copy: true, + ) +endforeach + +files_to_install = copied_files + configured_files +python3.install_sources( + files_to_install, + pure: true, + subdir: 'staslib', +) + +#=============================================================================== +# Make a list of modules to lint +skip = ['stafd.idl', 'stacd.idl'] +foreach file: files_to_install + fname = fs.name('@0@'.format(file)) + if fname not in skip + modules_to_lint += file + endif +endforeach + diff --git a/staslib/service.py b/staslib/service.py new file mode 100644 index 0000000..ce4769d --- /dev/null +++ b/staslib/service.py @@ -0,0 +1,878 @@ +# Copyright (c) 2022, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). +# +# Authors: Martin Belanger <Martin.Belanger@dell.com> +# +'''This module defines the base Service object from +which the Staf and the Stac objects are derived.''' + +import json +import logging +import pathlib +import subprocess +from itertools import filterfalse +import dasbus.error +import dasbus.client.observer +import dasbus.client.proxy + +from gi.repository import GLib +from systemd.daemon import notify as sd_notify +from staslib import avahi, conf, ctrl, defs, gutil, iputil, stas, timeparse, trid, udev + + +# ****************************************************************************** +class CtrlTerminator: + '''The Controller Terminator is used to gracefully disconnect from + controllers. All communications with controllers is handled by the kernel. + Once we make a request to the kernel to perform an operation (e.g. connect), + we have to wait for it to complete before requesting another operation. This + is particularly important when we want to disconnect from a controller while + there are pending operations, especially a pending connect. + + The "connect" operation is especially unpredictable because all connect + requests are made through the blocking interface "/dev/nvme-fabrics". This + means that once a "connect" operation has been submitted, and depending on + how many connect requests are made concurrently, it can take several seconds + for a connect to be processed by the kernel. + + While connect or other operations are being performed, it is possible + that a disconnect may be requested (e.g. someone or something changes the + configuration to remove a controller). Because it is not possible to + terminate a pending operation request, we have to wait for it to complete + before we can issue a disconnect. Failure to do that will result in + operations being performed by the kernel in reverse order. For example, + a disconnect may be executed before a pending connect has had a chance to + complete. And this will result in controllers that are supposed to be + disconnected to be connected without nvme-stas knowing about it. + + The Controller Terminator is used when we need to disconnect from a + controller. It will make sure that there are no pending operations before + issuing a disconnect. + ''' + + DISPOSAL_AUDIT_PERIOD_SEC = 30 + + def __init__(self): + self._udev = udev.UDEV + self._controllers = list() # The list of controllers to dispose of. + self._audit_tmr = gutil.GTimer(self.DISPOSAL_AUDIT_PERIOD_SEC, self._on_disposal_check) + + def dispose(self, controller: ctrl.Controller, on_controller_removed_cb, keep_connection: bool): + '''Invoked by a service (stafd or stacd) to dispose of a controller''' + if controller.all_ops_completed(): + logging.debug( + 'CtrlTerminator.dispose() - %s | %s: Invoke disconnect()', controller.tid, controller.device + ) + controller.disconnect(on_controller_removed_cb, keep_connection) + else: + logging.debug( + 'CtrlTerminator.dispose() - %s | %s: Add controller to garbage disposal', + controller.tid, + controller.device, + ) + self._controllers.append((controller, keep_connection, on_controller_removed_cb, controller.tid)) + + self._udev.register_for_action_events('add', self._on_kernel_events) + self._udev.register_for_action_events('remove', self._on_kernel_events) + + if self._audit_tmr.time_remaining() == 0: + self._audit_tmr.start() + + def pending_disposal(self, tid): + '''Check whether @tid is pending disposal''' + for controller in self._controllers: + if controller.tid == tid: + return True + return False + + def info(self): + '''@brief Get info about this object (used for debug)''' + info = { + 'terminator.audit timer': str(self._audit_tmr), + } + for controller, _, _, tid in self._controllers: + info[f'terminator.controller.{tid}'] = str(controller.info()) + return info + + def kill(self): + '''Stop Controller Terminator and release resources.''' + self._audit_tmr.stop() + self._audit_tmr = None + + if self._udev: + self._udev.unregister_for_action_events('add', self._on_kernel_events) + self._udev.unregister_for_action_events('remove', self._on_kernel_events) + self._udev = None + + for controller, keep_connection, on_controller_removed_cb, _ in self._controllers: + controller.disconnect(on_controller_removed_cb, keep_connection) + + self._controllers.clear() + + def _on_kernel_events(self, udev_obj): + logging.debug('CtrlTerminator._on_kernel_events() - %s event received', udev_obj.action) + self._disposal_check() + + def _on_disposal_check(self, *_user_data): + logging.debug('CtrlTerminator._on_disposal_check()- Periodic audit') + return GLib.SOURCE_REMOVE if self._disposal_check() else GLib.SOURCE_CONTINUE + + @staticmethod + def _keep_or_terminate(args): + '''Return False if controller is to be kept. True if controller + was terminated and can be removed from the list.''' + controller, keep_connection, on_controller_removed_cb, tid = args + if controller.all_ops_completed(): + logging.debug( + 'CtrlTerminator._keep_or_terminate()- %s | %s: Disconnecting controller', + tid, + controller.device, + ) + controller.disconnect(on_controller_removed_cb, keep_connection) + return True + + return False + + def _disposal_check(self): + # Iterate over the list, terminating (disconnecting) those controllers + # that have no pending operations, and remove those controllers from the + # list (only keep controllers that still have operations pending). + self._controllers[:] = filterfalse(self._keep_or_terminate, self._controllers) + disposal_complete = len(self._controllers) == 0 + + if disposal_complete: + logging.debug('CtrlTerminator._disposal_check() - Disposal complete') + self._audit_tmr.stop() + self._udev.unregister_for_action_events('add', self._on_kernel_events) + self._udev.unregister_for_action_events('remove', self._on_kernel_events) + else: + self._audit_tmr.start() # Restart timer + + return disposal_complete + + +# ****************************************************************************** +class Service(stas.ServiceABC): + '''@brief Base class used to manage a STorage Appliance Service''' + + def __init__(self, args, default_conf, reload_hdlr): + self._udev = udev.UDEV + self._terminator = CtrlTerminator() + + super().__init__(args, default_conf, reload_hdlr) + + def _release_resources(self): + logging.debug('Service._release_resources()') + super()._release_resources() + + if self._terminator: + self._terminator.kill() + + self._udev = None + self._terminator = None + + def _disconnect_all(self): + '''Tell all controller objects to disconnect''' + keep_connections = self._keep_connections_on_exit() + controllers = self._controllers.values() + logging.debug( + 'Service._stop_hdlr() - Controller count = %s, keep_connections = %s', + len(controllers), + keep_connections, + ) + for controller in controllers: + self._terminator.dispose(controller, self._on_final_disconnect, keep_connections) + + def info(self) -> dict: + '''@brief Get the status info for this object (used for debug)''' + info = super().info() + if self._terminator: + info.update(self._terminator.info()) + return info + + @stas.ServiceABC.tron.setter + def tron(self, value): + '''@brief Set Trace ON property''' + super(__class__, self.__class__).tron.__set__(self, value) + + +# ****************************************************************************** +class Stac(Service): + '''STorage Appliance Connector (STAC)''' + + CONF_STABILITY_LONG_SOAK_TIME_SEC = 10 # pylint: disable=invalid-name + ADD_EVENT_SOAK_TIME_SEC = 1 + + def __init__(self, args, dbus): + default_conf = { + ('Global', 'tron'): False, + ('Global', 'hdr-digest'): False, + ('Global', 'data-digest'): False, + ('Global', 'kato'): None, # None to let the driver decide the default + ('Global', 'nr-io-queues'): None, # None to let the driver decide the default + ('Global', 'nr-write-queues'): None, # None to let the driver decide the default + ('Global', 'nr-poll-queues'): None, # None to let the driver decide the default + ('Global', 'queue-size'): None, # None to let the driver decide the default + ('Global', 'reconnect-delay'): None, # None to let the driver decide the default + ('Global', 'ctrl-loss-tmo'): None, # None to let the driver decide the default + ('Global', 'disable-sqflow'): None, # None to let the driver decide the default + ('Global', 'ignore-iface'): False, + ('Global', 'ip-family'): (4, 6), + ('Controllers', 'controller'): list(), + ('Controllers', 'exclude'): list(), + ('I/O controller connection management', 'disconnect-scope'): 'only-stas-connections', + ('I/O controller connection management', 'disconnect-trtypes'): ['tcp'], + ('I/O controller connection management', 'connect-attempts-on-ncc'): 0, + } + + super().__init__(args, default_conf, self._reload_hdlr) + + self._add_event_soak_tmr = gutil.GTimer(self.ADD_EVENT_SOAK_TIME_SEC, self._on_add_event_soaked) + + self._config_connections_audit() + + # Create the D-Bus instance. + self._config_dbus(dbus, defs.STACD_DBUS_NAME, defs.STACD_DBUS_PATH) + + # Connect to STAF D-Bus interface + self._staf = None + self._staf_watcher = dasbus.client.observer.DBusObserver(self._sysbus, defs.STAFD_DBUS_NAME) + self._staf_watcher.service_available.connect(self._connect_to_staf) + self._staf_watcher.service_unavailable.connect(self._disconnect_from_staf) + self._staf_watcher.connect_once_available() + + def _release_resources(self): + logging.debug('Stac._release_resources()') + + if self._add_event_soak_tmr: + self._add_event_soak_tmr.kill() + + if self._udev: + self._udev.unregister_for_action_events('add', self._on_add_event) + + self._destroy_staf_comlink(self._staf_watcher) + if self._staf_watcher is not None: + self._staf_watcher.disconnect() + + super()._release_resources() + + self._staf = None + self._staf_watcher = None + self._add_event_soak_tmr = None + + def _dump_last_known_config(self, controllers): + config = list(controllers.keys()) + logging.debug('Stac._dump_last_known_config() - IOC count = %s', len(config)) + self._write_lkc(config) + + def _load_last_known_config(self): + config = self._read_lkc() or list() + logging.debug('Stac._load_last_known_config() - IOC count = %s', len(config)) + + controllers = {} + for tid in config: + # Only create Ioc objects if there is already a connection in the kernel + # First, regenerate the TID (in case of soft. upgrade and TID object + # has changed internally) + tid = trid.TID(tid.as_dict()) + if udev.UDEV.find_nvme_ioc_device(tid) is not None: + controllers[tid] = ctrl.Ioc(self, tid) + + return controllers + + def _audit_all_connections(self, tids): + '''A host should only connect to I/O controllers that have been zoned + for that host or a manual "controller" entry exists in stacd.conf. + A host should disconnect from an I/O controller when that I/O controller + is removed from the zone or a "controller" entry is manually removed + from stacd.conf. stacd will audit connections if "disconnect-scope= + all-connections-matching-disconnect-trtypes". stacd will delete any + connection that is not supposed to exist. + ''' + logging.debug('Stac._audit_all_connections() - tids = %s', tids) + num_controllers = len(self._controllers) + for tid in tids: + if tid not in self._controllers and not self._terminator.pending_disposal(tid): + self._controllers[tid] = ctrl.Ioc(self, tid) + + if num_controllers != len(self._controllers): + self._cfg_soak_tmr.start(self.CONF_STABILITY_SOAK_TIME_SEC) + + def _on_add_event(self, udev_obj): + '''@brief This function is called when a "add" event is received from + the kernel for an NVMe device. This is used to trigger an audit and make + sure that the connection to an I/O controller is allowed. + + WARNING: There is a race condition with the "add" event from the kernel. + The kernel sends the "add" event a bit early and the sysfs attributes + associated with the nvme object are not always fully initialized. + To workaround this problem we use a soaking timer to give time for the + sysfs attributes to stabilize. + ''' + logging.debug('Stac._on_add_event(() - Received "add" event: %s', udev_obj.sys_name) + self._add_event_soak_tmr.start() + + def _on_add_event_soaked(self): + '''@brief After the add event has been soaking for ADD_EVENT_SOAK_TIME_SEC + seconds, we can audit the connections. + ''' + if self._alive(): + svc_conf = conf.SvcConf() + if svc_conf.disconnect_scope == 'all-connections-matching-disconnect-trtypes': + self._audit_all_connections(self._udev.get_nvme_ioc_tids(svc_conf.disconnect_trtypes)) + return GLib.SOURCE_REMOVE + + def _config_connections_audit(self): + '''This function checks the "disconnect_scope" parameter to determine + whether audits should be performed. Audits are enabled when + "disconnect_scope == all-connections-matching-disconnect-trtypes". + ''' + svc_conf = conf.SvcConf() + if svc_conf.disconnect_scope == 'all-connections-matching-disconnect-trtypes': + if not self._udev.is_action_cback_registered('add', self._on_add_event): + self._udev.register_for_action_events('add', self._on_add_event) + self._audit_all_connections(self._udev.get_nvme_ioc_tids(svc_conf.disconnect_trtypes)) + else: + self._udev.unregister_for_action_events('add', self._on_add_event) + + def _keep_connections_on_exit(self): + '''@brief Determine whether connections should remain when the + process exits. + ''' + return True + + def _reload_hdlr(self): + '''@brief Reload configuration file. This is triggered by the SIGHUP + signal, which can be sent with "systemctl reload stacd". + ''' + if not self._alive(): + return GLib.SOURCE_REMOVE + + sd_notify('RELOADING=1') + service_cnf = conf.SvcConf() + service_cnf.reload() + self.tron = service_cnf.tron + self._config_connections_audit() + self._cfg_soak_tmr.start(self.CONF_STABILITY_SOAK_TIME_SEC) + + for controller in self._controllers.values(): + controller.reload_hdlr() + + sd_notify('READY=1') + return GLib.SOURCE_CONTINUE + + def _get_log_pages_from_stafd(self): + if self._staf: + try: + return json.loads(self._staf.get_all_log_pages(True)) + except dasbus.error.DBusError: + pass + + return list() + + def _config_ctrls_finish(self, configured_ctrl_list: list): # pylint: disable=too-many-locals + '''@param configured_ctrl_list: list of TIDs''' + # This is a callback function, which may be called after the service + # has been signalled to stop. So let's make sure the service is still + # alive and well before continuing. + if not self._alive(): + logging.debug('Stac._config_ctrls_finish() - Exiting because service is no longer alive') + return + + # Eliminate invalid entries from stacd.conf "controller list". + configured_ctrl_list = [ + tid for tid in configured_ctrl_list if '' not in (tid.transport, tid.traddr, tid.trsvcid, tid.subsysnqn) + ] + + logging.debug('Stac._config_ctrls_finish() - configured_ctrl_list = %s', configured_ctrl_list) + + discovered_ctrls = dict() + for staf_data in self._get_log_pages_from_stafd(): + host_traddr = staf_data['discovery-controller']['host-traddr'] + host_iface = staf_data['discovery-controller']['host-iface'] + for dlpe in staf_data['log-pages']: + if dlpe.get('subtype') == 'nvme': # eliminate discovery controllers + tid = stas.tid_from_dlpe(dlpe, host_traddr, host_iface) + discovered_ctrls[tid] = dlpe + + discovered_ctrl_list = list(discovered_ctrls.keys()) + logging.debug('Stac._config_ctrls_finish() - discovered_ctrl_list = %s', discovered_ctrl_list) + + controllers = stas.remove_excluded(configured_ctrl_list + discovered_ctrl_list) + controllers = iputil.remove_invalid_addresses(controllers) + + new_controller_tids = set(controllers) + cur_controller_tids = set(self._controllers.keys()) + controllers_to_add = new_controller_tids - cur_controller_tids + controllers_to_del = cur_controller_tids - new_controller_tids + + logging.debug('Stac._config_ctrls_finish() - controllers_to_add = %s', list(controllers_to_add)) + logging.debug('Stac._config_ctrls_finish() - controllers_to_del = %s', list(controllers_to_del)) + + svc_conf = conf.SvcConf() + no_disconnect = svc_conf.disconnect_scope == 'no-disconnect' + match_trtypes = svc_conf.disconnect_scope == 'all-connections-matching-disconnect-trtypes' + logging.debug( + 'Stac._config_ctrls_finish() - no_disconnect=%s, match_trtypes=%s, svc_conf.disconnect_trtypes=%s', + no_disconnect, + match_trtypes, + svc_conf.disconnect_trtypes, + ) + for tid in controllers_to_del: + controller = self._controllers.pop(tid, None) + if controller is not None: + keep_connection = no_disconnect or (match_trtypes and tid.transport not in svc_conf.disconnect_trtypes) + self._terminator.dispose(controller, self.remove_controller, keep_connection) + + for tid in controllers_to_add: + self._controllers[tid] = ctrl.Ioc(self, tid) + + for tid, controller in self._controllers.items(): + if tid in discovered_ctrls: + dlpe = discovered_ctrls[tid] + controller.update_dlpe(dlpe) + + self._dump_last_known_config(self._controllers) + + def _connect_to_staf(self, _): + '''@brief Hook up DBus signal handlers for signals from stafd.''' + if not self._alive(): + return + + try: + self._staf = self._sysbus.get_proxy(defs.STAFD_DBUS_NAME, defs.STAFD_DBUS_PATH) + self._staf.log_pages_changed.connect(self._log_pages_changed) + self._staf.dc_removed.connect(self._dc_removed) + self._cfg_soak_tmr.start() + + # Make sure timer is set back to its normal value. + self._cfg_soak_tmr.set_timeout(self.CONF_STABILITY_SOAK_TIME_SEC) + logging.debug('Stac._connect_to_staf() - Connected to staf') + except dasbus.error.DBusError: + logging.error('Failed to connect to staf') + + def _destroy_staf_comlink(self, watcher): # pylint: disable=unused-argument + if self._staf: + self._staf.log_pages_changed.disconnect(self._log_pages_changed) + self._staf.dc_removed.disconnect(self._dc_removed) + dasbus.client.proxy.disconnect_proxy(self._staf) + self._staf = None + + def _disconnect_from_staf(self, watcher): + self._destroy_staf_comlink(watcher) + + # When we lose connectivity with stafd, the most logical explanation + # is that stafd restarted. In that case, it may take some time for stafd + # to re-populate its log pages cache. So let's give stafd plenty of time + # to update its log pages cache and send log pages change notifications + # before triggering a stacd re-config. We do this by momentarily + # increasing the config soak timer to a longer period. + if self._cfg_soak_tmr: + self._cfg_soak_tmr.set_timeout(self.CONF_STABILITY_LONG_SOAK_TIME_SEC) + + logging.debug('Stac._disconnect_from_staf() - Disconnected from staf') + + def _log_pages_changed( # pylint: disable=too-many-arguments + self, transport, traddr, trsvcid, host_traddr, host_iface, subsysnqn, device + ): + if not self._alive(): + return + + logging.debug( + 'Stac._log_pages_changed() - transport=%s, traddr=%s, trsvcid=%s, host_traddr=%s, host_iface=%s, subsysnqn=%s, device=%s', + transport, + traddr, + trsvcid, + host_traddr, + host_iface, + subsysnqn, + device, + ) + if self._cfg_soak_tmr: + self._cfg_soak_tmr.start(self.CONF_STABILITY_SOAK_TIME_SEC) + + def _dc_removed(self): + if not self._alive(): + return + + logging.debug('Stac._dc_removed()') + if self._cfg_soak_tmr: + self._cfg_soak_tmr.start(self.CONF_STABILITY_SOAK_TIME_SEC) + + +# ****************************************************************************** +# Only keep legacy FC rule (not even sure this is still in use today, but just to be safe). +UDEV_RULE_OVERRIDE = r''' +ACTION=="change", SUBSYSTEM=="fc", ENV{FC_EVENT}=="nvmediscovery", \ + ENV{NVMEFC_HOST_TRADDR}=="*", ENV{NVMEFC_TRADDR}=="*", \ + RUN+="%s --no-block start nvmf-connect@--transport=fc\t--traddr=$env{NVMEFC_TRADDR}\t--trsvcid=none\t--host-traddr=$env{NVMEFC_HOST_TRADDR}.service" +''' + + +def _udev_rule_ctrl(suppress): + '''@brief We override the standard udev rule installed by nvme-cli, i.e. + '/usr/lib/udev/rules.d/70-nvmf-autoconnect.rules', with a copy into + /run/udev/rules.d. The goal is to suppress the udev rule that controls TCP + connections to I/O controllers. This is to avoid race conditions between + stacd and udevd. This is configurable. See "udev-rule" in stacd.conf + for details. + + @param enable: When True, override nvme-cli's udev rule and prevent TCP I/O + Controller connections by nvme-cli. When False, allow nvme-cli's udev rule + to make TCP I/O connections. + @type enable: bool + ''' + udev_rule_file = pathlib.Path('/run/udev/rules.d', '70-nvmf-autoconnect.rules') + if suppress: + if not udev_rule_file.exists(): + pathlib.Path('/run/udev/rules.d').mkdir(parents=True, exist_ok=True) + text = UDEV_RULE_OVERRIDE % (defs.SYSTEMCTL) + udev_rule_file.write_text(text) # pylint: disable=unspecified-encoding + else: + try: + udev_rule_file.unlink() + except FileNotFoundError: + pass + + +def _is_dlp_changed_aen(udev_obj): + '''Check whether we received a Change of Discovery Log Page AEN''' + nvme_aen = udev_obj.get('NVME_AEN') + if not isinstance(nvme_aen, str): + return False + + aen = int(nvme_aen, 16) + if aen != ctrl.DLP_CHANGED: + return False + + logging.info( + '%s - Received AEN: Change of Discovery Log Page (%s)', + udev_obj.sys_name, + nvme_aen, + ) + return True + + +def _event_matches(udev_obj, nvme_events): + '''Check whether we received an NVMe Event matching + one of the events listed in @nvme_events''' + nvme_event = udev_obj.get('NVME_EVENT') + if nvme_event not in nvme_events: + return False + + logging.info('%s - Received "%s" event', udev_obj.sys_name, nvme_event) + return True + + +# ****************************************************************************** +class Staf(Service): + '''STorage Appliance Finder (STAF)''' + + def __init__(self, args, dbus): + default_conf = { + ('Global', 'tron'): False, + ('Global', 'hdr-digest'): False, + ('Global', 'data-digest'): False, + ('Global', 'kato'): 30, + ('Global', 'queue-size'): None, # None to let the driver decide the default + ('Global', 'reconnect-delay'): None, # None to let the driver decide the default + ('Global', 'ctrl-loss-tmo'): None, # None to let the driver decide the default + ('Global', 'disable-sqflow'): None, # None to let the driver decide the default + ('Global', 'persistent-connections'): False, # Deprecated + ('Discovery controller connection management', 'persistent-connections'): True, + ('Discovery controller connection management', 'zeroconf-connections-persistence'): timeparse.timeparse( + '72hours' + ), + ('Global', 'ignore-iface'): False, + ('Global', 'ip-family'): (4, 6), + ('Global', 'pleo'): True, + ('Service Discovery', 'zeroconf'): True, + ('Controllers', 'controller'): list(), + ('Controllers', 'exclude'): list(), + } + + super().__init__(args, default_conf, self._reload_hdlr) + + self._avahi = avahi.Avahi(self._sysbus, self._avahi_change) + self._avahi.config_stypes(conf.SvcConf().stypes) + + # Create the D-Bus instance. + self._config_dbus(dbus, defs.STAFD_DBUS_NAME, defs.STAFD_DBUS_PATH) + + self._udev.register_for_action_events('change', self._nvme_cli_interop) + _udev_rule_ctrl(True) + + def info(self) -> dict: + '''@brief Get the status info for this object (used for debug)''' + info = super().info() + info['avahi'] = self._avahi.info() + return info + + def _release_resources(self): + logging.debug('Staf._release_resources()') + if self._udev: + self._udev.unregister_for_action_events('change', self._nvme_cli_interop) + + super()._release_resources() + + _udev_rule_ctrl(False) + if self._avahi: + self._avahi.kill() + self._avahi = None + + def _dump_last_known_config(self, controllers): + config = {tid: {'log_pages': dc.log_pages(), 'origin': dc.origin} for tid, dc in controllers.items()} + logging.debug('Staf._dump_last_known_config() - DC count = %s', len(config)) + self._write_lkc(config) + + def _load_last_known_config(self): + config = self._read_lkc() or dict() + logging.debug('Staf._load_last_known_config() - DC count = %s', len(config)) + + controllers = {} + for tid, data in config.items(): + if isinstance(data, dict): + log_pages = data.get('log_pages') + origin = data.get('origin') + else: + log_pages = data + origin = None + + # Regenerate the TID (in case of soft. upgrade and TID object + # has changed internally) + tid = trid.TID(tid.as_dict()) + controllers[tid] = ctrl.Dc(self, tid, log_pages, origin) + + return controllers + + def _keep_connections_on_exit(self): + '''@brief Determine whether connections should remain when the + process exits. + ''' + return conf.SvcConf().persistent_connections + + def _reload_hdlr(self): + '''@brief Reload configuration file. This is triggered by the SIGHUP + signal, which can be sent with "systemctl reload stafd". + ''' + if not self._alive(): + return GLib.SOURCE_REMOVE + + sd_notify('RELOADING=1') + service_cnf = conf.SvcConf() + service_cnf.reload() + self.tron = service_cnf.tron + self._avahi.kick_start() # Make sure Avahi is running + self._avahi.config_stypes(service_cnf.stypes) + self._cfg_soak_tmr.start() + + for controller in self._controllers.values(): + controller.reload_hdlr() + + sd_notify('READY=1') + return GLib.SOURCE_CONTINUE + + def is_avahi_reported(self, tid): + '''@brief Return whether @tid is being reported by the Avahi daemon. + @return: True if the Avahi daemon is reporting it, False otherwise. + ''' + for cid in self._avahi.get_controllers(): + if trid.TID(cid) == tid: + return True + return False + + def log_pages_changed(self, controller, device): + '''@brief Function invoked when a controller's cached log pages + have changed. This will emit a D-Bus signal to inform + other applications that the cached log pages have changed. + ''' + self._dbus_iface.log_pages_changed.emit( + controller.tid.transport, + controller.tid.traddr, + controller.tid.trsvcid, + controller.tid.host_traddr, + controller.tid.host_iface, + controller.tid.subsysnqn, + device, + ) + + def dc_removed(self): + '''@brief Function invoked when a controller's cached log pages + have changed. This will emit a D-Bus signal to inform + other applications that the cached log pages have changed. + ''' + self._dbus_iface.dc_removed.emit() + + def _referrals(self) -> list: + return [ + stas.tid_from_dlpe(dlpe, controller.tid.host_traddr, controller.tid.host_iface) + for controller in self.get_controllers() + for dlpe in controller.referrals() + ] + + def _config_ctrls_finish(self, configured_ctrl_list: list): + '''@brief Finish discovery controllers configuration after + hostnames (if any) have been resolved. All the logic associated + with discovery controller creation/deletion is found here. To + avoid calling this algorith repetitively for each and every events, + it is called after a soaking period controlled by self._cfg_soak_tmr. + + @param configured_ctrl_list: List of TIDs configured in stafd.conf with + all hostnames resolved to their corresponding IP addresses. + ''' + # This is a callback function, which may be called after the service + # has been signalled to stop. So let's make sure the service is still + # alive and well before continuing. + if not self._alive(): + logging.debug('Staf._config_ctrls_finish() - Exiting because service is no longer alive') + return + + # Eliminate invalid entries from stafd.conf "controller list". + controllers = list() + for tid in configured_ctrl_list: + if '' in (tid.transport, tid.traddr, tid.trsvcid): + continue + if not tid.subsysnqn: + cid = tid.as_dict() + cid['subsysnqn'] = defs.WELL_KNOWN_DISC_NQN + controllers.append(trid.TID(cid)) + else: + controllers.append(tid) + configured_ctrl_list = controllers + + # Get the Avahi-discovered list and the referrals. + discovered_ctrl_list = [trid.TID(cid) for cid in self._avahi.get_controllers()] + referral_ctrl_list = self._referrals() + logging.debug('Staf._config_ctrls_finish() - configured_ctrl_list = %s', configured_ctrl_list) + logging.debug('Staf._config_ctrls_finish() - discovered_ctrl_list = %s', discovered_ctrl_list) + logging.debug('Staf._config_ctrls_finish() - referral_ctrl_list = %s', referral_ctrl_list) + + all_ctrls = configured_ctrl_list + discovered_ctrl_list + referral_ctrl_list + controllers = stas.remove_excluded(all_ctrls) + controllers = iputil.remove_invalid_addresses(controllers) + + new_controller_tids = set(controllers) + cur_controller_tids = set(self._controllers.keys()) + controllers_to_add = new_controller_tids - cur_controller_tids + controllers_to_del = cur_controller_tids - new_controller_tids + + # Make a list list of excluded and invalid controllers + must_remove_list = set(all_ctrls) - new_controller_tids + + # Find "discovered" controllers that have not responded + # in a while and add them to controllers that must be removed. + must_remove_list.update({tid for tid, controller in self._controllers.items() if controller.is_unresponsive()}) + + # Do not remove Avahi-discovered DCs from controllers_to_del unless + # marked as "must-be-removed" (must_remove_list). This is to account for + # the case where mDNS discovery is momentarily disabled (e.g. Avahi + # daemon restarts). We don't want to delete connections because of + # temporary mDNS impairments. Removal of Avahi-discovered DCs will be + # handled differently and only if the connection cannot be established + # for a long period of time. + logging.debug('Staf._config_ctrls_finish() - must_remove_list = %s', list(must_remove_list)) + controllers_to_del = { + tid + for tid in controllers_to_del + if tid in must_remove_list or self._controllers[tid].origin != 'discovered' + } + + logging.debug('Staf._config_ctrls_finish() - controllers_to_add = %s', list(controllers_to_add)) + logging.debug('Staf._config_ctrls_finish() - controllers_to_del = %s', list(controllers_to_del)) + + # Delete controllers + for tid in controllers_to_del: + controller = self._controllers.pop(tid, None) + if controller is not None: + self._terminator.dispose(controller, self.remove_controller, keep_connection=False) + + if len(controllers_to_del) > 0: + self.dc_removed() # Let other apps (e.g. stacd) know that discovery controllers were removed. + + # Add controllers + for tid in controllers_to_add: + self._controllers[tid] = ctrl.Dc(self, tid) + + # Update "origin" on all DC objects + for tid, controller in self._controllers.items(): + origin = ( + 'configured' + if tid in configured_ctrl_list + else 'referral' + if tid in referral_ctrl_list + else 'discovered' + if tid in discovered_ctrl_list + else None + ) + if origin is not None: + controller.origin = origin + + self._dump_last_known_config(self._controllers) + + def _avahi_change(self): + if self._alive() and self._cfg_soak_tmr is not None: + self._cfg_soak_tmr.start() + + def controller_unresponsive(self, tid): + '''@brief Function invoked when a controller becomes unresponsive and + needs to be removed. + ''' + if self._alive() and self._cfg_soak_tmr is not None: + logging.debug('Staf.controller_unresponsive() - tid = %s', tid) + self._cfg_soak_tmr.start() + + def referrals_changed(self): + '''@brief Function invoked when a controller's cached referrals + have changed. + ''' + if self._alive() and self._cfg_soak_tmr is not None: + logging.debug('Staf.referrals_changed()') + self._cfg_soak_tmr.start() + + def _nvme_cli_interop(self, udev_obj): + '''Interoperability with nvme-cli: + stafd will invoke nvme-cli's connect-all the same way nvme-cli's udev + rules would do normally. This is for the case where a user has an hybrid + configuration where some controllers are configured through nvme-stas + and others through nvme-cli. This is not an optimal configuration. It + would be better if everything was configured through nvme-stas, however + support for hybrid configuration was requested by users (actually only + one user requested this).''' + + # Looking for 'change' events only + if udev_obj.action != 'change': + return + + # Looking for events from Discovery Controllers only + if not udev.Udev.is_dc_device(udev_obj): + return + + # Is the controller already being monitored by stafd? + for controller in self.get_controllers(): + if controller.device == udev_obj.sys_name: + return + + # Did we receive a Change of DLP AEN or an NVME Event indicating 'connect' or 'rediscover'? + if not _is_dlp_changed_aen(udev_obj) and not _event_matches(udev_obj, ('connected', 'rediscover')): + return + + # We need to invoke "nvme connect-all" using nvme-cli's nvmf-connect@.service + # NOTE: Eventually, we'll be able to drop --host-traddr and --host-iface from + # the parameters passed to nvmf-connect@.service. A fix was added to connect-all + # to infer these two values from the device used to connect to the DC. + # Ref: https://github.com/linux-nvme/nvme-cli/pull/1812 + cnf = [ + ('--device', udev_obj.sys_name), + ('--host-traddr', udev_obj.properties.get('NVME_HOST_TRADDR', None)), + ('--host-iface', udev_obj.properties.get('NVME_HOST_IFACE', None)), + ] + # Use systemd's escaped syntax (i.e. '=' is replaced by '\x3d', '\t' by '\x09', etc. + options = r'\x09'.join( + [fr'{option}\x3d{value}' for option, value in cnf if value not in (None, 'none', 'None', '')] + ) + logging.info('Invoking: systemctl start nvmf-connect@%s.service', options) + cmd = [defs.SYSTEMCTL, '--quiet', '--no-block', 'start', fr'nvmf-connect@{options}.service'] + subprocess.run(cmd, check=False) diff --git a/staslib/singleton.py b/staslib/singleton.py new file mode 100644 index 0000000..2171186 --- /dev/null +++ b/staslib/singleton.py @@ -0,0 +1,23 @@ +# Copyright (c) 2022, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). +# +# Authors: Martin Belanger <Martin.Belanger@dell.com> +# +'''Implementation of a singleton pattern''' + + +class Singleton(type): + '''metaclass implementation of a singleton pattern''' + + _instances = {} + + def __call__(cls, *args, **kwargs): + if cls not in cls._instances: + # This variable declaration is required to force a + # strong reference on the instance. + instance = super(Singleton, cls).__call__(*args, **kwargs) + cls._instances[cls] = instance + return cls._instances[cls] diff --git a/staslib/stacd.idl b/staslib/stacd.idl new file mode 100644 index 0000000..efefbbe --- /dev/null +++ b/staslib/stacd.idl @@ -0,0 +1,27 @@ +<node> + <interface name="@STACD_DBUS_NAME@.debug"> + <property name="tron" type="b" access="readwrite"/> + <property name="log_level" type="s" access="read"/> + <method name="process_info"> + <arg direction="out" type="s" name="info_json"/> + </method> + <method name="controller_info"> + <arg direction="in" type="s" name="transport"/> + <arg direction="in" type="s" name="traddr"/> + <arg direction="in" type="s" name="trsvcid"/> + <arg direction="in" type="s" name="host_traddr"/> + <arg direction="in" type="s" name="host_iface"/> + <arg direction="in" type="s" name="subsysnqn"/> + <arg direction="out" type="s" name="info_json"/> + </method> + </interface> + + <interface name="@STACD_DBUS_NAME@"> + <method name="list_controllers"> + <arg direction="in" type="b" name="detailed"/> + <arg direction="out" type="aa{ss}" name="controller_list"/> + </method> + </interface> +</node> + + diff --git a/staslib/stafd.idl b/staslib/stafd.idl new file mode 100644 index 0000000..8c98ffe --- /dev/null +++ b/staslib/stafd.idl @@ -0,0 +1,49 @@ +<node> + <interface name="@STAFD_DBUS_NAME@.debug"> + <property name="tron" type="b" access="readwrite"/> + <property name="log_level" type="s" access="read"/> + <method name="process_info"> + <arg direction="out" type="s" name="info_json"/> + </method> + <method name="controller_info"> + <arg direction="in" type="s" name="transport"/> + <arg direction="in" type="s" name="traddr"/> + <arg direction="in" type="s" name="trsvcid"/> + <arg direction="in" type="s" name="host_traddr"/> + <arg direction="in" type="s" name="host_iface"/> + <arg direction="in" type="s" name="subsysnqn"/> + <arg direction="out" type="s" name="info_json"/> + </method> + </interface> + + <interface name="@STAFD_DBUS_NAME@"> + <method name="list_controllers"> + <arg direction="in" type="b" name="detailed"/> + <arg direction="out" type="aa{ss}" name="controller_list"/> + </method> + <method name="get_log_pages"> + <arg direction="in" type="s" name="transport"/> + <arg direction="in" type="s" name="traddr"/> + <arg direction="in" type="s" name="trsvcid"/> + <arg direction="in" type="s" name="host_traddr"/> + <arg direction="in" type="s" name="host_iface"/> + <arg direction="in" type="s" name="subsysnqn"/> + <arg direction="out" type="aa{ss}" name="log_pages"/> + </method> + <method name="get_all_log_pages"> + <arg direction="in" type="b" name="detailed"/> + <arg direction="out" type="s" name="log_pages_json"/> + </method> + <signal name="log_pages_changed"> + <arg direction="out" type="s" name="transport"/> + <arg direction="out" type="s" name="traddr"/> + <arg direction="out" type="s" name="trsvcid"/> + <arg direction="out" type="s" name="host_traddr"/> + <arg direction="out" type="s" name="host_iface"/> + <arg direction="out" type="s" name="subsysnqn"/> + <arg direction="out" type="s" name="device"/> + </signal> + <signal name="dc_removed"></signal> + </interface> +</node> + diff --git a/staslib/stas.py b/staslib/stas.py new file mode 100644 index 0000000..95afb94 --- /dev/null +++ b/staslib/stas.py @@ -0,0 +1,554 @@ +# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). +# +# Authors: Martin Belanger <Martin.Belanger@dell.com> +# +'''Library for staf/stac. You will find here common code for stafd and stacd +including the Abstract Base Classes (ABC) for Controllers and Services''' + +import os +import sys +import abc +import signal +import pickle +import logging +import dasbus.connection +from gi.repository import Gio, GLib +from systemd.daemon import notify as sd_notify +from staslib import conf, defs, gutil, log, trid + +try: + # Python 3.9 or later + # This is the preferred way, but may not be available before Python 3.9 + from importlib.resources import files +except ImportError: + try: + # Pre Python 3.9 backport of importlib.resources (if installed) + from importlib_resources import files + except ImportError: + # Less efficient, but avalable on older versions of Python + import pkg_resources + + def load_idl(idl_fname): + '''@brief Load D-Bus Interface Description Language File''' + try: + return pkg_resources.resource_string('staslib', idl_fname).decode() + except (FileNotFoundError, AttributeError): + pass + + return '' + + else: + + def load_idl(idl_fname): + '''@brief Load D-Bus Interface Description Language File''' + try: + return files('staslib').joinpath(idl_fname).read_text() # pylint: disable=unspecified-encoding + except FileNotFoundError: + pass + + return '' + +else: + + def load_idl(idl_fname): + '''@brief Load D-Bus Interface Description Language File''' + try: + return files('staslib').joinpath(idl_fname).read_text() # pylint: disable=unspecified-encoding + except FileNotFoundError: + pass + + return '' + + +# ****************************************************************************** +def check_if_allowed_to_continue(): + '''@brief Let's perform some basic checks before going too far. There are + a few pre-requisites that need to be met before this program + is allowed to proceed: + + 1) The program needs to have root privileges + 2) The nvme-tcp kernel module must be loaded + + @return This function will only return if all conditions listed above + are met. Otherwise the program exits. + ''' + # 1) Check root privileges + if os.geteuid() != 0: + sys.exit(f'Permission denied. You need root privileges to run {defs.PROG_NAME}.') + + # 2) Check that nvme-tcp kernel module is running + if not os.path.exists('/dev/nvme-fabrics'): + # There's no point going any further if the kernel module hasn't been loaded + sys.exit('Fatal error: missing nvme-tcp kernel module') + + +# ****************************************************************************** +def tid_from_dlpe(dlpe, host_traddr, host_iface): + '''@brief Take a Discovery Log Page Entry and return a Controller ID as a dict.''' + cid = { + 'transport': dlpe['trtype'], + 'traddr': dlpe['traddr'], + 'trsvcid': dlpe['trsvcid'], + 'host-traddr': host_traddr, + 'host-iface': host_iface, + 'subsysnqn': dlpe['subnqn'], + } + return trid.TID(cid) + + +# ****************************************************************************** +def _excluded(excluded_ctrl_list, controller: dict): + '''@brief Check if @controller is excluded.''' + for excluded_ctrl in excluded_ctrl_list: + test_results = [val == controller.get(key, None) for key, val in excluded_ctrl.items()] + if all(test_results): + return True + return False + + +# ****************************************************************************** +def remove_excluded(controllers: list): + '''@brief Remove excluded controllers from the list of controllers. + @param controllers: List of TIDs + ''' + excluded_ctrl_list = conf.SvcConf().get_excluded() + if excluded_ctrl_list: + logging.debug('remove_excluded() - excluded_ctrl_list = %s', excluded_ctrl_list) + controllers = [ + controller for controller in controllers if not _excluded(excluded_ctrl_list, controller.as_dict()) + ] + return controllers + + +# ****************************************************************************** +class ControllerABC(abc.ABC): + '''@brief Base class used to manage the connection to a controller.''' + + CONNECT_RETRY_PERIOD_SEC = 60 + FAST_CONNECT_RETRY_PERIOD_SEC = 3 + + def __init__(self, tid: trid.TID, service, discovery_ctrl: bool = False): + self._tid = tid + self._serv = service # Refers to the parent service (either Staf or Stac) + self.set_level_from_tron(self._serv.tron) + self._cancellable = Gio.Cancellable() + self._connect_attempts = 0 + self._retry_connect_tmr = gutil.GTimer(self.CONNECT_RETRY_PERIOD_SEC, self._on_try_to_connect) + self._discovery_ctrl = discovery_ctrl + self._try_to_connect_deferred = gutil.Deferred(self._try_to_connect) + self._try_to_connect_deferred.schedule() + + def _release_resources(self): + # Remove pending deferred from main loop + if self._try_to_connect_deferred: + self._try_to_connect_deferred.cancel() + + if self._retry_connect_tmr is not None: + self._retry_connect_tmr.kill() + + if self._alive(): + self._cancellable.cancel() + + self._tid = None + self._serv = None + self._cancellable = None + self._retry_connect_tmr = None + self._try_to_connect_deferred = None + + @property + def id(self) -> str: + '''@brief Return the Transport ID as a printable string''' + return str(self.tid) + + @property + def tid(self): + '''@brief Return the Transport ID object''' + return self._tid + + def controller_id_dict(self) -> dict: + '''@brief return the controller ID as a dict.''' + return {k: str(v) for k, v in self.tid.as_dict().items()} + + def details(self) -> dict: + '''@brief return detailed debug info about this controller''' + return self.info() + + def info(self) -> dict: + '''@brief Get the controller info for this object''' + info = self.controller_id_dict() + info['connect attempts'] = str(self._connect_attempts) + info['retry connect timer'] = str(self._retry_connect_tmr) + return info + + def cancel(self): + '''@brief Used to cancel pending operations.''' + if self._alive(): + logging.debug('ControllerABC.cancel() - %s', self.id) + self._cancellable.cancel() + + def kill(self): + '''@brief Used to release all resources associated with this object.''' + logging.debug('ControllerABC.kill() - %s', self.id) + self._release_resources() + + def _alive(self): + '''There may be race condition where a queued event gets processed + after the object is no longer configured (i.e. alive). This method + can be used by callback functions to make sure the object is still + alive before processing further. + ''' + return self._cancellable and not self._cancellable.is_cancelled() + + def _on_try_to_connect(self): + if self._alive(): + self._try_to_connect_deferred.schedule() + return GLib.SOURCE_REMOVE + + def _should_try_to_reconnect(self): # pylint: disable=no-self-use + return True + + def _try_to_connect(self): + if not self._alive(): + return GLib.SOURCE_REMOVE + + # This is a deferred function call. Make sure + # the source of the deferred is still good. + source = GLib.main_current_source() + if source and source.is_destroyed(): + return GLib.SOURCE_REMOVE + + self._connect_attempts += 1 + + self._do_connect() + + return GLib.SOURCE_REMOVE + + @abc.abstractmethod + def set_level_from_tron(self, tron): + '''Set log level based on TRON''' + + @abc.abstractmethod + def _do_connect(self): + '''Perform connection''' + + @abc.abstractmethod + def _on_aen(self, aen: int): + '''Event handler when an AEN is received''' + + @abc.abstractmethod + def _on_nvme_event(self, nvme_event): + '''Event handler when an nvme_event is received''' + + @abc.abstractmethod + def _on_ctrl_removed(self, udev_obj): + '''Called when the associated nvme device (/dev/nvmeX) is removed + from the system by the kernel. + ''' + + @abc.abstractmethod + def _find_existing_connection(self): + '''Check if there is an existing connection that matches this Controller's TID''' + + @abc.abstractmethod + def all_ops_completed(self) -> bool: + '''@brief Returns True if all operations have completed. False otherwise.''' + + @abc.abstractmethod + def connected(self): + '''@brief Return whether a connection is established''' + + @abc.abstractmethod + def disconnect(self, disconnected_cb, keep_connection): + '''@brief Issue an asynchronous disconnect command to a Controller. + Once the async command has completed, the callback 'disconnected_cb' + will be invoked. If a controller is already disconnected, then the + callback will be added to the main loop's next idle slot to be executed + ASAP. + ''' + + @abc.abstractmethod + def reload_hdlr(self): + '''@brief This is called when a "reload" signal is received.''' + + +# ****************************************************************************** +class ServiceABC(abc.ABC): # pylint: disable=too-many-instance-attributes + '''@brief Base class used to manage a STorage Appliance Service''' + + CONF_STABILITY_SOAK_TIME_SEC = 1.5 + + def __init__(self, args, default_conf, reload_hdlr): + service_conf = conf.SvcConf(default_conf=default_conf) + service_conf.set_conf_file(args.conf_file) # reload configuration + self._tron = args.tron or service_conf.tron + log.set_level_from_tron(self._tron) + + self._lkc_file = os.path.join( + os.environ.get('RUNTIME_DIRECTORY', os.path.join('/run', defs.PROG_NAME)), 'last-known-config.pickle' + ) + self._loop = GLib.MainLoop() + self._cancellable = Gio.Cancellable() + self._resolver = gutil.NameResolver() + self._controllers = self._load_last_known_config() + self._dbus_iface = None + self._cfg_soak_tmr = gutil.GTimer(self.CONF_STABILITY_SOAK_TIME_SEC, self._on_config_ctrls) + self._sysbus = dasbus.connection.SystemMessageBus() + + GLib.unix_signal_add(GLib.PRIORITY_HIGH, signal.SIGINT, self._stop_hdlr) # CTRL-C + GLib.unix_signal_add(GLib.PRIORITY_HIGH, signal.SIGTERM, self._stop_hdlr) # systemctl stop stafd + GLib.unix_signal_add(GLib.PRIORITY_HIGH, signal.SIGHUP, reload_hdlr) # systemctl reload stafd + + nvme_options = conf.NvmeOptions() + if not nvme_options.host_iface_supp or not nvme_options.discovery_supp: + logging.warning( + 'Kernel does not appear to support all the options needed to run this program. Consider updating to a later kernel version.' + ) + + # We don't want to apply configuration changes right away. + # Often, multiple changes will occur in a short amount of time (sub-second). + # We want to wait until there are no more changes before applying them + # to the system. The following timer acts as a "soak period". Changes + # will be applied by calling self._on_config_ctrls() at the end of + # the soak period. + self._cfg_soak_tmr.start() + + def _release_resources(self): + logging.debug('ServiceABC._release_resources()') + + if self._alive(): + self._cancellable.cancel() + + if self._cfg_soak_tmr is not None: + self._cfg_soak_tmr.kill() + + self._controllers.clear() + + if self._sysbus: + self._sysbus.disconnect() + + self._cfg_soak_tmr = None + self._cancellable = None + self._resolver = None + self._lkc_file = None + self._sysbus = None + + def _config_dbus(self, iface_obj, bus_name: str, obj_name: str): + self._dbus_iface = iface_obj + self._sysbus.publish_object(obj_name, iface_obj) + self._sysbus.register_service(bus_name) + + @property + def tron(self): + '''@brief Get Trace ON property''' + return self._tron + + @tron.setter + def tron(self, value): + '''@brief Set Trace ON property''' + self._tron = value + log.set_level_from_tron(self._tron) + for controller in self._controllers.values(): + controller.set_level_from_tron(self._tron) + + def run(self): + '''@brief Start the main loop execution''' + try: + self._loop.run() + except Exception as ex: # pylint: disable=broad-except + logging.critical('exception: %s', ex) + + self._loop = None + + def info(self) -> dict: + '''@brief Get the status info for this object (used for debug)''' + nvme_options = conf.NvmeOptions() + info = conf.SysConf().as_dict() + info['last known config file'] = self._lkc_file + info['config soak timer'] = str(self._cfg_soak_tmr) + info['kernel support.TP8013'] = str(nvme_options.discovery_supp) + info['kernel support.host_iface'] = str(nvme_options.host_iface_supp) + return info + + def get_controllers(self) -> dict: + '''@brief return the list of controller objects''' + return self._controllers.values() + + def get_controller( + self, transport: str, traddr: str, trsvcid: str, host_traddr: str, host_iface: str, subsysnqn: str + ): # pylint: disable=too-many-arguments + '''@brief get the specified controller object from the list of controllers''' + cid = { + 'transport': transport, + 'traddr': traddr, + 'trsvcid': trsvcid, + 'host-traddr': host_traddr, + 'host-iface': host_iface, + 'subsysnqn': subsysnqn, + } + return self._controllers.get(trid.TID(cid)) + + def _remove_ctrl_from_dict(self, controller, shutdown=False): + tid_to_pop = controller.tid + if not tid_to_pop: + # Being paranoid. This should not happen, but let's say the + # controller object has been purged, but it is somehow still + # listed in self._controllers. + for tid, _controller in self._controllers.items(): + if _controller is controller: + tid_to_pop = tid + break + + if tid_to_pop: + logging.debug('ServiceABC._remove_ctrl_from_dict()- %s | %s', tid_to_pop, controller.device) + popped = self._controllers.pop(tid_to_pop, None) + if not shutdown and popped is not None and self._cfg_soak_tmr: + self._cfg_soak_tmr.start() + else: + logging.debug('ServiceABC._remove_ctrl_from_dict()- already removed') + + def remove_controller(self, controller, success): # pylint: disable=unused-argument + '''@brief remove the specified controller object from the list of controllers + @param controller: the controller object + @param success: whether the disconnect was successful''' + logging.debug('ServiceABC.remove_controller()') + if isinstance(controller, ControllerABC): + self._remove_ctrl_from_dict(controller) + controller.kill() + + def _alive(self): + '''It's a good idea to check that this object hasn't been + cancelled (i.e. is still alive) when entering a callback function. + Callback functrions can be invoked after, for example, a process has + been signalled to stop or restart, in which case it makes no sense to + proceed with the callback. + ''' + return self._cancellable and not self._cancellable.is_cancelled() + + def _cancel(self): + logging.debug('ServiceABC._cancel()') + if self._alive(): + self._cancellable.cancel() + + for controller in self._controllers.values(): + controller.cancel() + + def _stop_hdlr(self): + logging.debug('ServiceABC._stop_hdlr()') + sd_notify('STOPPING=1') + + self._cancel() # Cancel pending operations + + self._dump_last_known_config(self._controllers) + + if len(self._controllers) == 0: + GLib.idle_add(self._exit) + else: + self._disconnect_all() + + return GLib.SOURCE_REMOVE + + def _on_final_disconnect(self, controller, success): + '''Callback invoked after a controller is disconnected. + THIS IS USED DURING PROCESS SHUTDOWN TO WAIT FOR ALL CONTROLLERS TO BE + DISCONNECTED BEFORE EXITING THE PROGRAM. ONLY CALL ON SHUTDOWN! + @param controller: the controller object + @param success: whether the disconnect operation was successful + ''' + logging.debug( + 'ServiceABC._on_final_disconnect() - %s | %s: disconnect %s', + controller.id, + controller.device, + 'succeeded' if success else 'failed', + ) + + self._remove_ctrl_from_dict(controller, True) + controller.kill() + + # When all controllers have disconnected, we can finish the clean up + if len(self._controllers) == 0: + # Defer exit to the next main loop's idle period. + GLib.idle_add(self._exit) + + def _exit(self): + logging.debug('ServiceABC._exit()') + self._release_resources() + self._loop.quit() + + def _on_config_ctrls(self, *_user_data): + if self._alive(): + self._config_ctrls() + return GLib.SOURCE_REMOVE + + def _config_ctrls(self): + '''@brief Start controllers configuration.''' + # The configuration file may contain controllers and/or excluded + # controllers with traddr specified as hostname instead of IP address. + # Because of this, we need to remove those excluded elements before + # running name resolution. And we will need to remove excluded + # elements after name resolution is complete (i.e. in the calback + # function _config_ctrls_finish) + logging.debug('ServiceABC._config_ctrls()') + configured_controllers = [trid.TID(cid) for cid in conf.SvcConf().get_controllers()] + configured_controllers = remove_excluded(configured_controllers) + self._resolver.resolve_ctrl_async(self._cancellable, configured_controllers, self._config_ctrls_finish) + + def _read_lkc(self): + '''@brief Read Last Known Config from file''' + try: + with open(self._lkc_file, 'rb') as file: + return pickle.load(file) + except (FileNotFoundError, AttributeError, EOFError): + return None + + def _write_lkc(self, config): + '''@brief Write Last Known Config to file, and if config is empty + make sure the file is emptied.''' + try: + # Note that if config is empty we still + # want to open/close the file to empty it. + with open(self._lkc_file, 'wb') as file: + if config: + pickle.dump(config, file) + except FileNotFoundError as ex: + logging.error('Unable to save last known config: %s', ex) + + @abc.abstractmethod + def _disconnect_all(self): + '''Tell all controller objects to disconnect''' + + @abc.abstractmethod + def _keep_connections_on_exit(self): + '''@brief Determine whether connections should remain when the + process exits. + + NOTE) This is the base class method used to define the interface. + It must be overloaded by a child class. + ''' + + @abc.abstractmethod + def _config_ctrls_finish(self, configured_ctrl_list): + '''@brief Finish controllers configuration after hostnames (if any) + have been resolved. + + Configuring controllers must be done asynchronously in 2 steps. + In the first step, host names get resolved to find their IP addresses. + Name resolution can take a while, especially when an external name + resolution server is used. Once that step completed, the callback + method _config_ctrls_finish() (i.e. this method), gets invoked to + complete the controller configuration. + + NOTE) This is the base class method used to define the interface. + It must be overloaded by a child class. + ''' + + @abc.abstractmethod + def _load_last_known_config(self): + '''Load last known config from file (if any)''' + + @abc.abstractmethod + def _dump_last_known_config(self, controllers): + '''Save last known config to file''' diff --git a/staslib/timeparse.py b/staslib/timeparse.py new file mode 100644 index 0000000..5295fc4 --- /dev/null +++ b/staslib/timeparse.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +''' +This module was borrowed and modified from: https://github.com/wroberts/pytimeparse + +timeparse.py +(c) Will Roberts <wildwilhelm@gmail.com> 1 February, 2014 + +Implements a single function, `timeparse`, which can parse various +kinds of time expressions. +''' + +# MIT LICENSE +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation files +# (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS +# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import re + +SIGN = r'(?P<sign>[+|-])?' +DAYS = r'(?P<days>[\d.]+)\s*(?:d|dys?|days?)' +HOURS = r'(?P<hours>[\d.]+)\s*(?:h|hrs?|hours?)' +MINS = r'(?P<mins>[\d.]+)\s*(?:m|(mins?)|(minutes?))' +SECS = r'(?P<secs>[\d.]+)\s*(?:s|secs?|seconds?)' +SEPARATORS = r'[,/]' +SECCLOCK = r':(?P<secs>\d{2}(?:\.\d+)?)' +MINCLOCK = r'(?P<mins>\d{1,2}):(?P<secs>\d{2}(?:\.\d+)?)' +HOURCLOCK = r'(?P<hours>\d+):(?P<mins>\d{2}):(?P<secs>\d{2}(?:\.\d+)?)' +DAYCLOCK = r'(?P<days>\d+):(?P<hours>\d{2}):(?P<mins>\d{2}):(?P<secs>\d{2}(?:\.\d+)?)' + + +def _opt(string): + return f'(?:{string})?' + + +def _optsep(string): + return fr'(?:{string}\s*(?:{SEPARATORS}\s*)?)?' + + +TIMEFORMATS = [ + fr'{_optsep(DAYS)}\s*{_optsep(HOURS)}\s*{_optsep(MINS)}\s*{_opt(SECS)}', + f'{MINCLOCK}', + fr'{_optsep(DAYS)}\s*{HOURCLOCK}', + f'{DAYCLOCK}', + f'{SECCLOCK}', +] + +COMPILED_SIGN = re.compile(r'\s*' + SIGN + r'\s*(?P<unsigned>.*)$') +COMPILED_TIMEFORMATS = [re.compile(r'\s*' + timefmt + r'\s*$', re.I) for timefmt in TIMEFORMATS] + +MULTIPLIERS = { + 'days': 60 * 60 * 24, + 'hours': 60 * 60, + 'mins': 60, + 'secs': 1, +} + + +def timeparse(sval): + ''' + Parse a time expression, returning it as a number of seconds. If + possible, the return value will be an `int`; if this is not + possible, the return will be a `float`. Returns `None` if a time + expression cannot be parsed from the given string. + + Arguments: + - `sval`: the string value to parse + + >>> timeparse('1:24') + 84 + >>> timeparse(':22') + 22 + >>> timeparse('1 minute, 24 secs') + 84 + >>> timeparse('1m24s') + 84 + >>> timeparse('1.2 minutes') + 72 + >>> timeparse('1.2 seconds') + 1.2 + + Time expressions can be signed. + + >>> timeparse('- 1 minute') + -60 + >>> timeparse('+ 1 minute') + 60 + ''' + try: + return float(sval) + except TypeError: + pass + except ValueError: + match = COMPILED_SIGN.match(sval) + sign = -1 if match.groupdict()['sign'] == '-' else 1 + sval = match.groupdict()['unsigned'] + for timefmt in COMPILED_TIMEFORMATS: + match = timefmt.match(sval) + if match and match.group(0).strip(): + mdict = match.groupdict() + # if all of the fields are integer numbers + if all(v.isdigit() for v in list(mdict.values()) if v): + return sign * sum((MULTIPLIERS[k] * int(v, 10) for (k, v) in list(mdict.items()) if v is not None)) + + # if SECS is an integer number + if 'secs' not in mdict or mdict['secs'] is None or mdict['secs'].isdigit(): + # we will return an integer + return sign * int( + sum( + ( + MULTIPLIERS[k] * float(v) + for (k, v) in list(mdict.items()) + if k != 'secs' and v is not None + ) + ) + ) + (int(mdict['secs'], 10) if mdict['secs'] else 0) + + # SECS is a float, we will return a float + return sign * sum((MULTIPLIERS[k] * float(v) for (k, v) in list(mdict.items()) if v is not None)) + + return None diff --git a/staslib/trid.py b/staslib/trid.py new file mode 100644 index 0000000..ea40b7d --- /dev/null +++ b/staslib/trid.py @@ -0,0 +1,137 @@ +# Copyright (c) 2022, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). +# +# Authors: Martin Belanger <Martin.Belanger@dell.com> +# +'''This module defines the Transport Identifier Object, which is used +throughout nvme-stas to uniquely identify a Controller''' + +import hashlib +from staslib import conf + + +class TID: # pylint: disable=too-many-instance-attributes + '''Transport Identifier''' + + RDMA_IP_PORT = '4420' + DISC_IP_PORT = '8009' + + def __init__(self, cid: dict): + '''@param cid: Controller Identifier. A dictionary with the following + contents. + { + # Transport parameters + 'transport': str, # [mandatory] + 'traddr': str, # [mandatory] + 'subsysnqn': str, # [mandatory] + 'trsvcid': str, # [optional] + 'host-traddr': str, # [optional] + 'host-iface': str, # [optional] + + # Connection parameters + 'dhchap-ctrl-secret': str, # [optional] + 'hdr-digest': str, # [optional] + 'data-digest': str, # [optional] + 'nr-io-queues': str, # [optional] + 'nr-write-queues': str, # [optional] + 'nr-poll-queues': str, # [optional] + 'queue-size': str, # [optional] + 'kato': str, # [optional] + 'reconnect-delay': str, # [optional] + 'ctrl-loss-tmo': str, # [optional] + 'disable-sqflow': str, # [optional] + } + ''' + self._cfg = { + k: v + for k, v in cid.items() + if k not in ('transport', 'traddr', 'subsysnqn', 'trsvcid', 'host-traddr', 'host-iface') + } + self._transport = cid.get('transport', '') + self._traddr = cid.get('traddr', '') + self._trsvcid = '' + if self._transport in ('tcp', 'rdma'): + trsvcid = cid.get('trsvcid', None) + self._trsvcid = ( + trsvcid if trsvcid else (TID.RDMA_IP_PORT if self._transport == 'rdma' else TID.DISC_IP_PORT) + ) + self._host_traddr = cid.get('host-traddr', '') + self._host_iface = '' if conf.SvcConf().ignore_iface else cid.get('host-iface', '') + self._subsysnqn = cid.get('subsysnqn', '') + self._shortkey = (self._transport, self._traddr, self._trsvcid, self._subsysnqn, self._host_traddr) + self._key = (self._transport, self._traddr, self._trsvcid, self._subsysnqn, self._host_traddr, self._host_iface) + self._hash = int.from_bytes( + hashlib.md5(''.join(self._key).encode('utf-8')).digest(), 'big' + ) # We need a consistent hash between restarts + self._id = f'({self._transport}, {self._traddr}, {self._trsvcid}{", " + self._subsysnqn if self._subsysnqn else ""}{", " + self._host_iface if self._host_iface else ""}{", " + self._host_traddr if self._host_traddr else ""})' # pylint: disable=line-too-long + + @property + def transport(self): # pylint: disable=missing-function-docstring + return self._transport + + @property + def traddr(self): # pylint: disable=missing-function-docstring + return self._traddr + + @property + def trsvcid(self): # pylint: disable=missing-function-docstring + return self._trsvcid + + @property + def host_traddr(self): # pylint: disable=missing-function-docstring + return self._host_traddr + + @property + def host_iface(self): # pylint: disable=missing-function-docstring + return self._host_iface + + @property + def subsysnqn(self): # pylint: disable=missing-function-docstring + return self._subsysnqn + + @property + def cfg(self): # pylint: disable=missing-function-docstring + return self._cfg + + def as_dict(self): + '''Return object members as a dictionary''' + data = { + 'transport': self.transport, + 'traddr': self.traddr, + 'subsysnqn': self.subsysnqn, + 'trsvcid': self.trsvcid, + 'host-traddr': self.host_traddr, + 'host-iface': self.host_iface, + } + data.update(self._cfg) + return data + + def __str__(self): + return self._id + + def __repr__(self): + return self._id + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + + if self._host_iface and other._host_iface: + return self._key == other._key + + return self._shortkey == other._shortkey + + def __ne__(self, other): + if not isinstance(other, self.__class__): + return True + + if self._host_iface and other._host_iface: + return self._key != other._key + + return self._shortkey != other._shortkey + + def __hash__(self): + return self._hash diff --git a/staslib/udev.py b/staslib/udev.py new file mode 100644 index 0000000..12ef61b --- /dev/null +++ b/staslib/udev.py @@ -0,0 +1,334 @@ +# Copyright (c) 2022, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). +# +# Authors: Martin Belanger <Martin.Belanger@dell.com> +# +'''This module provides functions to access nvme devices using the pyudev module''' + +import os +import time +import logging +import pyudev +from gi.repository import GLib +from staslib import defs, iputil, trid + + +# ****************************************************************************** +class Udev: + '''@brief Udev event monitor. Provide a way to register for udev events. + WARNING: THE singleton.Singleton PATTERN CANNOT BE USED WITH THIS CLASS. + IT INTERFERES WITH THE pyudev INTERNALS, WHICH CAUSES OBJECT CLEAN UP TO FAIL. + ''' + + def __init__(self): + self._log_event_soak_time = 0 + self._log_event_count = 0 + self._device_event_registry = dict() + self._action_event_registry = dict() + self._context = pyudev.Context() + self._monitor = pyudev.Monitor.from_netlink(self._context) + self._monitor.filter_by(subsystem='nvme') + self._event_source = GLib.io_add_watch( + self._monitor.fileno(), + GLib.PRIORITY_HIGH, + GLib.IO_IN, + self._process_udev_event, + ) + self._monitor.start() + + def release_resources(self): + '''Release all resources used by this object''' + if self._event_source is not None: + GLib.source_remove(self._event_source) + + if self._monitor is not None: + self._monitor.remove_filter() + + self._event_source = None + self._monitor = None + self._context = None + self._device_event_registry = None + self._action_event_registry = None + + def get_nvme_device(self, sys_name): + '''@brief Get the udev device object associated with an nvme device. + @param sys_name: The device system name (e.g. 'nvme1') + @return A pyudev.device._device.Device object + ''' + device_node = os.path.join('/dev', sys_name) + try: + return pyudev.Devices.from_device_file(self._context, device_node) + except pyudev.DeviceNotFoundByFileError as ex: + logging.error("Udev.get_nvme_device() - Error: %s", ex) + return None + + def is_action_cback_registered(self, action: str, user_cback): + '''Returns True if @user_cback is registered for @action. False otherwise. + @param action: one of 'add', 'remove', 'change'. + @param user_cback: A callback function with this signature: cback(udev_obj) + ''' + return user_cback in self._action_event_registry.get(action, set()) + + def register_for_action_events(self, action: str, user_cback): + '''@brief Register a callback function to be called when udev events + for a specific action are received. + @param action: one of 'add', 'remove', 'change'. + ''' + self._action_event_registry.setdefault(action, set()).add(user_cback) + + def unregister_for_action_events(self, action: str, user_cback): + '''@brief The opposite of register_for_action_events()''' + try: + self._action_event_registry.get(action, set()).remove(user_cback) + except KeyError: # Raise if user_cback already removed + pass + + def register_for_device_events(self, sys_name: str, user_cback): + '''@brief Register a callback function to be called when udev events + are received for a specific nvme device. + @param sys_name: The device system name (e.g. 'nvme1') + ''' + if sys_name: + self._device_event_registry[sys_name] = user_cback + + def unregister_for_device_events(self, user_cback): + '''@brief The opposite of register_for_device_events()''' + entries = list(self._device_event_registry.items()) + for sys_name, _user_cback in entries: + if user_cback == _user_cback: + self._device_event_registry.pop(sys_name, None) + break + + def get_attributes(self, sys_name: str, attr_ids) -> dict: + '''@brief Get all the attributes associated with device @sys_name''' + attrs = {attr_id: '' for attr_id in attr_ids} + if sys_name and sys_name != 'nvme?': + udev = self.get_nvme_device(sys_name) + if udev is not None: + for attr_id in attr_ids: + try: + value = udev.attributes.asstring(attr_id).strip() + attrs[attr_id] = '' if value == '(efault)' else value + except Exception: # pylint: disable=broad-except + pass + + return attrs + + @staticmethod + def is_dc_device(device): + '''@brief check whether device refers to a Discovery Controller''' + subsysnqn = device.attributes.get('subsysnqn') + if subsysnqn is not None and subsysnqn.decode() == defs.WELL_KNOWN_DISC_NQN: + return True + + # Note: Prior to 5.18 linux didn't expose the cntrltype through + # the sysfs. So, this may return None on older kernels. + cntrltype = device.attributes.get('cntrltype') + if cntrltype is not None and cntrltype.decode() == 'discovery': + return True + + # Imply Discovery controller based on the absence of children. + # Discovery Controllers have no children devices + if len(list(device.children)) == 0: + return True + + return False + + @staticmethod + def is_ioc_device(device): + '''@brief check whether device refers to an I/O Controller''' + # Note: Prior to 5.18 linux didn't expose the cntrltype through + # the sysfs. So, this may return None on older kernels. + cntrltype = device.attributes.get('cntrltype') + if cntrltype is not None and cntrltype.decode() == 'io': + return True + + # Imply I/O controller based on the presence of children. + # I/O Controllers have children devices + if len(list(device.children)) != 0: + return True + + return False + + def find_nvme_dc_device(self, tid): + '''@brief Find the nvme device associated with the specified + Discovery Controller. + @return The device if a match is found, None otherwise. + ''' + for device in self._context.list_devices( + subsystem='nvme', NVME_TRADDR=tid.traddr, NVME_TRSVCID=tid.trsvcid, NVME_TRTYPE=tid.transport + ): + if not self.is_dc_device(device): + continue + + if self.get_tid(device) != tid: + continue + + return device + + return None + + def find_nvme_ioc_device(self, tid): + '''@brief Find the nvme device associated with the specified + I/O Controller. + @return The device if a match is found, None otherwise. + ''' + for device in self._context.list_devices( + subsystem='nvme', NVME_TRADDR=tid.traddr, NVME_TRSVCID=tid.trsvcid, NVME_TRTYPE=tid.transport + ): + if not self.is_ioc_device(device): + continue + + if self.get_tid(device) != tid: + continue + + return device + + return None + + def get_nvme_ioc_tids(self, transports): + '''@brief Find all the I/O controller nvme devices in the system. + @return A list of pyudev.device._device.Device objects + ''' + tids = [] + for device in self._context.list_devices(subsystem='nvme'): + if device.properties.get('NVME_TRTYPE', '') not in transports: + continue + + if not self.is_ioc_device(device): + continue + + tids.append(self.get_tid(device)) + + return tids + + def _process_udev_event(self, event_source, condition): # pylint: disable=unused-argument + if condition == GLib.IO_IN: + event_count = 0 + while True: + try: + device = self._monitor.poll(timeout=0) + except EnvironmentError as ex: + device = None + # This event seems to happen in bursts. So, let's suppress + # logging for 2 seconds to avoid filling the syslog. + self._log_event_count += 1 + now = time.time() + if now > self._log_event_soak_time: + logging.debug('Udev._process_udev_event() - %s [%s]', ex, self._log_event_count) + self._log_event_soak_time = now + 2 + self._log_event_count = 0 + + if device is None: + break + + event_count += 1 + self._device_event(device, event_count) + + return GLib.SOURCE_CONTINUE + + @staticmethod + def __cback_names(action_cbacks, device_cback): + names = [] + for cback in action_cbacks: + names.append(cback.__name__ + '()') + if device_cback: + names.append(device_cback.__name__ + '()') + return names + + def _device_event(self, device, event_count): + action_cbacks = self._action_event_registry.get(device.action, set()) + device_cback = self._device_event_registry.get(device.sys_name, None) + + logging.debug( + 'Udev._device_event() - %-8s %-6s %-8s %s', + f'{device.sys_name}:', + device.action, + f'{event_count:2}:{device.sequence_number}', + self.__cback_names(action_cbacks, device_cback), + ) + + for action_cback in action_cbacks: + GLib.idle_add(action_cback, device) + + if device_cback is not None: + GLib.idle_add(device_cback, device) + + @staticmethod + def _get_property(device, prop, default=''): + prop = device.properties.get(prop, default) + return '' if prop.lower() == 'none' else prop + + @staticmethod + def _get_attribute(device, attr_id, default=''): + try: + attr = device.attributes.asstring(attr_id).strip() + except Exception: # pylint: disable=broad-except + attr = default + + return '' if attr.lower() == 'none' else attr + + @staticmethod + def get_key_from_attr(device, attr, key, delim=','): + '''Get attribute specified by attr, which is composed of key=value pairs. + Then return the value associated with key. + @param device: The Device object + @param attr: The device's attribute to get + @param key: The key to look for in the attribute + @param delim: Delimiter used between key=value pairs. + @example: + "address" attribute contains "trtype=tcp,traddr=10.10.1.100,trsvcid=4420,host_traddr=10.10.1.50" + ''' + attr_str = Udev._get_attribute(device, attr) + if not attr_str: + return '' + + if key[-1] != '=': + key += '=' + start = attr_str.find(key) + if start < 0: + return '' + start += len(key) + + end = attr_str.find(delim, start) + if end < 0: + return attr_str[start:] + + return attr_str[start:end] + + @staticmethod + def _get_host_iface(device): + host_iface = Udev._get_property(device, 'NVME_HOST_IFACE') + if not host_iface: + # We'll try to find the interface from the source address on + # the connection. Only available if kernel exposes the source + # address (src_addr) in the "address" attribute. + src_addr = Udev.get_key_from_attr(device, 'address', 'src_addr=') + host_iface = iputil.get_interface(src_addr) + return host_iface + + @staticmethod + def get_tid(device): + '''@brief return the Transport ID associated with a udev device''' + cid = { + 'transport': Udev._get_property(device, 'NVME_TRTYPE'), + 'traddr': Udev._get_property(device, 'NVME_TRADDR'), + 'trsvcid': Udev._get_property(device, 'NVME_TRSVCID'), + 'host-traddr': Udev._get_property(device, 'NVME_HOST_TRADDR'), + 'host-iface': Udev._get_host_iface(device), + 'subsysnqn': Udev._get_attribute(device, 'subsysnqn'), + } + return trid.TID(cid) + + +UDEV = Udev() # Singleton + + +def shutdown(): + '''Destroy the UDEV singleton''' + global UDEV # pylint: disable=global-statement,global-variable-not-assigned + UDEV.release_resources() + del UDEV diff --git a/staslib/version.py b/staslib/version.py new file mode 100644 index 0000000..999d916 --- /dev/null +++ b/staslib/version.py @@ -0,0 +1,64 @@ +# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). +# +# Authors: Martin Belanger <Martin.Belanger@dell.com> +# +''' distutils (and hence LooseVersion) is being deprecated. None of the + suggested replacements (e.g. from pkg_resources import parse_version) quite + work with Linux kernel versions the way LooseVersion does. + + It was suggested to simply lift the LooseVersion code and vendor it in, + which is what this module is about. +''' + +import re + + +class KernelVersion: + '''Code loosely lifted from distutils's LooseVersion''' + + component_re = re.compile(r'(\d+ | [a-z]+ | \.)', re.VERBOSE) + + def __init__(self, string: str): + self.string = string + self.version = self.__parse(string) + + def __str__(self): + return self.string + + def __repr__(self): + return f'KernelVersion ("{self}")' + + def __eq__(self, other): + return self.version == self.__version(other) + + def __lt__(self, other): + return self.version < self.__version(other) + + def __le__(self, other): + return self.version <= self.__version(other) + + def __gt__(self, other): + return self.version > self.__version(other) + + def __ge__(self, other): + return self.version >= self.__version(other) + + @staticmethod + def __version(obj): + return obj.version if isinstance(obj, KernelVersion) else KernelVersion.__parse(obj) + + @staticmethod + def __parse(string): + components = [] + for item in KernelVersion.component_re.split(string): + if item and item != '.': + try: + components.append(int(item)) + except ValueError: + pass + + return components diff --git a/subprojects/libnvme.wrap b/subprojects/libnvme.wrap new file mode 100644 index 0000000..6da2d89 --- /dev/null +++ b/subprojects/libnvme.wrap @@ -0,0 +1,6 @@ +[wrap-git] +url = https://github.com/linux-nvme/libnvme.git +revision = v1.3 + +[provide] +libnvme = libnvme_dep diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/test/__init__.py diff --git a/test/meson.build b/test/meson.build new file mode 100644 index 0000000..251140c --- /dev/null +++ b/test/meson.build @@ -0,0 +1,152 @@ +# Copyright (c) 2022, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). +# +# Authors: Martin Belanger <Martin.Belanger@dell.com> +# + +test_env = environment({'MALLOC_PERTURB_': '0'}) + +libnvme_location = '?' + +# We require libnvme in order to run the tests. We have two choices, either +# run the tests using a pre-installed version of libnvme (i.e. from /usr) or +# build libnvme as a meson subproject and run the tests using that version +# of libnvme. The decision to use one method over the other is controlled +# by the option "libnvme-sel". Note that if a pre-intalled libnvme is selected +# but one cannot be found, then we fall back to using the subproject libnvme. +if get_option('libnvme-sel') == 'pre-installed' + # Check if a pre-installed libnvme can be found + rr = run_command(python3, '-c', 'import libnvme; print(f"{libnvme.__path__[0]}")', check: false, env: test_env) + if rr.returncode() == 0 + libnvme_location = rr.stdout().strip() + endif +endif + +if libnvme_location == '?' + # Second, if libnvme is not already installed or "libnvme-sel" is not + # set to "pre-installed", let's fallback to using the subproject. + libnvme_dep = dependency('python3-libnvme', fallback: ['libnvme', 'libnvme_dep'], required: false) + + test_env.prepend('PYTHONPATH', PYTHONPATH) # This sets the path to look in the build directory + rr = run_command(python3, '-c', 'import libnvme; print(f"{libnvme.__path__[0]}")', check: false, env: test_env) + if rr.returncode() == 0 + libnvme_location = rr.stdout().strip() + endif +endif + +if libnvme_location == '?' + warning('Missing runtime package needed to run the tests: python3-libnvme.') +else + message('\n\n\u001b[32m\u001b[1mNOTE: Tests will be using @0@\u001b[0m\n'.format(libnvme_location)) + #--------------------------------------------------------------------------- + # pylint and pyflakes + if modules_to_lint.length() != 0 + pylint = find_program('pylint', required: false) + pyflakes = find_program('pyflakes3', required: false) + if not pyflakes.found() + temp = find_program('pyflakes', required: false) + if temp.found() and run_command(temp, '--version', check: false).stdout().contains('Python 3') + pyflakes = temp + endif + endif + + rcfile = meson.current_source_dir() / 'pylint.rc' + + if pylint.found() + test('pylint', pylint, args: ['--rcfile=' + rcfile] + modules_to_lint, env: test_env) + else + warning('Skiping some of the tests because "pylint" is missing.') + endif + if pyflakes.found() + test('pyflakes', pyflakes, args: modules_to_lint, env: test_env) + else + warning('Skiping some of the tests because "pyflakes" is missing.') + endif + endif + + #--------------------------------------------------------------------------- + # Check dependencies + dbus_is_active = false + avahi_is_active = false + systemctl = find_program('systemctl', required: false) + if systemctl.found() + rr = run_command(systemctl, 'is-active', 'dbus.service', check: false) + dbus_is_active = rr.returncode() == 0 and rr.stdout().strip() == 'active' + if not dbus_is_active + warning('Dbus daemon is not running') + endif + + rr = run_command(systemctl, 'is-active', 'avahi-daemon.service', check: false) + avahi_is_active = rr.returncode() == 0 and rr.stdout().strip() == 'active' + if not avahi_is_active + warning('Avahi daemon is not running') + endif + endif + + want_avahi_test = dbus_is_active and avahi_is_active + + #--------------------------------------------------------------------------- + # Unit tests + things_to_test = [ + ['Test Configuration', 'test-config.py', []], + ['Test Controller', 'test-controller.py', ['pyfakefs']], + ['Test GTimer', 'test-gtimer.py', []], + ['Test iputil', 'test-iputil.py', []], + ['Test KernelVersion', 'test-version.py', []], + ['Test log', 'test-log.py', ['pyfakefs']], + ['Test NvmeOptions', 'test-nvme_options.py', ['pyfakefs']], + ['Test Service', 'test-service.py', ['pyfakefs']], + ['Test TID', 'test-transport_id.py', []], + ['Test Udev', 'test-udev.py', []], + ['Test timeparse', 'test-timeparse.py', []], + ] + + # The Avahi test requires the Avahi and the Dbus daemons to be running. + if want_avahi_test + things_to_test += [['Test Avahi', 'test-avahi.py', []]] + else + warning('Skip Avahi Test due to missing dependencies') + endif + + foreach thing: things_to_test + msg = thing[0] + + # Check whether all dependencies can be found + missing_deps = [] + deps = thing[2] + foreach dep : deps + rr = run_command(python3, '-c', 'import @0@'.format(dep), check: false) + if rr.returncode() != 0 + missing_deps += [dep] + endif + endforeach + + if missing_deps.length() == 0 + # Allow the test to run if all dependencies are available + script = meson.current_source_dir() / thing[1] + test(msg, python3, args: script, env: test_env) + else + warning('"@0@" requires python module "@1@"'.format(msg, missing_deps)) + endif + endforeach +endif + + +#------------------------------------------------------------------------------- +# Make sure code complies with minimum Python version requirement. +tools = [ + meson.current_source_dir() / '../doc', + meson.current_source_dir() / '../utils', +] +vermin = find_program('vermin', required: false) +if vermin.found() + if modules_to_lint.length() != 0 + test('vermin code', vermin, args: ['--config-file', meson.current_source_dir() / 'vermin.conf'] + modules_to_lint, env: test_env) + endif + test('vermin tools', vermin, args: ['--config-file', meson.current_source_dir() / 'vermin-tools.conf'] + tools, env: test_env) +else + warning('Skiping some of the tests because "vermin" is missing.') +endif diff --git a/test/pylint.rc b/test/pylint.rc new file mode 100644 index 0000000..ce56f98 --- /dev/null +++ b/test/pylint.rc @@ -0,0 +1,500 @@ +# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). +# +# Authors: Martin Belanger <Martin.Belanger@dell.com> +[MASTER] + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code +extension-pkg-whitelist= + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Add files or directories matching the regex patterns to the blacklist. The +# regex matches against base names, not paths. +ignore-patterns= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. +jobs=1 + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# Specify a configuration file. +#rcfile= + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED +confidence= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +disable=print-statement, + parameter-unpacking, + unpacking-in-except, + old-raise-syntax, + backtick, + long-suffix, + old-ne-operator, + old-octal-literal, + import-star-module-level, + raw-checker-failed, + bad-inline-option, + locally-disabled, + locally-enabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + apply-builtin, + basestring-builtin, + buffer-builtin, + cmp-builtin, + coerce-builtin, + execfile-builtin, + file-builtin, + long-builtin, + raw_input-builtin, + reduce-builtin, + standarderror-builtin, + unicode-builtin, + xrange-builtin, + coerce-method, + delslice-method, + getslice-method, + setslice-method, + no-absolute-import, + old-division, + dict-iter-method, + dict-view-method, + next-method-called, + metaclass-assignment, + indexing-exception, + raising-string, + reload-builtin, + oct-method, + hex-method, + nonzero-method, + cmp-method, + input-builtin, + round-builtin, + intern-builtin, + unichr-builtin, + map-builtin-not-iterating, + zip-builtin-not-iterating, + range-builtin-not-iterating, + filter-builtin-not-iterating, + using-cmp-argument, + eq-without-hash, + div-method, + idiv-method, + rdiv-method, + exception-message-attribute, + invalid-str-codec, + sys-max-int, + bad-python3-import, + deprecated-string-function, + deprecated-str-translate-call, + use-list-literal, + use-dict-literal, + bad-option-value, + R0801, + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable= + + +[REPORTS] + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +#msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio).You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Tells whether to display a full report or only the messages +reports=no + +# Activate the evaluation score. +score=yes + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + + +[SPELLING] + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_,_cb + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,future.builtins + + +[SIMILARITIES] + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )?<?https?://\S+>?$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=200 + +# Maximum number of lines in a module +max-module-lines=2000 + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma,dict-separator + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + + +[LOGGING] + +# Logging modules to check that the string format arguments are in logging +# function parameter format +logging-modules=logging + + +[BASIC] + +# Naming hint for argument names +argument-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Regular expression matching correct argument names +argument-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Naming hint for attribute names +attr-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Regular expression matching correct attribute names +attr-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Naming hint for class attribute names +class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Regular expression matching correct class attribute names +class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Naming hint for class names +class-name-hint=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression matching correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Naming hint for constant names +const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression matching correct constant names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming hint for function names +function-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Regular expression matching correct function names +function-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_,op,ls,f,ip,id + +# Include a hint for the correct naming format with invalid-name +include-naming-hint=no + +# Naming hint for inline iteration names +inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ + +# Regular expression matching correct inline iteration names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Naming hint for method names +method-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Regular expression matching correct method names +method-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Naming hint for module names +module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression matching correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +property-classes=abc.abstractproperty + +# Naming hint for variable names +variable-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Regular expression matching correct variable names +variable-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in a if statement +max-bool-expr=5 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of statements in function / method body +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + + +[IMPORTS] + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub,TERMIOS,Bastion,rexec + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception diff --git a/test/test-avahi.py b/test/test-avahi.py new file mode 100755 index 0000000..3529104 --- /dev/null +++ b/test/test-avahi.py @@ -0,0 +1,41 @@ +#!/usr/bin/python3 +import shutil +import logging +import unittest +from staslib import avahi +import dasbus.connection +import subprocess + +SYSTEMCTL = shutil.which('systemctl') + + +class Test(unittest.TestCase): + '''Unit tests for class Avahi''' + + def test_new(self): + sysbus = dasbus.connection.SystemMessageBus() + srv = avahi.Avahi(sysbus, lambda: "ok") + self.assertEqual(srv.info(), {'avahi wake up timer': '60.0s [off]', 'service types': [], 'services': {}}) + self.assertEqual(srv.get_controllers(), []) + + try: + # Check that the Avahi daemon is running + subprocess.run([SYSTEMCTL, 'is-active', 'avahi-daemon.service'], check=True) + self.assertFalse(srv._on_kick_avahi()) + except subprocess.CalledProcessError: + self.assertTrue(srv._on_kick_avahi()) + + with self.assertLogs(logger=logging.getLogger(), level='INFO') as captured: + srv._avahi_available(None) + self.assertEqual(len(captured.records), 1) + self.assertEqual(captured.records[0].getMessage(), "avahi-daemon service available, zeroconf supported.") + with self.assertLogs(logger=logging.getLogger(), level='WARN') as captured: + srv._avahi_unavailable(None) + self.assertEqual(len(captured.records), 1) + self.assertEqual(captured.records[0].getMessage(), "avahi-daemon not available, zeroconf not supported.") + srv.kill() + self.assertEqual(srv.info(), {'avahi wake up timer': 'None', 'service types': [], 'services': {}}) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/test-config.py b/test/test-config.py new file mode 100755 index 0000000..1480fc3 --- /dev/null +++ b/test/test-config.py @@ -0,0 +1,264 @@ +#!/usr/bin/python3 +import os +import unittest +from staslib import conf + + +class StasProcessConfUnitTest(unittest.TestCase): + '''Process config unit tests''' + + FNAME = '/tmp/stas-process-config-test' + + @classmethod + def setUpClass(cls): + '''Create a temporary configuration file''' + data = [ + '[Global]\n', + 'tron=true\n', + 'kato=200\n', + 'ip-family=ipv6\n', + '\n', + '[I/O controller connection management]\n', + 'disconnect-scope = joe\n', + 'disconnect-trtypes = bob\n', + 'connect-attempts-on-ncc = 1\n', + '\n', + '[Controllers]\n', + 'controller=transport=tcp;traddr=100.100.100.100;host-iface=enp0s8\n', + 'controller=transport=tcp;traddr=100.100.100.200;host-iface=enp0s7;dhchap-ctrl-secret=super-secret;hdr-digest=true;data-digest=true;nr-io-queues=8;nr-write-queues=6;nr-poll-queues=4;queue-size=400;kato=71;reconnect-delay=13;ctrl-loss-tmo=666;disable-sqflow=true\n', + 'exclude=transport=tcp;traddr=10.10.10.10\n', + ] + with open(StasProcessConfUnitTest.FNAME, 'w') as f: # # pylint: disable=unspecified-encoding + f.writelines(data) + + @classmethod + def tearDownClass(cls): + '''Delete the temporary configuration file''' + if os.path.exists(StasProcessConfUnitTest.FNAME): + os.remove(StasProcessConfUnitTest.FNAME) + + def test_config(self): + '''Check we can read the temporary configuration file''' + + default_conf = { + ('Global', 'tron'): False, + ('Global', 'hdr-digest'): False, + ('Global', 'data-digest'): False, + ('Global', 'kato'): None, # None to let the driver decide the default + ('Global', 'nr-io-queues'): None, # None to let the driver decide the default + ('Global', 'nr-write-queues'): None, # None to let the driver decide the default + ('Global', 'nr-poll-queues'): None, # None to let the driver decide the default + ('Global', 'queue-size'): None, # None to let the driver decide the default + ('Global', 'reconnect-delay'): None, # None to let the driver decide the default + ('Global', 'ctrl-loss-tmo'): None, # None to let the driver decide the default + ('Global', 'disable-sqflow'): None, # None to let the driver decide the default + ('Global', 'ignore-iface'): False, + ('Global', 'ip-family'): (4, 6), + ('Global', 'persistent-connections'): False, # Deprecated + ('Discovery controller connection management', 'persistent-connections'): True, + ('Global', 'pleo'): True, + ('Service Discovery', 'zeroconf'): True, + ('Controllers', 'controller'): list(), + ('Controllers', 'exclude'): list(), + ('I/O controller connection management', 'disconnect-scope'): 'only-stas-connections', + ('I/O controller connection management', 'disconnect-trtypes'): ['tcp'], + ('I/O controller connection management', 'connect-attempts-on-ncc'): 0, + } + + service_conf = conf.SvcConf(default_conf=default_conf) + service_conf.set_conf_file(StasProcessConfUnitTest.FNAME) + self.assertEqual(service_conf.conf_file, StasProcessConfUnitTest.FNAME) + self.assertTrue(service_conf.tron) + self.assertTrue(getattr(service_conf, 'tron')) + self.assertFalse(service_conf.hdr_digest) + self.assertFalse(service_conf.data_digest) + self.assertTrue(service_conf.persistent_connections) + self.assertTrue(service_conf.pleo_enabled) + self.assertEqual(service_conf.disconnect_scope, 'only-stas-connections') + self.assertEqual(service_conf.disconnect_trtypes, ['tcp']) + self.assertFalse(service_conf.ignore_iface) + self.assertIn(6, service_conf.ip_family) + self.assertNotIn(4, service_conf.ip_family) + self.assertEqual(service_conf.kato, 200) + self.assertEqual( + service_conf.get_controllers(), + [ + { + 'transport': 'tcp', + 'traddr': '100.100.100.100', + 'host-iface': 'enp0s8', + }, + { + 'transport': 'tcp', + 'traddr': '100.100.100.200', + 'host-iface': 'enp0s7', + 'dhchap-ctrl-secret': 'super-secret', + 'hdr-digest': True, + 'data-digest': True, + 'nr-io-queues': 8, + 'nr-write-queues': 6, + 'nr-poll-queues': 4, + 'queue-size': 400, + 'kato': 71, + 'reconnect-delay': 13, + 'ctrl-loss-tmo': 666, + 'disable-sqflow': True, + }, + ], + ) + + self.assertEqual(service_conf.get_excluded(), [{'transport': 'tcp', 'traddr': '10.10.10.10'}]) + + stypes = service_conf.stypes + self.assertIn('_nvme-disc._tcp', stypes) + + self.assertTrue(service_conf.zeroconf_enabled) + self.assertEqual(service_conf.connect_attempts_on_ncc, 2) + data = [ + '[I/O controller connection management]\n', + 'disconnect-trtypes = tcp+rdma+fc\n', + 'connect-attempts-on-ncc = hello\n', + ] + with open(StasProcessConfUnitTest.FNAME, 'w') as f: # pylint: disable=unspecified-encoding + f.writelines(data) + service_conf.reload() + self.assertEqual(service_conf.connect_attempts_on_ncc, 0) + self.assertEqual(set(service_conf.disconnect_trtypes), set(['fc', 'tcp', 'rdma'])) + + data = [ + '[Global]\n', + 'ip-family=ipv4\n', + ] + with open(StasProcessConfUnitTest.FNAME, 'w') as f: # pylint: disable=unspecified-encoding + f.writelines(data) + service_conf.reload() + self.assertIn(4, service_conf.ip_family) + self.assertNotIn(6, service_conf.ip_family) + + data = [ + '[Global]\n', + 'ip-family=ipv4+ipv6\n', + ] + with open(StasProcessConfUnitTest.FNAME, 'w') as f: # pylint: disable=unspecified-encoding + f.writelines(data) + service_conf.reload() + self.assertIn(4, service_conf.ip_family) + self.assertIn(6, service_conf.ip_family) + + data = [ + '[Global]\n', + 'ip-family=ipv6+ipv4\n', + ] + with open(StasProcessConfUnitTest.FNAME, 'w') as f: # pylint: disable=unspecified-encoding + f.writelines(data) + service_conf.reload() + self.assertIn(4, service_conf.ip_family) + self.assertIn(6, service_conf.ip_family) + + self.assertRaises(KeyError, service_conf.get_option, 'Babylon', 5) + + +class StasSysConfUnitTest(unittest.TestCase): + '''Sys config unit tests''' + + FNAME_1 = '/tmp/stas-sys-config-test-1' + FNAME_2 = '/tmp/stas-sys-config-test-2' + FNAME_3 = '/tmp/stas-sys-config-test-3' + FNAME_4 = '/tmp/stas-sys-config-test-4' + NQN = 'nqn.2014-08.org.nvmexpress:uuid:9aae2691-b275-4b64-8bfe-5da429a2bab9' + ID = '56529e15-0f3e-4ede-87e2-63932a4adb99' + KEY = 'DHHC-1:03:qwertyuioplkjhgfdsazxcvbnm0123456789QWERTYUIOPLKJHGFDSAZXCVBNM010101010101010101010101010101:' + SYMNAME = 'Bart-Simpson' + + DATA = { + FNAME_1: [ + '[Host]\n', + f'nqn={NQN}\n', + f'id={ID}\n', + f'key={KEY}\n', + f'symname={SYMNAME}\n', + ], + FNAME_2: [ + '[Host]\n', + 'nqn=file:///dev/null\n', + ], + FNAME_3: [ + '[Host]\n', + 'nqn=qnq.2014-08.org.nvmexpress:uuid:9aae2691-b275-4b64-8bfe-5da429a2bab9\n', + f'id={ID}\n', + ], + FNAME_4: [ + '[Host]\n', + 'nqn=file:///some/non/exisiting/file/!@#\n', + 'id=file:///some/non/exisiting/file/!@#\n', + 'symname=file:///some/non/exisiting/file/!@#\n', + ], + } + + @classmethod + def setUpClass(cls): + '''Create a temporary configuration file''' + for file, data in StasSysConfUnitTest.DATA.items(): + with open(file, 'w') as f: # pylint: disable=unspecified-encoding + f.writelines(data) + + @classmethod + def tearDownClass(cls): + '''Delete the temporary configuration file''' + for file in StasSysConfUnitTest.DATA.keys(): + if os.path.exists(file): + os.remove(file) + + def test_config_1(self): + '''Check we can read the temporary configuration file''' + system_conf = conf.SysConf() + system_conf.set_conf_file(StasSysConfUnitTest.FNAME_1) + self.assertEqual(system_conf.conf_file, StasSysConfUnitTest.FNAME_1) + self.assertEqual(system_conf.hostnqn, StasSysConfUnitTest.NQN) + self.assertEqual(system_conf.hostid, StasSysConfUnitTest.ID) + self.assertEqual(system_conf.hostsymname, StasSysConfUnitTest.SYMNAME) + self.assertEqual( + system_conf.as_dict(), + { + 'hostnqn': StasSysConfUnitTest.NQN, + 'hostid': StasSysConfUnitTest.ID, + 'hostkey': StasSysConfUnitTest.KEY, + 'symname': StasSysConfUnitTest.SYMNAME, + }, + ) + + def test_config_2(self): + '''Check we can read from /dev/null or missing 'id' definition''' + system_conf = conf.SysConf() + system_conf.set_conf_file(StasSysConfUnitTest.FNAME_2) + self.assertEqual(system_conf.conf_file, StasSysConfUnitTest.FNAME_2) + self.assertIsNone(system_conf.hostnqn) + self.assertIsNone(system_conf.hostsymname) + + def test_config_3(self): + '''Check we can read an invalid NQN string''' + system_conf = conf.SysConf() + system_conf.set_conf_file(StasSysConfUnitTest.FNAME_3) + self.assertEqual(system_conf.conf_file, StasSysConfUnitTest.FNAME_3) + self.assertRaises(SystemExit, lambda: system_conf.hostnqn) + self.assertEqual(system_conf.hostid, StasSysConfUnitTest.ID) + self.assertIsNone(system_conf.hostsymname) + + def test_config_4(self): + '''Check we can read the temporary configuration file''' + system_conf = conf.SysConf() + system_conf.set_conf_file(StasSysConfUnitTest.FNAME_4) + self.assertEqual(system_conf.conf_file, StasSysConfUnitTest.FNAME_4) + self.assertRaises(SystemExit, lambda: system_conf.hostnqn) + self.assertRaises(SystemExit, lambda: system_conf.hostid) + self.assertIsNone(system_conf.hostsymname) + + def test_config_missing_file(self): + '''Check what happens when conf file is missing''' + system_conf = conf.SysConf() + system_conf.set_conf_file('/just/some/ramdom/file/name') + self.assertIsNone(system_conf.hostsymname) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/test-controller.py b/test/test-controller.py new file mode 100755 index 0000000..d33a6a2 --- /dev/null +++ b/test/test-controller.py @@ -0,0 +1,302 @@ +#!/usr/bin/python3 +import logging +import unittest +from staslib import conf, ctrl, timeparse, trid +from pyfakefs.fake_filesystem_unittest import TestCase + + +class TestController(ctrl.Controller): + def _find_existing_connection(self): + pass + + def _on_aen(self, aen: int): + pass + + def _on_nvme_event(self, nvme_event): + pass + + def reload_hdlr(self): + pass + + +class TestDc(ctrl.Dc): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._connected = True + + class Ctrl: + def __init__(this): + this.name = 'nvme666' + + def connected(this): + return self._connected + + def disconnect(this): + pass + + self._ctrl = Ctrl() + + def _find_existing_connection(self): + pass + + def _on_aen(self, aen: int): + pass + + def _on_nvme_event(self, nvme_event): + pass + + def reload_hdlr(self): + pass + + def set_connected(self, value): + self._connected = value + + def connected(self): + return self._connected + + +class TestStaf: + def is_avahi_reported(self, tid): + return False + + def controller_unresponsive(self, tid): + pass + + @property + def tron(self): + return True + + +stafd_conf_1 = ''' +[Global] +tron=false +hdr-digest=false +data-digest=false +kato=30 +queue-size=128 +reconnect-delay=10 +ctrl-loss-tmo=600 +disable-sqflow=false +ignore-iface=false +ip-family=ipv4+ipv6 +pleo=enabled + +[Service Discovery] +zeroconf=enabled + +[Discovery controller connection management] +persistent-connections=true +zeroconf-connections-persistence=10 seconds +''' + +stafd_conf_2 = ''' +[Discovery controller connection management] +zeroconf-connections-persistence=-1 +''' + + +class Test(TestCase): + '''Unit tests for class Controller''' + + def setUp(self): + self.setUpPyfakefs() + + self.fs.create_file( + '/etc/nvme/hostnqn', contents='nqn.2014-08.org.nvmexpress:uuid:01234567-0123-0123-0123-0123456789ab\n' + ) + self.fs.create_file('/etc/nvme/hostid', contents='01234567-89ab-cdef-0123-456789abcdef\n') + self.fs.create_file( + '/dev/nvme-fabrics', + contents='instance=-1,cntlid=-1,transport=%s,traddr=%s,trsvcid=%s,nqn=%s,queue_size=%d,nr_io_queues=%d,reconnect_delay=%d,ctrl_loss_tmo=%d,keep_alive_tmo=%d,hostnqn=%s,host_traddr=%s,host_iface=%s,hostid=%s,disable_sqflow,hdr_digest,data_digest,nr_write_queues=%d,nr_poll_queues=%d,tos=%d,fast_io_fail_tmo=%d,discovery,dhchap_secret=%s,dhchap_ctrl_secret=%s\n', + ) + + self.NVME_TID = trid.TID( + { + 'transport': 'tcp', + 'traddr': '10.10.10.10', + 'subsysnqn': 'nqn.1988-11.com.dell:SFSS:2:20220208134025e8', + 'trsvcid': '8009', + 'host-traddr': '1.2.3.4', + 'host-iface': 'wlp0s20f3', + } + ) + + default_conf = { + ('Global', 'tron'): False, + ('Global', 'hdr-digest'): False, + ('Global', 'data-digest'): False, + ('Global', 'kato'): None, # None to let the driver decide the default + ('Global', 'queue-size'): None, # None to let the driver decide the default + ('Global', 'reconnect-delay'): None, # None to let the driver decide the default + ('Global', 'ctrl-loss-tmo'): None, # None to let the driver decide the default + ('Global', 'disable-sqflow'): None, # None to let the driver decide the default + ('Global', 'persistent-connections'): True, + ('Discovery controller connection management', 'persistent-connections'): True, + ('Discovery controller connection management', 'zeroconf-connections-persistence'): timeparse.timeparse( + '72hours' + ), + ('Global', 'ignore-iface'): False, + ('Global', 'ip-family'): (4, 6), + ('Global', 'pleo'): True, + ('Service Discovery', 'zeroconf'): True, + ('Controllers', 'controller'): list(), + ('Controllers', 'exclude'): list(), + } + + self.stafd_conf_file1 = '/etc/stas/stafd1.conf' + self.fs.create_file(self.stafd_conf_file1, contents=stafd_conf_1) + + self.stafd_conf_file2 = '/etc/stas/stafd2.conf' + self.fs.create_file(self.stafd_conf_file2, contents=stafd_conf_2) + + self.svcconf = conf.SvcConf(default_conf=default_conf) + self.svcconf.set_conf_file(self.stafd_conf_file1) + + def tearDown(self): + pass + + def test_cannot_instantiate_concrete_classes_if_abstract_method_are_not_implemented(self): + # Make sure we can't instantiate the ABC directly (Abstract Base Class). + class Controller(ctrl.Controller): + pass + + self.assertRaises(TypeError, lambda: ctrl.Controller(tid=self.NVME_TID)) + + def test_get_device(self): + controller = TestController(tid=self.NVME_TID, service=TestStaf()) + self.assertEqual(controller._connect_attempts, 0) + controller._try_to_connect() + self.assertEqual(controller._connect_attempts, 1) + self.assertEqual( + controller.id, "(tcp, 10.10.10.10, 8009, nqn.1988-11.com.dell:SFSS:2:20220208134025e8, wlp0s20f3, 1.2.3.4)" + ) + # raise Exception(controller._connect_op) + self.assertEqual( + str(controller.tid), + "(tcp, 10.10.10.10, 8009, nqn.1988-11.com.dell:SFSS:2:20220208134025e8, wlp0s20f3, 1.2.3.4)", + ) + self.assertEqual(controller.device, 'nvme?') + self.assertEqual( + controller.controller_id_dict(), + { + 'transport': 'tcp', + 'traddr': '10.10.10.10', + 'trsvcid': '8009', + 'host-traddr': '1.2.3.4', + 'host-iface': 'wlp0s20f3', + 'subsysnqn': 'nqn.1988-11.com.dell:SFSS:2:20220208134025e8', + 'device': 'nvme?', + }, + ) + + self.assertEqual( + controller.info(), + { + 'transport': 'tcp', + 'traddr': '10.10.10.10', + 'subsysnqn': 'nqn.1988-11.com.dell:SFSS:2:20220208134025e8', + 'trsvcid': '8009', + 'host-traddr': '1.2.3.4', + 'host-iface': 'wlp0s20f3', + 'device': 'nvme?', + 'connect attempts': '1', + 'retry connect timer': '60.0s [off]', + 'connect operation': "{'fail count': 0, 'completed': False, 'alive': True}", + }, + ) + self.assertEqual( + controller.details(), + { + 'dctype': '', + 'cntrltype': '', + 'connected': 'False', + 'transport': 'tcp', + 'traddr': '10.10.10.10', + 'trsvcid': '8009', + 'host-traddr': '1.2.3.4', + 'host-iface': 'wlp0s20f3', + 'subsysnqn': 'nqn.1988-11.com.dell:SFSS:2:20220208134025e8', + 'device': 'nvme?', + 'connect attempts': '1', + 'retry connect timer': '60.0s [off]', + 'hostid': '', + 'hostnqn': '', + 'model': '', + 'serial': '', + 'connect operation': "{'fail count': 0, 'completed': False, 'alive': True}", + }, + ) + + # print(controller._connect_op) + self.assertEqual(controller.cancel(), None) + self.assertEqual(controller.kill(), None) + self.assertIsNone(controller.disconnect(lambda *args: None, True)) + + def test_connect(self): + controller = TestController(tid=self.NVME_TID, service=TestStaf()) + self.assertEqual(controller._connect_attempts, 0) + controller._find_existing_connection = lambda: None + with self.assertLogs(logger=logging.getLogger(), level='DEBUG') as captured: + controller._try_to_connect() + self.assertTrue(len(captured.records) > 0) + self.assertTrue( + captured.records[0] + .getMessage() + .startswith( + "Controller._do_connect() - (tcp, 10.10.10.10, 8009, nqn.1988-11.com.dell:SFSS:2:20220208134025e8, wlp0s20f3, 1.2.3.4) Connecting to nvme control with cfg={" + ) + ) + self.assertEqual(controller._connect_attempts, 1) + + def test_dlp_supp_opts_as_string(self): + dlp_supp_opts = 0x7 + opts = ctrl.dlp_supp_opts_as_string(dlp_supp_opts) + self.assertEqual(['EXTDLPES', 'PLEOS', 'ALLSUBES'], opts) + + def test_ncc(self): + dlpe = {'eflags': '4'} + ncc = ctrl.get_ncc(ctrl.get_eflags(dlpe)) + self.assertTrue(ncc) + + dlpe = {} + ncc = ctrl.get_ncc(ctrl.get_eflags(dlpe)) + self.assertFalse(ncc) + + def test_dc(self): + self.svcconf.set_conf_file(self.stafd_conf_file1) + + controller = TestDc(TestStaf(), tid=self.NVME_TID) + controller.set_connected(True) + controller.origin = 'discovered' + + with self.assertLogs(logger=logging.getLogger(), level='DEBUG') as captured: + controller.origin = 'blah' + self.assertEqual(len(captured.records), 1) + self.assertNotEqual(-1, captured.records[0].getMessage().find("Trying to set invalid origin to blah")) + + controller.set_connected(False) + with self.assertLogs(logger=logging.getLogger(), level='DEBUG') as captured: + controller.origin = 'discovered' + self.assertEqual(len(captured.records), 1) + self.assertNotEqual( + -1, captured.records[0].getMessage().find("Controller is not responding. Will be removed by") + ) + + self.svcconf.set_conf_file(self.stafd_conf_file2) + with self.assertLogs(logger=logging.getLogger(), level='DEBUG') as captured: + controller.origin = 'discovered' + self.assertEqual(len(captured.records), 1) + self.assertNotEqual(-1, captured.records[0].getMessage().find("Controller not responding. Retrying...")) + + controller.set_connected(True) + with self.assertLogs(logger=logging.getLogger(), level='DEBUG') as captured: + controller.disconnect(lambda *args: None, keep_connection=False) + self.assertEqual(len(captured.records), 2) + self.assertNotEqual(-1, captured.records[0].getMessage().find("nvme666: keep_connection=False")) + self.assertNotEqual(-1, captured.records[1].getMessage().find("nvme666 - Disconnect initiated")) + + # def test_disconnect(self): + + +if __name__ == '__main__': + unittest.main() diff --git a/test/test-gtimer.py b/test/test-gtimer.py new file mode 100755 index 0000000..8562049 --- /dev/null +++ b/test/test-gtimer.py @@ -0,0 +1,43 @@ +#!/usr/bin/python3 +import unittest +from staslib import gutil + + +class Test(unittest.TestCase): + '''Unit tests for class GTimer''' + + def test_new_timer(self): + tmr = gutil.GTimer(interval_sec=5) + self.assertEqual(tmr.get_timeout(), 5) + self.assertEqual(tmr.time_remaining(), 0) + self.assertEqual(str(tmr), '5.0s [off]') + tmr.set_timeout(new_interval_sec=18) + self.assertEqual(tmr.get_timeout(), 18) + self.assertEqual(tmr.time_remaining(), 0) + + def test_callback(self): + tmr = gutil.GTimer(interval_sec=1, user_cback=lambda: "ok") + self.assertEqual(tmr._callback(), "ok") + tmr.set_callback(user_cback=lambda: "notok") + self.assertEqual(tmr._callback(), "notok") + tmr.kill() + self.assertEqual(tmr._user_cback, None) + self.assertRaises(TypeError, tmr._user_cback) + + def test_start_timer(self): + tmr = gutil.GTimer(interval_sec=1, user_cback=lambda: "ok") + self.assertEqual(str(tmr), '1.0s [off]') + tmr.start() + self.assertNotEqual(tmr.time_remaining(), 0) + self.assertNotEqual(str(tmr), '1.0s [off]') + + def test_clear(self): + tmr = gutil.GTimer(interval_sec=1, user_cback=lambda: "ok") + tmr.start() + tmr.clear() + self.assertEqual(tmr.time_remaining(), 0) + self.assertEqual(str(tmr), '1.0s [0s]') + + +if __name__ == '__main__': + unittest.main() diff --git a/test/test-iputil.py b/test/test-iputil.py new file mode 100755 index 0000000..8f364ad --- /dev/null +++ b/test/test-iputil.py @@ -0,0 +1,66 @@ +#!/usr/bin/python3 +import json +import shutil +import logging +import unittest +import ipaddress +import subprocess +from staslib import iputil, log, trid + +IP = shutil.which('ip') + + +class Test(unittest.TestCase): + '''iputil.py unit tests''' + + def setUp(self): + log.init(syslog=False) + self.logger = logging.getLogger() + self.logger.setLevel(logging.INFO) + + # Retrieve the list of Interfaces and all the associated IP addresses + # using standard bash utility (ip address). We'll use this to make sure + # iputil.get_interface() returns the same data as "ip address". + try: + cmd = [IP, '-j', 'address', 'show'] + p = subprocess.run(cmd, stdout=subprocess.PIPE, check=True) + self.ifaces = json.loads(p.stdout.decode().strip()) + except subprocess.CalledProcessError: + self.ifaces = [] + + def test_get_interface(self): + '''Check that get_interface() returns the right info''' + for iface in self.ifaces: + for addr_entry in iface['addr_info']: + addr = ipaddress.ip_address(addr_entry['local']) + # Link local addresses may appear on more than one interface and therefore cannot be used. + if not addr.is_link_local: + self.assertEqual(iface['ifname'], iputil.get_interface(str(addr))) + + self.assertEqual('', iputil.get_interface('255.255.255.255')) + + def test_remove_invalid_addresses(self): + good_tcp = trid.TID({'transport': 'tcp', 'traddr': '1.1.1.1', 'subsysnqn': '', 'trsvcid': '8009'}) + bad_tcp = trid.TID({'transport': 'tcp', 'traddr': '555.555.555.555', 'subsysnqn': '', 'trsvcid': '8009'}) + any_fc = trid.TID({'transport': 'fc', 'traddr': 'blah', 'subsysnqn': ''}) + bad_trtype = trid.TID({'transport': 'whatever', 'traddr': 'blah', 'subsysnqn': ''}) + + l1 = [ + good_tcp, + bad_tcp, + any_fc, + bad_trtype, + ] + l2 = iputil.remove_invalid_addresses(l1) + + self.assertNotEqual(l1, l2) + + self.assertIn(good_tcp, l2) + self.assertIn(any_fc, l2) # We currently don't check for invalid FC (all FCs are allowed) + + self.assertNotIn(bad_tcp, l2) + self.assertNotIn(bad_trtype, l2) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test-log.py b/test/test-log.py new file mode 100755 index 0000000..2ae1155 --- /dev/null +++ b/test/test-log.py @@ -0,0 +1,101 @@ +#!/usr/bin/python3 +import logging +import unittest +from pyfakefs.fake_filesystem_unittest import TestCase +from staslib import log + + +class StaslibLogTest(TestCase): + '''Test for log.py module''' + + def setUp(self): + self.setUpPyfakefs() + + def test_log_with_systemd_journal(self): + '''Check that we can set the handler to systemd.journal.JournalHandler''' + try: + # We can't proceed with this test if the + # module systemd.journal is not installed. + import systemd.journal # pylint: disable=import-outside-toplevel + except ModuleNotFoundError: + return + + log.init(syslog=True) + + logger = logging.getLogger() + handler = logger.handlers[-1] + + self.assertIsInstance(handler, systemd.journal.JournalHandler) + + self.assertEqual(log.level(), 'INFO') + + log.set_level_from_tron(tron=True) + self.assertEqual(log.level(), 'DEBUG') + log.set_level_from_tron(tron=False) + self.assertEqual(log.level(), 'INFO') + + logger.removeHandler(handler) + handler.close() + + def test_log_with_syslog_handler(self): + '''Check that we can set the handler to logging.handlers.SysLogHandler''' + try: + # The log.py module uses systemd.journal.JournalHandler() as the + # default logging handler (if present). Therefore, in order to force + # log.py to use SysLogHandler as the handler, we need to mock + # systemd.journal.JournalHandler() with an invalid class. + import systemd.journal # pylint: disable=import-outside-toplevel + except ModuleNotFoundError: + original_handler = None + else: + + class MockJournalHandler: + def __new__(cls, *args, **kwargs): + raise ModuleNotFoundError + + original_handler = systemd.journal.JournalHandler + systemd.journal.JournalHandler = MockJournalHandler + + log.init(syslog=True) + + logger = logging.getLogger() + handler = logger.handlers[-1] + + self.assertIsInstance(handler, logging.handlers.SysLogHandler) + + self.assertEqual(log.level(), 'INFO') + + log.set_level_from_tron(tron=True) + self.assertEqual(log.level(), 'DEBUG') + log.set_level_from_tron(tron=False) + self.assertEqual(log.level(), 'INFO') + + logger.removeHandler(handler) + handler.close() + + if original_handler is not None: + # Restore original systemd.journal.JournalHandler() + systemd.journal.JournalHandler = original_handler + + def test_log_with_stdout(self): + '''Check that we can set the handler to logging.StreamHandler (i.e. stdout)''' + log.init(syslog=False) + + logger = logging.getLogger() + handler = logger.handlers[-1] + + self.assertIsInstance(handler, logging.StreamHandler) + + self.assertEqual(log.level(), 'DEBUG') + + log.set_level_from_tron(tron=True) + self.assertEqual(log.level(), 'DEBUG') + log.set_level_from_tron(tron=False) + self.assertEqual(log.level(), 'INFO') + + logger.removeHandler(handler) + handler.close() + + +if __name__ == '__main__': + unittest.main() diff --git a/test/test-nvme_options.py b/test/test-nvme_options.py new file mode 100755 index 0000000..428f22a --- /dev/null +++ b/test/test-nvme_options.py @@ -0,0 +1,61 @@ +#!/usr/bin/python3 +import os +import logging +import unittest +from staslib import conf, log +from pyfakefs.fake_filesystem_unittest import TestCase + + +class Test(TestCase): + """Unit tests for class NvmeOptions""" + + def setUp(self): + self.setUpPyfakefs() + log.init(syslog=False) + self.logger = logging.getLogger() + self.logger.setLevel(logging.INFO) + + def tearDown(self): + # No longer need self.tearDownPyfakefs() + pass + + def test_fabrics_empty_file(self): + self.assertFalse(os.path.exists("/dev/nvme-fabrics")) + # TODO: this is a bug + self.fs.create_file("/dev/nvme-fabrics") + self.assertTrue(os.path.exists('/dev/nvme-fabrics')) + nvme_options = conf.NvmeOptions() + self.assertIsInstance(nvme_options.discovery_supp, bool) + self.assertIsInstance(nvme_options.host_iface_supp, bool) + del nvme_options + + def test_fabrics_wrong_file(self): + self.assertFalse(os.path.exists("/dev/nvme-fabrics")) + self.fs.create_file("/dev/nvme-fabrics", contents="blah") + self.assertTrue(os.path.exists('/dev/nvme-fabrics')) + nvme_options = conf.NvmeOptions() + self.assertIsInstance(nvme_options.discovery_supp, bool) + self.assertIsInstance(nvme_options.host_iface_supp, bool) + del nvme_options + + def test_fabrics_correct_file(self): + self.assertFalse(os.path.exists("/dev/nvme-fabrics")) + self.fs.create_file( + '/dev/nvme-fabrics', contents='host_iface=%s,discovery,dhchap_secret=%s,dhchap_ctrl_secret=%s\n' + ) + self.assertTrue(os.path.exists('/dev/nvme-fabrics')) + nvme_options = conf.NvmeOptions() + self.assertTrue(nvme_options.discovery_supp) + self.assertTrue(nvme_options.host_iface_supp) + self.assertTrue(nvme_options.dhchap_hostkey_supp) + self.assertTrue(nvme_options.dhchap_ctrlkey_supp) + self.assertEqual( + nvme_options.get(), + {'discovery': True, 'host_iface': True, 'dhchap_secret': True, 'dhchap_ctrl_secret': True}, + ) + self.assertTrue(str(nvme_options).startswith("supported options:")) + del nvme_options + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test-service.py b/test/test-service.py new file mode 100755 index 0000000..ef1cb6e --- /dev/null +++ b/test/test-service.py @@ -0,0 +1,71 @@ +#!/usr/bin/python3 +import os +import unittest +from staslib import service +from pyfakefs.fake_filesystem_unittest import TestCase + + +class Args: + def __init__(self): + self.tron = True + self.syslog = True + self.conf_file = '/dev/null' + + +class TestService(service.Service): + def _config_ctrls_finish(self, configured_ctrl_list): + pass + + def _dump_last_known_config(self, controllers): + pass + + def _keep_connections_on_exit(self): + pass + + def _load_last_known_config(self): + return dict() + + +class Test(TestCase): + '''Unit tests for class Service''' + + def setUp(self): + self.setUpPyfakefs() + + os.environ['RUNTIME_DIRECTORY'] = "/run" + self.fs.create_file( + '/etc/nvme/hostnqn', contents='nqn.2014-08.org.nvmexpress:uuid:01234567-0123-0123-0123-0123456789ab\n' + ) + self.fs.create_file('/etc/nvme/hostid', contents='01234567-89ab-cdef-0123-456789abcdef\n') + self.fs.create_file( + '/dev/nvme-fabrics', + contents='instance=-1,cntlid=-1,transport=%s,traddr=%s,trsvcid=%s,nqn=%s,queue_size=%d,nr_io_queues=%d,reconnect_delay=%d,ctrl_loss_tmo=%d,keep_alive_tmo=%d,hostnqn=%s,host_traddr=%s,host_iface=%s,hostid=%s,disable_sqflow,hdr_digest,data_digest,nr_write_queues=%d,nr_poll_queues=%d,tos=%d,fast_io_fail_tmo=%d,discovery,dhchap_secret=%s,dhchap_ctrl_secret=%s\n', + ) + + def test_cannot_instantiate_concrete_classes_if_abstract_method_are_not_implemented(self): + # Make sure we can't instantiate the ABC directly (Abstract Base Class). + class Service(service.Service): + pass + + self.assertRaises(TypeError, lambda: Service(Args(), reload_hdlr=lambda x: x)) + + def test_get_controller(self): + srv = TestService(Args(), default_conf={}, reload_hdlr=lambda x: x) + + self.assertEqual(list(srv.get_controllers()), list()) + self.assertEqual( + srv.get_controller( + transport='tcp', + traddr='10.10.10.10', + trsvcid='8009', + host_traddr='1.2.3.4', + host_iface='wlp0s20f3', + subsysnqn='nqn.1988-11.com.dell:SFSS:2:20220208134025e8', + ), + None, + ) + self.assertEqual(srv.remove_controller(controller=None, success=True), None) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/test-timeparse.py b/test/test-timeparse.py new file mode 100755 index 0000000..1abf1b0 --- /dev/null +++ b/test/test-timeparse.py @@ -0,0 +1,32 @@ +#!/usr/bin/python3 +import unittest +from staslib import timeparse + + +class StasTimeparseUnitTest(unittest.TestCase): + '''Time parse unit tests''' + + def test_timeparse(self): + '''Check that timeparse() converts time spans properly''' + self.assertEqual(timeparse.timeparse('1'), 1) + self.assertEqual(timeparse.timeparse('1s'), 1) + self.assertEqual(timeparse.timeparse('1 sec'), 1) + self.assertEqual(timeparse.timeparse('1 second'), 1) + self.assertEqual(timeparse.timeparse('1 seconds'), 1) + self.assertEqual(timeparse.timeparse('1:01'), 61) + self.assertEqual(timeparse.timeparse('1 day'), 24 * 60 * 60) + self.assertEqual(timeparse.timeparse('1 hour'), 60 * 60) + self.assertEqual(timeparse.timeparse('1 min'), 60) + self.assertEqual(timeparse.timeparse('0.5'), 0.5) + self.assertEqual(timeparse.timeparse('-1'), -1) + self.assertEqual(timeparse.timeparse(':22'), 22) + self.assertEqual(timeparse.timeparse('1 minute, 24 secs'), 84) + self.assertEqual(timeparse.timeparse('1.2 minutes'), 72) + self.assertEqual(timeparse.timeparse('1.2 seconds'), 1.2) + self.assertEqual(timeparse.timeparse('- 1 minute'), -60) + self.assertEqual(timeparse.timeparse('+ 1 minute'), 60) + self.assertIsNone(timeparse.timeparse('blah')) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/test-transport_id.py b/test/test-transport_id.py new file mode 100755 index 0000000..36953dd --- /dev/null +++ b/test/test-transport_id.py @@ -0,0 +1,87 @@ +#!/usr/bin/python3 +import unittest +from staslib import trid + + +class Test(unittest.TestCase): + '''Unit test for class TRID''' + + TRANSPORT = 'tcp' + TRADDR = '10.10.10.10' + OTHER_TRADDR = '1.1.1.1' + SUBSYSNQN = 'nqn.1988-11.com.dell:SFSS:2:20220208134025e8' + TRSVCID = '8009' + HOST_TRADDR = '1.2.3.4' + HOST_IFACE = 'wlp0s20f3' + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.cid = { + 'transport': Test.TRANSPORT, + 'traddr': Test.TRADDR, + 'subsysnqn': Test.SUBSYSNQN, + 'trsvcid': Test.TRSVCID, + 'host-traddr': Test.HOST_TRADDR, + 'host-iface': Test.HOST_IFACE, + } + self.other_cid = { + 'transport': Test.TRANSPORT, + 'traddr': Test.OTHER_TRADDR, + 'subsysnqn': Test.SUBSYSNQN, + 'trsvcid': Test.TRSVCID, + 'host-traddr': Test.HOST_TRADDR, + 'host-iface': Test.HOST_IFACE, + } + + self.tid = trid.TID(self.cid) + self.other_tid = trid.TID(self.other_cid) + + def test_hash(self): + '''Check that a hash exists''' + self.assertIsInstance(self.tid._hash, int) + + def test_transport(self): + '''Check that transport is set''' + self.assertEqual(self.tid.transport, Test.TRANSPORT) + + def test_traddr(self): + '''Check that traddr is set''' + self.assertEqual(self.tid.traddr, Test.TRADDR) + + def test_trsvcid(self): + '''Check that trsvcid is set''' + self.assertEqual(self.tid.trsvcid, Test.TRSVCID) + + def test_host_traddr(self): + '''Check that host_traddr is set''' + self.assertEqual(self.tid.host_traddr, Test.HOST_TRADDR) + + def test_host_iface(self): + '''Check that host_iface is set''' + self.assertEqual(self.tid.host_iface, Test.HOST_IFACE) + + def test_subsysnqn(self): + '''Check that subsysnqn is set''' + self.assertEqual(self.tid.subsysnqn, Test.SUBSYSNQN) + + def test_as_dict(self): + '''Check that a TRID can be converted back to the original Dict it was created with''' + self.assertDictEqual(self.tid.as_dict(), self.cid) + + def test_str(self): + '''Check that a TRID can be represented as a string''' + self.assertTrue(str(self.tid).startswith(f'({Test.TRANSPORT},')) + + def test_eq(self): + '''Check that two TRID objects can be tested for equality''' + self.assertEqual(self.tid, trid.TID(self.cid)) + self.assertFalse(self.tid == 'blah') + + def test_ne(self): + '''Check that two TID objects can be tested for non-equality''' + self.assertNotEqual(self.tid, self.other_tid) + self.assertNotEqual(self.tid, 'hello') + + +if __name__ == '__main__': + unittest.main() diff --git a/test/test-udev.py b/test/test-udev.py new file mode 100755 index 0000000..3798d6c --- /dev/null +++ b/test/test-udev.py @@ -0,0 +1,41 @@ +#!/usr/bin/python3 +import unittest +from staslib import udev + + +class Test(unittest.TestCase): + '''Unit tests for class Udev''' + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + @classmethod + def tearDownClass(cls): + '''Release resources''' + udev.shutdown() + + def test_get_device(self): + dev = udev.UDEV.get_nvme_device('null') + self.assertEqual(dev.device_node, '/dev/null') + + def test_get_bad_device(self): + self.assertIsNone(udev.UDEV.get_nvme_device('bozo')) + + def test_get_key_from_attr(self): + device = udev.UDEV.get_nvme_device('null') + + devname = udev.UDEV.get_key_from_attr(device, 'uevent', 'DEVNAME=', '\n') + self.assertEqual(devname, 'null') + + devname = udev.UDEV.get_key_from_attr(device, 'uevent', 'DEVNAME', '\n') + self.assertEqual(devname, 'null') + + devmode = udev.UDEV.get_key_from_attr(device, 'uevent', 'DEVMODE', '\n') + self.assertEqual(devmode, '0666') + + bogus = udev.UDEV.get_key_from_attr(device, 'bogus', 'BOGUS', '\n') + self.assertEqual(bogus, '') + + +if __name__ == '__main__': + unittest.main() diff --git a/test/test-version.py b/test/test-version.py new file mode 100755 index 0000000..8f62d85 --- /dev/null +++ b/test/test-version.py @@ -0,0 +1,46 @@ +#!/usr/bin/python3 +import unittest +from staslib.version import KernelVersion + + +class VersionUnitTests(unittest.TestCase): + '''Unit tests for class KernelVersion''' + + version = KernelVersion('5.8.0-63-generic') + + def test_str(self): + self.assertIsInstance(str(self.version), str) + + def test_repr(self): + self.assertIsInstance(repr(self.version), str) + + def test_eq(self): + '''Test equality''' + self.assertEqual(self.version, '5.8.0-63') + self.assertNotEqual(self.version, '5.8.0') + + def test_lt(self): + '''Test lower than''' + self.assertTrue(self.version < '5.9') + self.assertFalse(self.version < '5.7') + + def test_le(self): + '''Test lower equal''' + self.assertTrue(self.version <= '5.8.0-63') + self.assertTrue(self.version <= '5.8.1') + self.assertFalse(self.version <= '5.7') + + def test_gt(self): + '''Test greater than''' + self.assertTrue(self.version > '5.8') + self.assertFalse(self.version > '5.9') + + def test_ge(self): + '''Test greater equal''' + self.assertTrue(self.version >= '5.8.0-63') + self.assertTrue(self.version >= '5.7.0') + self.assertFalse(self.version >= '5.9') + + +if __name__ == '__main__': + unittest.main() diff --git a/test/vermin-tools.conf b/test/vermin-tools.conf new file mode 100644 index 0000000..7c2b6c7 --- /dev/null +++ b/test/vermin-tools.conf @@ -0,0 +1,144 @@ +[vermin] +### Quiet mode ### +# It only prints the final versions verdict. +# +#quiet = no + +### Verbosity ### +# Verbosity level 1 to 4. -v, -vv, -vvv, and -vvvv shows increasingly more information. Turned off +# at level 0. +# +#verbose = 0 +verbose = 3 + +### Dump AST node visits ### +# Only for debugging. +# +#print_visits = no + +### Matching target versions ### +# Target version that files must abide by. Can be specified once or twice. +# A '-' can be appended to match target version or smaller, like '3.5-'. +# If not met Vermin will exit with code 1. +# Note that the amount of target versions must match the amount of minimum required versions +# detected. +# +# Examples: +#targets = 2.6- +#targets = 2.3 +# 3,4 +#targets = 2,7 +# 3,9- +targets = 3.8 + +### Concurrent processing ### +# Use N concurrent processes to detect and analyze files. Defaults to 0, meaning all cores +# available. +# +#processes = 0 + +### Ignore incompatible versions and warnings ### +# However, if no compatible versions are found then incompatible versions will be shown in the end +# to not have an absence of results. +# +#ignore_incomp = no + +### Lax mode ### +# It ignores conditionals (if, ternary, for, async for, while, with, try, bool op) on AST traversal, +# which can be useful when minimum versions are detected in conditionals that it is known does not +# affect the results. +# +# Note: It is better to use excludes or `# novermin`/`# novm` in the source code instead. +# +#lax = no + +### Hidden analysis ### +# Analyze 'hidden' files and folders starting with '.' (ignored by default when not specified +# directly). +# +#analyze_hidden = no + +### Tips ### +# Possibly show helpful tips at the end, like those relating to backports or lax mode. +# +#show_tips = yes +show_tips = no + +### Pessimistic mode ### +# Syntax errors are interpreted as the major Python version in use being incompatible. +# +#pessimistic = no + +### Exclusions ### +# Exclude full names, like 'email.parser.FeedParser', from analysis. Useful to ignore conditional +# logic that can trigger incompatible results. It's more fine grained than lax mode. +# +# Exclude 'foo.bar.baz' module/member: foo.bar.baz +# Exclude 'foo' kwarg: somemodule.func(foo) +# Exclude 'bar' codecs error handler: ceh=bar +# Exclude 'baz' codecs encoding: ce=baz +# +# Example exclusions: +#exclusions = +# email.parser.FeedParser +# argparse.ArgumentParser(allow_abbrev) +exclusions = + importlib.resources + importlib.resources.files + importlib_resources + importlib_resources.files + +### Backports ### +# Some features are sometimes backported into packages, in repositories such as PyPi, that are +# widely used but aren't in the standard language. If such a backport is specified as being used, +# the results will reflect that instead. +# +# Get full list via `--help`. +# +# Example backports: +#backports = +# typing +# argparse + +### Features ### +# Some features are disabled by default due to being unstable but can be enabled explicitly. +# +# Get full list via `--help`. +# +# Example features: +#features = +# fstring-self-doc + +### Format ### +# Format to show results and output in. +# +# Get full list via `--help`. +# +#format = default + +### Annotations evaluation ### +# Instructs parser that annotations will be manually evaluated in code, which changes minimum +# versions in certain cases. Otherwise, function and variable annotations are not evaluated at +# definition time. Apply this argument if code uses `typing.get_type_hints` or +# `eval(obj.__annotations__)` or otherwise forces evaluation of annotations. +# +#eval_annotations = no + +### Violations ### +# +#only_show_violations = no +only_show_violations = yes + +### Parse comments ### +# Whether or not to parse comments, searching for "# novm" and "# novermin" to exclude anslysis of +# specific lines. If these comments aren't used in a particular code base, not parsing them can +# sometimes yield a speedup of 30-40%+. +# +#parse_comments = yes +parse_comments = no + +### Scan symlink folders ### +# Scan symlinks to folders to include in analysis. Symlinks to non-folders or top-level folders will +# always be scanned. +# +#scan_symlink_folders = no diff --git a/test/vermin.conf b/test/vermin.conf new file mode 100644 index 0000000..4dfc43f --- /dev/null +++ b/test/vermin.conf @@ -0,0 +1,144 @@ +[vermin] +### Quiet mode ### +# It only prints the final versions verdict. +# +#quiet = no + +### Verbosity ### +# Verbosity level 1 to 4. -v, -vv, -vvv, and -vvvv shows increasingly more information. Turned off +# at level 0. +# +#verbose = 0 +verbose = 3 + +### Dump AST node visits ### +# Only for debugging. +# +#print_visits = no + +### Matching target versions ### +# Target version that files must abide by. Can be specified once or twice. +# A '-' can be appended to match target version or smaller, like '3.5-'. +# If not met Vermin will exit with code 1. +# Note that the amount of target versions must match the amount of minimum required versions +# detected. +# +# Examples: +#targets = 2.6- +#targets = 2.3 +# 3,4 +#targets = 2,7 +# 3,9- +targets = 3.6 + +### Concurrent processing ### +# Use N concurrent processes to detect and analyze files. Defaults to 0, meaning all cores +# available. +# +#processes = 0 + +### Ignore incompatible versions and warnings ### +# However, if no compatible versions are found then incompatible versions will be shown in the end +# to not have an absence of results. +# +#ignore_incomp = no + +### Lax mode ### +# It ignores conditionals (if, ternary, for, async for, while, with, try, bool op) on AST traversal, +# which can be useful when minimum versions are detected in conditionals that it is known does not +# affect the results. +# +# Note: It is better to use excludes or `# novermin`/`# novm` in the source code instead. +# +#lax = no + +### Hidden analysis ### +# Analyze 'hidden' files and folders starting with '.' (ignored by default when not specified +# directly). +# +#analyze_hidden = no + +### Tips ### +# Possibly show helpful tips at the end, like those relating to backports or lax mode. +# +#show_tips = yes +show_tips = no + +### Pessimistic mode ### +# Syntax errors are interpreted as the major Python version in use being incompatible. +# +#pessimistic = no + +### Exclusions ### +# Exclude full names, like 'email.parser.FeedParser', from analysis. Useful to ignore conditional +# logic that can trigger incompatible results. It's more fine grained than lax mode. +# +# Exclude 'foo.bar.baz' module/member: foo.bar.baz +# Exclude 'foo' kwarg: somemodule.func(foo) +# Exclude 'bar' codecs error handler: ceh=bar +# Exclude 'baz' codecs encoding: ce=baz +# +# Example exclusions: +#exclusions = +# email.parser.FeedParser +# argparse.ArgumentParser(allow_abbrev) +exclusions = + importlib.resources + importlib.resources.files + importlib_resources + importlib_resources.files + +### Backports ### +# Some features are sometimes backported into packages, in repositories such as PyPi, that are +# widely used but aren't in the standard language. If such a backport is specified as being used, +# the results will reflect that instead. +# +# Get full list via `--help`. +# +# Example backports: +#backports = +# typing +# argparse + +### Features ### +# Some features are disabled by default due to being unstable but can be enabled explicitly. +# +# Get full list via `--help`. +# +# Example features: +#features = +# fstring-self-doc + +### Format ### +# Format to show results and output in. +# +# Get full list via `--help`. +# +#format = default + +### Annotations evaluation ### +# Instructs parser that annotations will be manually evaluated in code, which changes minimum +# versions in certain cases. Otherwise, function and variable annotations are not evaluated at +# definition time. Apply this argument if code uses `typing.get_type_hints` or +# `eval(obj.__annotations__)` or otherwise forces evaluation of annotations. +# +#eval_annotations = no + +### Violations ### +# +#only_show_violations = no +only_show_violations = yes + +### Parse comments ### +# Whether or not to parse comments, searching for "# novm" and "# novermin" to exclude anslysis of +# specific lines. If these comments aren't used in a particular code base, not parsing them can +# sometimes yield a speedup of 30-40%+. +# +#parse_comments = yes +parse_comments = no + +### Scan symlink folders ### +# Scan symlinks to folders to include in analysis. Symlinks to non-folders or top-level folders will +# always be scanned. +# +#scan_symlink_folders = no diff --git a/usr/lib/systemd/system/meson.build b/usr/lib/systemd/system/meson.build new file mode 100644 index 0000000..0076b01 --- /dev/null +++ b/usr/lib/systemd/system/meson.build @@ -0,0 +1,37 @@ +# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). +# +# Authors: Martin Belanger <Martin.Belanger@dell.com> +# +sd_unit_dir = prefix / 'lib' / 'systemd' / 'system' + +configure_file( + input: 'stafd.in.service', + output: 'stafd.service', + install_dir: sd_unit_dir, + configuration: conf, +) + +configure_file( + input: 'stacd.in.service', + output: 'stacd.service', + install_dir: sd_unit_dir, + configuration: conf, +) + +configure_file( + input: 'stas-config@.service', + output: 'stas-config@.service', + install_dir: sd_unit_dir, + copy: true, +) + +configure_file( + input: 'stas-config.target', + output: 'stas-config.target', + install_dir: sd_unit_dir, + copy: true, +) diff --git a/usr/lib/systemd/system/stacd.in.service b/usr/lib/systemd/system/stacd.in.service new file mode 100644 index 0000000..77a4ad5 --- /dev/null +++ b/usr/lib/systemd/system/stacd.in.service @@ -0,0 +1,32 @@ +# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). +# +[Unit] +Description=STorage Appliance Connector (STAC) +Documentation=man:stacd.service(8) man:stacd(8) +Wants=modprobe@nvme_fabrics.service modprobe@nvme_tcp.service network.target stas-config.target +After=modprobe@nvme_fabrics.service modprobe@nvme_tcp.service network.target stas-config.target + +# Check that the nvme-tcp kernel module was previously +# loaded by checking for the presence of /dev/nvme-fabrics. +AssertPathExists=/dev/nvme-fabrics + +[Service] +Type=dbus +BusName=@STACD_DBUS_NAME@ +SyslogIdentifier=stacd + +ExecStart=/usr/bin/python3 -u /usr/sbin/stacd --syslog +ExecReload=/bin/kill -HUP $MAINPID + +# Run-time directory: /run/stacd +# Cache directory: /var/cache/stacd +RuntimeDirectory=stacd +CacheDirectory=stacd +RuntimeDirectoryPreserve=yes + +[Install] +WantedBy=multi-user.target diff --git a/usr/lib/systemd/system/stafd.in.service b/usr/lib/systemd/system/stafd.in.service new file mode 100644 index 0000000..01ddc2b --- /dev/null +++ b/usr/lib/systemd/system/stafd.in.service @@ -0,0 +1,35 @@ +# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). +# + +# Note that stafd can run w/o the avahi-daemon. However, if the avahi-daemon +# is present, stafd should start after it for proper process sequencing. +[Unit] +Description=STorage Appliance Finder (STAF) +Documentation=man:stafd.service(8) man:stafd(8) +Wants=modprobe@nvme_fabrics.service modprobe@nvme_tcp.service network.target stas-config.target +After=modprobe@nvme_fabrics.service modprobe@nvme_tcp.service network.target stas-config.target avahi-daemon.service + +# Check that the nvme-tcp kernel module was previously +# loaded by checking for the presence of /dev/nvme-fabrics. +AssertPathExists=/dev/nvme-fabrics + +[Service] +Type=dbus +BusName=@STAFD_DBUS_NAME@ +SyslogIdentifier=stafd + +ExecStart=/usr/bin/python3 -u /usr/sbin/stafd --syslog +ExecReload=/bin/kill -HUP $MAINPID + +# Run-time directory: /run/stafd +# Cache directory: /var/cache/stafd +RuntimeDirectory=stafd +CacheDirectory=stafd +RuntimeDirectoryPreserve=yes + +[Install] +WantedBy=multi-user.target diff --git a/usr/lib/systemd/system/stas-config.target b/usr/lib/systemd/system/stas-config.target new file mode 100644 index 0000000..af6d339 --- /dev/null +++ b/usr/lib/systemd/system/stas-config.target @@ -0,0 +1,13 @@ +# Copyright (c) 2022, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). +# +[Unit] +Description=Configuration generator for stacd.service and stafd.service +Documentation=man:stas-config.target(8) +Wants=stas-config@hostnqn.service +Wants=stas-config@hostid.service +PartOf=stacd.service +PartOf=stafd.service diff --git a/usr/lib/systemd/system/stas-config@.service b/usr/lib/systemd/system/stas-config@.service new file mode 100644 index 0000000..f070a3c --- /dev/null +++ b/usr/lib/systemd/system/stas-config@.service @@ -0,0 +1,17 @@ +# Copyright (c) 2022, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). +# +[Unit] +Description=nvme-stas /etc/nvme/%i auto-generation +Documentation=man:stas-config@.service(8) +ConditionFileNotEmpty=|!/etc/nvme/%i + +[Service] +Type=oneshot +ExecStart=/usr/bin/stasadm %i -f /etc/nvme/%i + +[Install] +WantedBy=stas-config.target diff --git a/utils/mk-discovery-conf.py b/utils/mk-discovery-conf.py new file mode 100755 index 0000000..a13c48e --- /dev/null +++ b/utils/mk-discovery-conf.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 + +# USAGE: stafctl ls | ./mk-discovery-conf.py + +import sys + +KEYS = [ + ('transport', None), + ('traddr', None), + ('subsysnqn', 'nqn'), + ('host-iface', None), + ('host-traddr', None), +] + +for ctrl in eval(sys.stdin.read()): + print(f"{' '.join([f'--{kout or kin}={ctrl[kin]}' for kin,kout in KEYS if ctrl[kin] != ''])}") diff --git a/utils/nvmet/loop.conf b/utils/nvmet/loop.conf new file mode 100644 index 0000000..79f8b21 --- /dev/null +++ b/utils/nvmet/loop.conf @@ -0,0 +1,20 @@ +# Config file format: Python, i.e. dict(), list(), int, str, etc... +# port ids (id) are integers 0...N +# namespaces are integers 0..N +# subsysnqn can be integers or strings +{ + 'ports': [ + { + 'id': 1, + 'trtype': 'loop', + } + ], + + 'subsystems': [ + { + 'subsysnqn': 'enterprise', + 'port': 1, + 'namespaces': [1] + }, + ] +} diff --git a/utils/nvmet/nvmet.conf b/utils/nvmet/nvmet.conf new file mode 100644 index 0000000..d3288e9 --- /dev/null +++ b/utils/nvmet/nvmet.conf @@ -0,0 +1,35 @@ +# Config file format: Python, i.e. dict(), list(), int, str, etc... +# port ids (id) are integers 0...N +# namespaces are integers 0..N +# subsysnqn can be integers or strings +{ + 'ports': [ + { + 'id': 1, + 'adrfam': 'ipv6', + 'traddr': '::', + #'adrfam': 'ipv4', + #'traddr': '0.0.0.0', + 'trsvcid': 8009, + 'trtype': 'tcp', + } + ], + + 'subsystems': [ + { + 'subsysnqn': 'nqn.1988-11.com.dell:PowerSANxxx:01:20210225100113-454f73093ceb4847a7bdfc6e34ae8e28', + 'port': 1, + 'namespaces': [1] + }, + { + 'subsysnqn': 'starfleet', + 'port': 1, + 'namespaces': [1, 2] + }, + { + 'subsysnqn': 'klingons', + 'port': 1, + 'namespaces': [1, 2, 3] + }, + ] +} diff --git a/utils/nvmet/nvmet.py b/utils/nvmet/nvmet.py new file mode 100755 index 0000000..baf6560 --- /dev/null +++ b/utils/nvmet/nvmet.py @@ -0,0 +1,405 @@ +#!/usr/bin/python3 +# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See the LICENSE file for details. +# +# This file is part of NVMe STorage Appliance Services (nvme-stas). +# +# Authors: Martin Belanger <Martin.Belanger@dell.com> + +# PYTHON_ARGCOMPLETE_OK + +import os +import sys +import pprint +import pathlib +import subprocess +from argparse import ArgumentParser + +VERSION = 1.0 +DEFAULT_CONFIG_FILE = './nvmet.conf' + + +class Fore: + RED = '\033[31m' + GREEN = '\033[32m' + + +class Style: + RESET_ALL = '\033[0m' + + +def _get_loaded_nvmet_modules(): + try: + cp = subprocess.run('/usr/sbin/lsmod', capture_output=True, text=True) + except TypeError: + # For older Python versions that don't support "capture_output" or "text" + cp = subprocess.run('/usr/sbin/lsmod', stdout=subprocess.PIPE, universal_newlines=True) + + if cp.returncode != 0 or not cp.stdout: + return [] + + output = [] + lines = cp.stdout.split('\n') + for line in lines: + if 'nvmet_' in line: + module = line.split()[0] + for end in ('loop', 'tcp', 'fc', 'rdma'): + if module.endswith(end): + output.append(module) + break + + return output + + +def _runcmd(cmd: list, quiet=False): + if not quiet: + print(' '.join(cmd)) + if args.dry_run: + return + subprocess.run(cmd) + + +def _modprobe(module: str, args: list = None, quiet=False): + cmd = ['/usr/sbin/modprobe', module] + if args: + cmd.extend(args) + _runcmd(cmd, quiet) + + +def _mkdir(dname: str): + print(f'mkdir -p "{dname}"') + if args.dry_run: + return + pathlib.Path(dname).mkdir(parents=True, exist_ok=True) + + +def _echo(value, fname: str): + print(f'echo -n "{value}" > "{fname}"') + if args.dry_run: + return + with open(fname, 'w') as f: + f.write(str(value)) + + +def _symlink(port: str, subsysnqn: str): + print( + f'$( cd "/sys/kernel/config/nvmet/ports/{port}/subsystems" && ln -s "../../../subsystems/{subsysnqn}" "{subsysnqn}" )' + ) + if args.dry_run: + return + target = os.path.join('/sys/kernel/config/nvmet/subsystems', subsysnqn) + link = pathlib.Path(os.path.join('/sys/kernel/config/nvmet/ports', port, 'subsystems', subsysnqn)) + link.symlink_to(target) + + +def _create_subsystem(subsysnqn: str) -> str: + print(f'###{Fore.GREEN} Create subsystem: {subsysnqn}{Style.RESET_ALL}') + dname = os.path.join('/sys/kernel/config/nvmet/subsystems/', subsysnqn) + _mkdir(dname) + _echo(1, os.path.join(dname, 'attr_allow_any_host')) + return dname + + +def _create_namespace(subsysnqn: str, id: str, node: str) -> str: + print(f'###{Fore.GREEN} Add namespace: {id}{Style.RESET_ALL}') + dname = os.path.join('/sys/kernel/config/nvmet/subsystems/', subsysnqn, 'namespaces', id) + _mkdir(dname) + _echo(node, os.path.join(dname, 'device_path')) + _echo(1, os.path.join(dname, 'enable')) + return dname + + +def _args_valid(id, traddr, trsvcid, trtype, adrfam): + if None in (id, trtype): + return False + + if trtype != 'loop' and None in (traddr, trsvcid, adrfam): + return False + + return True + + +def _create_port(port: str, traddr: str, trsvcid: str, trtype: str, adrfam: str): + '''@param port: This is a nvmet port and not a tcp port.''' + print(f'###{Fore.GREEN} Create port: {port} -> {traddr}:{trsvcid}{Style.RESET_ALL}') + dname = os.path.join('/sys/kernel/config/nvmet/ports', port) + _mkdir(dname) + _echo(trtype, os.path.join(dname, 'addr_trtype')) + if traddr: + _echo(traddr, os.path.join(dname, 'addr_traddr')) + if trsvcid: + _echo(trsvcid, os.path.join(dname, 'addr_trsvcid')) + if adrfam: + _echo(adrfam, os.path.join(dname, 'addr_adrfam')) + + +def _map_subsystems_to_ports(subsystems: list): + print(f'###{Fore.GREEN} Map subsystems to ports{Style.RESET_ALL}') + for subsystem in subsystems: + subsysnqn, port = subsystem.get('subsysnqn'), str(subsystem.get('port')) + if None not in (subsysnqn, port): + _symlink(port, subsysnqn) + + +def _read_config(fname: str) -> dict: + try: + with open(fname) as f: + return eval(f.read()) + except Exception as e: + sys.exit(f'Error reading config file. {e}') + + +def _read_attr_from_file(fname: str) -> str: + try: + with open(fname, 'r') as f: + return f.read().strip('\n') + except Exception as e: + sys.exit(f'Error reading attribute. {e}') + + +################################################################################ + + +def create(args): + # Need to be root to run this script + if not args.dry_run and os.geteuid() != 0: + sys.exit(f'Permission denied. You need root privileges to run {os.path.basename(__file__)}.') + + config = _read_config(args.conf_file) + + print('') + + # Create a dummy null block device (if one doesn't already exist) + dev_node = '/dev/nullb0' + _modprobe('null_blk', ['nr_devices=1']) + + ports = config.get('ports') + if ports is None: + sys.exit(f'Config file "{args.conf_file}" missing a "ports" section') + + subsystems = config.get('subsystems') + if subsystems is None: + sys.exit(f'Config file "{args.conf_file}" missing a "subsystems" section') + + # Extract the list of transport types found in the + # config file and load the corresponding kernel module. + _modprobe('nvmet') + trtypes = {port.get('trtype') for port in ports if port.get('trtype') is not None} + for trtype in trtypes: + if trtype in ('tcp', 'fc', 'rdma'): + _modprobe(f'nvmet_{trtype}') + elif trtype == 'loop': + _modprobe('nvmet_loop') + + for port in ports: + print('') + id, traddr, trsvcid, trtype, adrfam = ( + str(port.get('id')), + port.get('traddr'), + port.get('trsvcid'), + port.get('trtype'), + port.get('adrfam'), + ) + if _args_valid(id, traddr, trsvcid, trtype, adrfam): + _create_port(id, traddr, trsvcid, trtype, adrfam) + else: + print( + f'{Fore.RED}### Config file "{args.conf_file}" error in "ports" section: id={id}, traddr={traddr}, trsvcid={trsvcid}, trtype={trtype}, adrfam={adrfam}{Style.RESET_ALL}' + ) + + for subsystem in subsystems: + print('') + subsysnqn, port, namespaces = ( + subsystem.get('subsysnqn'), + str(subsystem.get('port')), + subsystem.get('namespaces'), + ) + if None not in (subsysnqn, port, namespaces): + _create_subsystem(subsysnqn) + for id in namespaces: + _create_namespace(subsysnqn, str(id), dev_node) + else: + print( + f'{Fore.RED}### Config file "{args.conf_file}" error in "subsystems" section: subsysnqn={subsysnqn}, port={port}, namespaces={namespaces}{Style.RESET_ALL}' + ) + + print('') + _map_subsystems_to_ports(subsystems) + + print('') + + +def clean(args): + # Need to be root to run this script + if not args.dry_run and os.geteuid() != 0: + sys.exit(f'Permission denied. You need root privileges to run {os.path.basename(__file__)}.') + + print('rm -f /sys/kernel/config/nvmet/ports/*/subsystems/*') + for dname in pathlib.Path('/sys/kernel/config/nvmet/ports').glob('*/subsystems/*'): + _runcmd(['rm', '-f', str(dname)], quiet=True) + + print('rmdir /sys/kernel/config/nvmet/ports/*') + for dname in pathlib.Path('/sys/kernel/config/nvmet/ports').glob('*'): + _runcmd(['rmdir', str(dname)], quiet=True) + + print('rmdir /sys/kernel/config/nvmet/subsystems/*/namespaces/*') + for dname in pathlib.Path('/sys/kernel/config/nvmet/subsystems').glob('*/namespaces/*'): + _runcmd(['rmdir', str(dname)], quiet=True) + + print('rmdir /sys/kernel/config/nvmet/subsystems/*') + for dname in pathlib.Path('/sys/kernel/config/nvmet/subsystems').glob('*'): + _runcmd(['rmdir', str(dname)], quiet=True) + + for module in _get_loaded_nvmet_modules(): + _modprobe(module, ['--remove']) + + _modprobe('nvmet', ['--remove']) + _modprobe('null_blk', ['--remove']) + + +def link(args): + port = str(args.port) + subsysnqn = str(args.subnqn) + if not args.dry_run: + if os.geteuid() != 0: + # Need to be root to run this script + sys.exit(f'Permission denied. You need root privileges to run {os.path.basename(__file__)}.') + + symlink = os.path.join('/sys/kernel/config/nvmet/ports', port, 'subsystems', subsysnqn) + if os.path.exists(symlink): + sys.exit(f'Symlink already exists: {symlink}') + + _symlink(port, subsysnqn) + + +def unlink(args): + port = str(args.port) + subsysnqn = str(args.subnqn) + symlink = os.path.join('/sys/kernel/config/nvmet/ports', port, 'subsystems', subsysnqn) + if not args.dry_run: + if os.geteuid() != 0: + # Need to be root to run this script + sys.exit(f'Permission denied. You need root privileges to run {os.path.basename(__file__)}.') + + if not os.path.exists(symlink): + sys.exit(f'No such symlink: {symlink}') + + _runcmd(['rm', symlink]) + + +def ls(args): + ports = list() + for port_path in pathlib.Path('/sys/kernel/config/nvmet/ports').glob('*'): + id = port_path.parts[-1] + port = { + 'id': int(id), + 'traddr': _read_attr_from_file(os.path.join('/sys/kernel/config/nvmet/ports', id, 'addr_traddr')), + 'trsvcid': _read_attr_from_file(os.path.join('/sys/kernel/config/nvmet/ports', id, 'addr_trsvcid')), + 'adrfam': _read_attr_from_file(os.path.join('/sys/kernel/config/nvmet/ports', id, 'addr_adrfam')), + 'trtype': _read_attr_from_file(os.path.join('/sys/kernel/config/nvmet/ports', id, 'addr_trtype')), + } + + ports.append(port) + + subsystems = dict() + for subsystem_path in pathlib.Path('/sys/kernel/config/nvmet/subsystems').glob('*'): + subsysnqn = subsystem_path.parts[-1] + namespaces_path = pathlib.Path(os.path.join('/sys/kernel/config/nvmet/subsystems', subsysnqn, 'namespaces')) + subsystems[subsysnqn] = { + 'port': None, + 'subsysnqn': subsysnqn, + 'namespaces': sorted([int(namespace_path.parts[-1]) for namespace_path in namespaces_path.glob('*')]), + } + + # Find the port that each subsystem is mapped to + for subsystem_path in pathlib.Path('/sys/kernel/config/nvmet/ports').glob('*/subsystems/*'): + subsysnqn = subsystem_path.parts[-1] + if subsysnqn in subsystems: + subsystems[subsysnqn]['port'] = int(subsystem_path.parts[-3]) + + output = { + 'ports': ports, + 'subsystems': list(subsystems.values()), + } + + if sys.version_info < (3, 8): + print(pprint.pformat(output, width=70)) + else: + print(pprint.pformat(output, width=70, sort_dicts=False)) + + print('') + + +################################################################################ + +parser = ArgumentParser(description="Create NVMe-oF Storage Subsystems") +parser.add_argument('-v', '--version', action='store_true', help='Print version, then exit', default=False) + +subparser = parser.add_subparsers(title='Commands', description='valid commands') + +prsr = subparser.add_parser('create', help='Create nvme targets') +prsr.add_argument( + '-f', + '--conf-file', + action='store', + help='Configuration file (default: %(default)s)', + default=DEFAULT_CONFIG_FILE, + type=str, + metavar='FILE', +) +prsr.add_argument( + '-d', '--dry-run', action='store_true', help='Just print what would be done. (default: %(default)s)', default=False +) +prsr.set_defaults(func=create) + +prsr = subparser.add_parser('clean', help='Remove all previously created nvme targets') +prsr.add_argument( + '-d', '--dry-run', action='store_true', help='Just print what would be done. (default: %(default)s)', default=False +) +prsr.set_defaults(func=clean) + +prsr = subparser.add_parser('ls', help='List ports and subsystems') +prsr.set_defaults(func=ls) + +prsr = subparser.add_parser('link', help='Map a subsystem to a port') +prsr.add_argument( + '-d', '--dry-run', action='store_true', help='Just print what would be done. (default: %(default)s)', default=False +) +prsr.add_argument('-p', '--port', action='store', type=int, help='nvmet port', required=True) +prsr.add_argument('-s', '--subnqn', action='store', type=str, help='nvmet subsystem NQN', required=True, metavar='NQN') +prsr.set_defaults(func=link) + +prsr = subparser.add_parser('unlink', help='Unmap a subsystem from a port') +prsr.add_argument( + '-d', '--dry-run', action='store_true', help='Just print what would be done. (default: %(default)s)', default=False +) +prsr.add_argument('-p', '--port', action='store', type=int, help='nvmet port', required=True) +prsr.add_argument('-s', '--subnqn', action='store', type=str, help='nvmet subsystem NQN', required=True, metavar='NQN') +prsr.set_defaults(func=unlink) + + +# ============================= +# Tab-completion. +# MUST BE CALLED BEFORE parser.parse_args() BELOW. +# Ref: https://kislyuk.github.io/argcomplete/ +# +# If you do have argcomplete installed, you also need to run +# "sudo activate-global-python-argcomplete3" to globally activate +# auto-completion. Ref: https://pypi.python.org/pypi/argcomplete#global-completion +try: + import argcomplete + + argcomplete.autocomplete(parser) +except ModuleNotFoundError: + # auto-complete is not necessary for the operation of this script. Just nice to have + pass + +args = parser.parse_args() + +if args.version: + print(f'{os.path.basename(__file__)} {VERSION}') + sys.exit(0) + +# Invoke the sub-command +args.func(args) |
