summaryrefslogtreecommitdiffstats
path: root/src/spdk/examples
diff options
context:
space:
mode:
Diffstat (limited to 'src/spdk/examples')
-rw-r--r--src/spdk/examples/Makefile44
-rw-r--r--src/spdk/examples/accel/Makefile44
-rw-r--r--src/spdk/examples/accel/perf/.gitignore1
-rw-r--r--src/spdk/examples/accel/perf/Makefile47
-rw-r--r--src/spdk/examples/accel/perf/accel_perf.c716
-rw-r--r--src/spdk/examples/bdev/Makefile48
-rw-r--r--src/spdk/examples/bdev/fio_plugin/.gitignore1
-rw-r--r--src/spdk/examples/bdev/fio_plugin/Makefile48
-rw-r--r--src/spdk/examples/bdev/fio_plugin/README.md74
-rw-r--r--src/spdk/examples/bdev/fio_plugin/bdev.conf.in3
-rw-r--r--src/spdk/examples/bdev/fio_plugin/example_config.fio16
-rw-r--r--src/spdk/examples/bdev/fio_plugin/fio_plugin.c838
-rw-r--r--src/spdk/examples/bdev/fio_plugin/full_bench.fio20
-rw-r--r--src/spdk/examples/bdev/hello_world/.gitignore1
-rw-r--r--src/spdk/examples/bdev/hello_world/Makefile44
-rw-r--r--src/spdk/examples/bdev/hello_world/bdev.conf3
-rw-r--r--src/spdk/examples/bdev/hello_world/hello_bdev.c295
-rw-r--r--src/spdk/examples/blob/Makefile44
-rw-r--r--src/spdk/examples/blob/cli/.gitignore1
-rw-r--r--src/spdk/examples/blob/cli/Makefile46
-rw-r--r--src/spdk/examples/blob/cli/README.md65
-rw-r--r--src/spdk/examples/blob/cli/blobcli.c1572
-rw-r--r--src/spdk/examples/blob/hello_world/.gitignore1
-rw-r--r--src/spdk/examples/blob/hello_world/Makefile45
-rw-r--r--src/spdk/examples/blob/hello_world/hello_blob.c498
-rw-r--r--src/spdk/examples/blob/hello_world/hello_blob.json17
-rw-r--r--src/spdk/examples/ioat/Makefile44
-rw-r--r--src/spdk/examples/ioat/perf/.gitignore1
-rw-r--r--src/spdk/examples/ioat/perf/Makefile43
-rw-r--r--src/spdk/examples/ioat/perf/perf.c596
-rw-r--r--src/spdk/examples/ioat/verify/.gitignore1
-rw-r--r--src/spdk/examples/ioat/verify/Makefile43
-rw-r--r--src/spdk/examples/ioat/verify/verify.c521
-rw-r--r--src/spdk/examples/nvme/Makefile47
-rw-r--r--src/spdk/examples/nvme/abort/.gitignore1
-rw-r--r--src/spdk/examples/nvme/abort/Makefile38
-rw-r--r--src/spdk/examples/nvme/abort/abort.c1144
-rw-r--r--src/spdk/examples/nvme/arbitration/.gitignore1
-rw-r--r--src/spdk/examples/nvme/arbitration/Makefile38
-rw-r--r--src/spdk/examples/nvme/arbitration/arbitration.c1158
-rw-r--r--src/spdk/examples/nvme/cmb_copy/.gitignore1
-rw-r--r--src/spdk/examples/nvme/cmb_copy/Makefile38
-rw-r--r--src/spdk/examples/nvme/cmb_copy/cmb_copy.c412
-rw-r--r--src/spdk/examples/nvme/fio_plugin/.gitignore1
-rw-r--r--src/spdk/examples/nvme/fio_plugin/Makefile51
-rw-r--r--src/spdk/examples/nvme/fio_plugin/README.md107
-rw-r--r--src/spdk/examples/nvme/fio_plugin/example_config.fio15
-rw-r--r--src/spdk/examples/nvme/fio_plugin/fio_plugin.c1267
-rw-r--r--src/spdk/examples/nvme/fio_plugin/full_bench.fio40
-rw-r--r--src/spdk/examples/nvme/fio_plugin/mock_sgl_config.fio17
-rw-r--r--src/spdk/examples/nvme/hello_world/.gitignore1
-rw-r--r--src/spdk/examples/nvme/hello_world/Makefile38
-rw-r--r--src/spdk/examples/nvme/hello_world/hello_world.c435
-rw-r--r--src/spdk/examples/nvme/hotplug/.gitignore1
-rw-r--r--src/spdk/examples/nvme/hotplug/Makefile38
-rw-r--r--src/spdk/examples/nvme/hotplug/hotplug.c525
-rw-r--r--src/spdk/examples/nvme/identify/.gitignore1
-rw-r--r--src/spdk/examples/nvme/identify/Makefile44
-rw-r--r--src/spdk/examples/nvme/identify/identify.c1827
-rw-r--r--src/spdk/examples/nvme/nvme_manage/.gitignore1
-rw-r--r--src/spdk/examples/nvme/nvme_manage/Makefile38
-rw-r--r--src/spdk/examples/nvme/nvme_manage/nvme_manage.c1703
-rw-r--r--src/spdk/examples/nvme/perf/.gitignore1
-rw-r--r--src/spdk/examples/nvme/perf/Makefile49
-rw-r--r--src/spdk/examples/nvme/perf/README.md5
-rw-r--r--src/spdk/examples/nvme/perf/perf.c2308
-rw-r--r--src/spdk/examples/nvme/reconnect/.gitignore1
-rw-r--r--src/spdk/examples/nvme/reconnect/Makefile38
-rw-r--r--src/spdk/examples/nvme/reconnect/reconnect.c1185
-rw-r--r--src/spdk/examples/nvmf/Makefile44
-rw-r--r--src/spdk/examples/nvmf/nvmf/.gitignore1
-rw-r--r--src/spdk/examples/nvmf/nvmf/Makefile52
-rw-r--r--src/spdk/examples/nvmf/nvmf/README.md31
-rw-r--r--src/spdk/examples/nvmf/nvmf/nvmf.c905
-rw-r--r--src/spdk/examples/sock/Makefile47
-rw-r--r--src/spdk/examples/sock/hello_world/.gitignore1
-rw-r--r--src/spdk/examples/sock/hello_world/Makefile43
-rw-r--r--src/spdk/examples/sock/hello_world/hello_sock.c442
-rw-r--r--src/spdk/examples/vmd/Makefile44
-rw-r--r--src/spdk/examples/vmd/led/.gitignore1
-rw-r--r--src/spdk/examples/vmd/led/Makefile44
-rw-r--r--src/spdk/examples/vmd/led/led.c214
-rw-r--r--src/spdk/examples/vmd/lsvmd/.gitignore1
-rw-r--r--src/spdk/examples/vmd/lsvmd/Makefile44
-rw-r--r--src/spdk/examples/vmd/lsvmd/lsvmd.c110
85 files changed, 20420 insertions, 0 deletions
diff --git a/src/spdk/examples/Makefile b/src/spdk/examples/Makefile
new file mode 100644
index 000000000..516cf83fc
--- /dev/null
+++ b/src/spdk/examples/Makefile
@@ -0,0 +1,44 @@
+#
+# BSD LICENSE
+#
+# Copyright (c) Intel Corporation.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Intel Corporation nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+DIRS-y += accel bdev blob ioat nvme sock vmd nvmf
+
+.PHONY: all clean $(DIRS-y)
+
+all: $(DIRS-y)
+clean: $(DIRS-y)
+
+include $(SPDK_ROOT_DIR)/mk/spdk.subdirs.mk
diff --git a/src/spdk/examples/accel/Makefile b/src/spdk/examples/accel/Makefile
new file mode 100644
index 000000000..55ede2195
--- /dev/null
+++ b/src/spdk/examples/accel/Makefile
@@ -0,0 +1,44 @@
+#
+# BSD LICENSE
+#
+# Copyright (c) Intel Corporation.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Intel Corporation nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+DIRS-y += perf
+
+.PHONY: all clean $(DIRS-y)
+
+all: $(DIRS-y)
+clean: $(DIRS-y)
+
+include $(SPDK_ROOT_DIR)/mk/spdk.subdirs.mk
diff --git a/src/spdk/examples/accel/perf/.gitignore b/src/spdk/examples/accel/perf/.gitignore
new file mode 100644
index 000000000..83fe5432f
--- /dev/null
+++ b/src/spdk/examples/accel/perf/.gitignore
@@ -0,0 +1 @@
+accel_perf
diff --git a/src/spdk/examples/accel/perf/Makefile b/src/spdk/examples/accel/perf/Makefile
new file mode 100644
index 000000000..b28f7c412
--- /dev/null
+++ b/src/spdk/examples/accel/perf/Makefile
@@ -0,0 +1,47 @@
+#
+# BSD LICENSE
+#
+# Copyright (c) Intel Corporation.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Intel Corporation nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+include $(SPDK_ROOT_DIR)/mk/spdk.modules.mk
+
+APP = accel_perf
+
+C_SRCS := accel_perf.c
+
+SPDK_LIB_LIST = $(ALL_MODULES_LIST)
+SPDK_LIB_LIST += $(EVENT_BDEV_SUBSYSTEM)
+SPDK_LIB_LIST += bdev accel event thread util conf trace \
+ log jsonrpc json rpc sock notify
+
+include $(SPDK_ROOT_DIR)/mk/spdk.app.mk
diff --git a/src/spdk/examples/accel/perf/accel_perf.c b/src/spdk/examples/accel/perf/accel_perf.c
new file mode 100644
index 000000000..2093253c1
--- /dev/null
+++ b/src/spdk/examples/accel/perf/accel_perf.c
@@ -0,0 +1,716 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+#include "spdk/thread.h"
+#include "spdk/env.h"
+#include "spdk/event.h"
+#include "spdk/log.h"
+#include "spdk/string.h"
+#include "spdk/accel_engine.h"
+#include "spdk/crc32.h"
+#include "spdk/util.h"
+
+#define DATA_PATTERN 0x5a
+#define ALIGN_4K 0x1000
+
+static uint64_t g_tsc_rate;
+static uint64_t g_tsc_us_rate;
+static uint64_t g_tsc_end;
+static int g_xfer_size_bytes = 4096;
+static int g_queue_depth = 32;
+static int g_time_in_sec = 5;
+static uint32_t g_crc32c_seed = 0;
+static int g_fail_percent_goal = 0;
+static uint8_t g_fill_pattern = 255;
+static bool g_verify = false;
+static const char *g_workload_type = NULL;
+static enum accel_capability g_workload_selection;
+static struct worker_thread *g_workers = NULL;
+static int g_num_workers = 0;
+static pthread_mutex_t g_workers_lock = PTHREAD_MUTEX_INITIALIZER;
+uint64_t g_capabilites;
+struct ap_task;
+
+struct worker_thread {
+ struct spdk_io_channel *ch;
+ uint64_t xfer_completed;
+ uint64_t xfer_failed;
+ uint64_t injected_miscompares;
+ uint64_t current_queue_depth;
+ TAILQ_HEAD(, ap_task) tasks;
+ struct worker_thread *next;
+ unsigned core;
+ struct spdk_thread *thread;
+ bool is_draining;
+ struct spdk_poller *is_draining_poller;
+ struct spdk_poller *stop_poller;
+};
+
+struct ap_task {
+ void *src;
+ void *dst;
+ void *dst2;
+ struct worker_thread *worker;
+ int status;
+ int expected_status; /* used for compare */
+ TAILQ_ENTRY(ap_task) link;
+};
+
+static void
+dump_user_config(struct spdk_app_opts *opts)
+{
+ printf("SPDK Configuration:\n");
+ printf("Core mask: %s\n\n", opts->reactor_mask);
+ printf("Accel Perf Configuration:\n");
+ printf("Workload Type: %s\n", g_workload_type);
+ if (g_workload_selection == ACCEL_CRC32C) {
+ printf("CRC-32C seed: %u\n", g_crc32c_seed);
+ } else if (g_workload_selection == ACCEL_FILL) {
+ printf("Fill pattern: 0x%x\n", g_fill_pattern);
+ } else if ((g_workload_selection == ACCEL_COMPARE) && g_fail_percent_goal > 0) {
+ printf("Failure inject: %u percent\n", g_fail_percent_goal);
+ }
+ printf("Transfer size: %u bytes\n", g_xfer_size_bytes);
+ printf("Queue depth: %u\n", g_queue_depth);
+ printf("Run time: %u seconds\n", g_time_in_sec);
+ printf("Verify: %s\n\n", g_verify ? "Yes" : "No");
+}
+
+static void
+usage(void)
+{
+ printf("accel_perf options:\n");
+ printf("\t[-h help message]\n");
+ printf("\t[-q queue depth]\n");
+ printf("\t[-n number of channels]\n");
+ printf("\t[-o transfer size in bytes]\n");
+ printf("\t[-t time in seconds]\n");
+ printf("\t[-w workload type must be one of these: copy, fill, crc32c, compare, dualcast\n");
+ printf("\t[-s for crc32c workload, use this seed value (default 0)\n");
+ printf("\t[-P for compare workload, percentage of operations that should miscompare (percent, default 0)\n");
+ printf("\t[-f for fill workload, use this BYTE value (default 255)\n");
+ printf("\t[-y verify result if this switch is on]\n");
+}
+
+static int
+parse_args(int argc, char *argv)
+{
+ switch (argc) {
+ case 'f':
+ g_fill_pattern = (uint8_t)spdk_strtol(optarg, 10);
+ break;
+ case 'o':
+ g_xfer_size_bytes = spdk_strtol(optarg, 10);
+ break;
+ case 'P':
+ g_fail_percent_goal = spdk_strtol(optarg, 10);
+ break;
+ case 'q':
+ g_queue_depth = spdk_strtol(optarg, 10);
+ break;
+ case 's':
+ g_crc32c_seed = spdk_strtol(optarg, 10);
+ break;
+ case 't':
+ g_time_in_sec = spdk_strtol(optarg, 10);
+ break;
+ case 'y':
+ g_verify = true;
+ break;
+ case 'w':
+ g_workload_type = optarg;
+ if (!strcmp(g_workload_type, "copy")) {
+ g_workload_selection = ACCEL_COPY;
+ } else if (!strcmp(g_workload_type, "fill")) {
+ g_workload_selection = ACCEL_FILL;
+ } else if (!strcmp(g_workload_type, "crc32c")) {
+ g_workload_selection = ACCEL_CRC32C;
+ } else if (!strcmp(g_workload_type, "compare")) {
+ g_workload_selection = ACCEL_COMPARE;
+ } else if (!strcmp(g_workload_type, "dualcast")) {
+ g_workload_selection = ACCEL_DUALCAST;
+ }
+ break;
+ default:
+ usage();
+ return 1;
+ }
+ return 0;
+}
+
+static void
+unregister_worker(void *arg1)
+{
+ struct worker_thread *worker = arg1;
+ struct ap_task *task;
+
+ while (!TAILQ_EMPTY(&worker->tasks)) {
+ task = TAILQ_FIRST(&worker->tasks);
+ TAILQ_REMOVE(&worker->tasks, task, link);
+ free(task);
+ }
+ spdk_put_io_channel(worker->ch);
+ pthread_mutex_lock(&g_workers_lock);
+ assert(g_num_workers >= 1);
+ if (--g_num_workers == 0) {
+ pthread_mutex_unlock(&g_workers_lock);
+ spdk_app_stop(0);
+ }
+ pthread_mutex_unlock(&g_workers_lock);
+}
+
+static void accel_done(void *ref, int status);
+
+static void
+_submit_single(void *arg1, void *arg2)
+{
+ struct worker_thread *worker = arg1;
+ struct ap_task *task = arg2;
+ int random_num;
+ int rc = 0;
+
+ assert(worker);
+
+ task->worker = worker;
+ task->worker->current_queue_depth++;
+ switch (g_workload_selection) {
+ case ACCEL_COPY:
+ rc = spdk_accel_submit_copy(worker->ch, task->dst, task->src,
+ g_xfer_size_bytes, accel_done, task);
+ break;
+ case ACCEL_FILL:
+ /* For fill use the first byte of the task->dst buffer */
+ rc = spdk_accel_submit_fill(worker->ch, task->dst, *(uint8_t *)task->src,
+ g_xfer_size_bytes, accel_done, task);
+ break;
+ case ACCEL_CRC32C:
+ rc = spdk_accel_submit_crc32c(worker->ch, (uint32_t *)task->dst,
+ task->src, g_crc32c_seed,
+ g_xfer_size_bytes, accel_done, task);
+ break;
+ case ACCEL_COMPARE:
+ random_num = rand() % 100;
+ if (random_num < g_fail_percent_goal) {
+ task->expected_status = -EILSEQ;
+ *(uint8_t *)task->dst = ~DATA_PATTERN;
+ } else {
+ task->expected_status = 0;
+ *(uint8_t *)task->dst = DATA_PATTERN;
+ }
+ rc = spdk_accel_submit_compare(worker->ch, task->dst, task->src,
+ g_xfer_size_bytes, accel_done, task);
+ break;
+ case ACCEL_DUALCAST:
+ rc = spdk_accel_submit_dualcast(worker->ch, task->dst, task->dst2,
+ task->src, g_xfer_size_bytes, accel_done, task);
+ break;
+ default:
+ assert(false);
+ break;
+
+ }
+
+ if (rc) {
+ accel_done(task, rc);
+ }
+}
+
+static void
+_accel_done(void *arg1)
+{
+ struct ap_task *task = arg1;
+ struct worker_thread *worker = task->worker;
+ uint32_t sw_crc32c;
+
+ assert(worker);
+ assert(worker->current_queue_depth > 0);
+
+ if (g_verify && task->status == 0) {
+ switch (g_workload_selection) {
+ case ACCEL_CRC32C:
+ /* calculate sw CRC-32C and compare to sw aceel result. */
+ sw_crc32c = spdk_crc32c_update(task->src, g_xfer_size_bytes, ~g_crc32c_seed);
+ if (*(uint32_t *)task->dst != sw_crc32c) {
+ SPDK_NOTICELOG("CRC-32C miscompare\n");
+ worker->xfer_failed++;
+ }
+ break;
+ case ACCEL_COPY:
+ if (memcmp(task->src, task->dst, g_xfer_size_bytes)) {
+ SPDK_NOTICELOG("Data miscompare\n");
+ worker->xfer_failed++;
+ }
+ break;
+ case ACCEL_DUALCAST:
+ if (memcmp(task->src, task->dst, g_xfer_size_bytes)) {
+ SPDK_NOTICELOG("Data miscompare, first destination\n");
+ worker->xfer_failed++;
+ }
+ if (memcmp(task->src, task->dst2, g_xfer_size_bytes)) {
+ SPDK_NOTICELOG("Data miscompare, second destination\n");
+ worker->xfer_failed++;
+ }
+ break;
+ case ACCEL_FILL:
+ if (memcmp(task->dst, task->src, g_xfer_size_bytes)) {
+ SPDK_NOTICELOG("Data miscompare\n");
+ worker->xfer_failed++;
+ }
+ break;
+ case ACCEL_COMPARE:
+ break;
+ default:
+ assert(false);
+ break;
+ }
+ }
+
+ if (task->expected_status == -EILSEQ) {
+ assert(task->status != 0);
+ worker->injected_miscompares++;
+ } else if (task->status) {
+ /* Expected to pass but API reported error. */
+ worker->xfer_failed++;
+ }
+
+ worker->xfer_completed++;
+ worker->current_queue_depth--;
+
+ if (!worker->is_draining) {
+ _submit_single(worker, task);
+ } else {
+ spdk_free(task->src);
+ spdk_free(task->dst);
+ if (g_workload_selection == ACCEL_DUALCAST) {
+ spdk_free(task->dst2);
+ }
+ TAILQ_INSERT_TAIL(&worker->tasks, task, link);
+ }
+}
+
+static void
+batch_done(void *cb_arg, int status)
+{
+ struct ap_task *task = (struct ap_task *)cb_arg;
+ struct worker_thread *worker = task->worker;
+
+ worker->current_queue_depth--;
+ TAILQ_INSERT_TAIL(&worker->tasks, task, link);
+}
+
+static int
+dump_result(void)
+{
+ uint64_t total_completed = 0;
+ uint64_t total_failed = 0;
+ uint64_t total_miscompared = 0;
+ uint64_t total_xfer_per_sec, total_bw_in_MiBps;
+ struct worker_thread *worker = g_workers;
+
+ printf("\nCore Transfers Bandwidth Failed Miscompares\n");
+ printf("-----------------------------------------------------------------\n");
+ while (worker != NULL) {
+
+ uint64_t xfer_per_sec = worker->xfer_completed / g_time_in_sec;
+ uint64_t bw_in_MiBps = (worker->xfer_completed * g_xfer_size_bytes) /
+ (g_time_in_sec * 1024 * 1024);
+
+ total_completed += worker->xfer_completed;
+ total_failed += worker->xfer_failed;
+ total_miscompared += worker->injected_miscompares;
+
+ if (xfer_per_sec) {
+ printf("%10d%12" PRIu64 "/s%8" PRIu64 " MiB/s%11" PRIu64 " %11" PRIu64 "\n",
+ worker->core, xfer_per_sec,
+ bw_in_MiBps, worker->xfer_failed, worker->injected_miscompares);
+ }
+
+ worker = worker->next;
+ }
+
+ total_xfer_per_sec = total_completed / g_time_in_sec;
+ total_bw_in_MiBps = (total_completed * g_xfer_size_bytes) /
+ (g_time_in_sec * 1024 * 1024);
+
+ printf("==================================================================\n");
+ printf("Total:%16" PRIu64 "/s%8" PRIu64 " MiB/s%11" PRIu64 " %11" PRIu64"\n\n",
+ total_xfer_per_sec, total_bw_in_MiBps, total_failed, total_miscompared);
+
+ return total_failed ? 1 : 0;
+}
+
+static int
+_check_draining(void *arg)
+{
+ struct worker_thread *worker = arg;
+
+ assert(worker);
+
+ if (worker->current_queue_depth == 0) {
+ spdk_poller_unregister(&worker->is_draining_poller);
+ unregister_worker(worker);
+ }
+
+ return -1;
+}
+
+static int
+_worker_stop(void *arg)
+{
+ struct worker_thread *worker = arg;
+
+ assert(worker);
+
+ spdk_poller_unregister(&worker->stop_poller);
+
+ /* now let the worker drain and check it's outstanding IO with a poller */
+ worker->is_draining = true;
+ worker->is_draining_poller = SPDK_POLLER_REGISTER(_check_draining, worker, 0);
+
+ return 0;
+}
+
+static void
+_init_thread_done(void *ctx)
+{
+}
+
+static int
+_get_task_data_bufs(struct ap_task *task)
+{
+ uint32_t align = 0;
+
+ /* For dualcast, the DSA HW requires 4K alignment on destination addresses but
+ * we do this for all engines to keep it simple.
+ */
+ if (g_workload_selection == ACCEL_DUALCAST) {
+ align = ALIGN_4K;
+ }
+
+ task->src = spdk_dma_zmalloc(g_xfer_size_bytes, 0, NULL);
+ if (task->src == NULL) {
+ fprintf(stderr, "Unable to alloc src buffer\n");
+ return -ENOMEM;
+ }
+ memset(task->src, DATA_PATTERN, g_xfer_size_bytes);
+
+ task->dst = spdk_dma_zmalloc(g_xfer_size_bytes, align, NULL);
+ if (task->dst == NULL) {
+ fprintf(stderr, "Unable to alloc dst buffer\n");
+ return -ENOMEM;
+ }
+
+ /* For compare we want the buffers to match, otherwise not. */
+ if (g_workload_selection == ACCEL_COMPARE) {
+ memset(task->dst, DATA_PATTERN, g_xfer_size_bytes);
+ } else {
+ memset(task->dst, ~DATA_PATTERN, g_xfer_size_bytes);
+ }
+
+ /* For fill, set the entire src buffer so we can check if verify is enabled. */
+ if (g_workload_selection == ACCEL_FILL) {
+ memset(task->src, g_fill_pattern, g_xfer_size_bytes);
+ }
+
+ if (g_workload_selection == ACCEL_DUALCAST) {
+ task->dst2 = spdk_dma_zmalloc(g_xfer_size_bytes, align, NULL);
+ if (task->dst2 == NULL) {
+ fprintf(stderr, "Unable to alloc dst buffer\n");
+ return -ENOMEM;
+ }
+ memset(task->dst2, ~DATA_PATTERN, g_xfer_size_bytes);
+ }
+
+ return 0;
+}
+
+static int
+_batch_prep_cmd(struct worker_thread *worker, struct ap_task *task, struct spdk_accel_batch *batch)
+{
+ int rc = 0;
+
+ switch (g_workload_selection) {
+ case ACCEL_COPY:
+ rc = spdk_accel_batch_prep_copy(worker->ch, batch, task->dst,
+ task->src, g_xfer_size_bytes, accel_done, task);
+ break;
+ case ACCEL_DUALCAST:
+ rc = spdk_accel_batch_prep_dualcast(worker->ch, batch, task->dst, task->dst2,
+ task->src, g_xfer_size_bytes, accel_done, task);
+ break;
+ case ACCEL_COMPARE:
+ rc = spdk_accel_batch_prep_compare(worker->ch, batch, task->dst, task->src,
+ g_xfer_size_bytes, accel_done, task);
+ break;
+ case ACCEL_FILL:
+ rc = spdk_accel_batch_prep_fill(worker->ch, batch, task->dst,
+ *(uint8_t *)task->src,
+ g_xfer_size_bytes, accel_done, task);
+ break;
+ case ACCEL_CRC32C:
+ rc = spdk_accel_batch_prep_crc32c(worker->ch, batch, (uint32_t *)task->dst,
+ task->src, g_crc32c_seed, g_xfer_size_bytes, accel_done, task);
+ break;
+ default:
+ assert(false);
+ break;
+ }
+
+ return rc;
+}
+
+static void
+_init_thread(void *arg1)
+{
+ struct worker_thread *worker;
+ struct ap_task *task;
+ int i, rc, max_per_batch, batch_count, num_tasks;
+ int remaining = g_queue_depth;
+ struct spdk_accel_batch *batch, *new_batch;
+
+ worker = calloc(1, sizeof(*worker));
+ if (worker == NULL) {
+ fprintf(stderr, "Unable to allocate worker\n");
+ return;
+ }
+
+ worker->core = spdk_env_get_current_core();
+ worker->thread = spdk_get_thread();
+ worker->next = g_workers;
+ worker->ch = spdk_accel_engine_get_io_channel();
+
+ max_per_batch = spdk_accel_batch_get_max(worker->ch);
+ assert(max_per_batch > 0);
+ num_tasks = g_queue_depth + spdk_divide_round_up(g_queue_depth, max_per_batch);
+
+ TAILQ_INIT(&worker->tasks);
+ for (i = 0; i < num_tasks; i++) {
+ task = calloc(1, sizeof(struct ap_task));
+ if (task == NULL) {
+ fprintf(stderr, "Could not allocate task.\n");
+ return;
+ /* TODO cleanup */
+ }
+ TAILQ_INSERT_TAIL(&worker->tasks, task, link);
+ }
+
+ /* Register a poller that will stop the worker at time elapsed */
+ worker->stop_poller = SPDK_POLLER_REGISTER(_worker_stop, worker,
+ g_time_in_sec * 1000000ULL);
+
+ g_workers = worker;
+ pthread_mutex_lock(&g_workers_lock);
+ g_num_workers++;
+ pthread_mutex_unlock(&g_workers_lock);
+
+ /* Batching is only possible if there is at least 2 operations. */
+ if (g_queue_depth > 1) {
+
+ /* Outter loop sets up each batch command, inner loop populates the
+ * batch descriptors.
+ */
+ do {
+ new_batch = spdk_accel_batch_create(worker->ch);
+ if (new_batch == NULL) {
+ break;
+ }
+
+ batch = new_batch;
+ batch_count = 0;
+
+ do {
+ if (!TAILQ_EMPTY(&worker->tasks)) {
+ task = TAILQ_FIRST(&worker->tasks);
+ TAILQ_REMOVE(&worker->tasks, task, link);
+ } else {
+ fprintf(stderr, "Unable to get accel_task\n");
+ goto error;
+ }
+ task->worker = worker;
+ task->worker->current_queue_depth++;
+
+ if (_get_task_data_bufs(task)) {
+ fprintf(stderr, "Unable to get data bufs\n");
+ goto error;
+ }
+
+ rc = _batch_prep_cmd(worker, task, batch);
+ if (rc) {
+ fprintf(stderr, "error preping command\n");
+ goto error;
+ }
+ remaining--;
+ batch_count++;
+ } while (batch_count < max_per_batch && remaining > 0);
+
+ /* Now send the batch command. */
+ if (!TAILQ_EMPTY(&worker->tasks)) {
+ task = TAILQ_FIRST(&worker->tasks);
+ TAILQ_REMOVE(&worker->tasks, task, link);
+ } else {
+ fprintf(stderr, "Unable to get accel_task\n");
+ goto error;
+ }
+ task->worker = worker;
+ task->worker->current_queue_depth++;
+
+ rc = spdk_accel_batch_submit(worker->ch, batch, batch_done, task);
+ if (rc) {
+ fprintf(stderr, "error ending batch %d\n", rc);
+ goto error;
+ }
+ /* We can't build a batch unless it has 2 descriptors (per spec). */
+ } while (remaining > 1);
+
+ /* If there are no more left, we're done. */
+ if (remaining == 0) {
+ return;
+ }
+ }
+
+ /* For engines that don't support batch or for the odd event that
+ * a batch ends with only one descriptor left.
+ */
+ for (i = 0; i < remaining; i++) {
+
+ if (!TAILQ_EMPTY(&worker->tasks)) {
+ task = TAILQ_FIRST(&worker->tasks);
+ TAILQ_REMOVE(&worker->tasks, task, link);
+ } else {
+ fprintf(stderr, "Unable to get accel_task\n");
+ goto error;
+ }
+
+ if (_get_task_data_bufs(task)) {
+ fprintf(stderr, "Unable to get data bufs\n");
+ goto error;
+ }
+
+ _submit_single(worker, task);
+ }
+ return;
+error:
+ /* TODO clean exit */
+ raise(SIGINT);
+ while (!TAILQ_EMPTY(&worker->tasks)) {
+ task = TAILQ_FIRST(&worker->tasks);
+ TAILQ_REMOVE(&worker->tasks, task, link);
+ free(task);
+ }
+ free(worker);
+ spdk_app_stop(-1);
+}
+
+static void
+accel_done(void *cb_arg, int status)
+{
+ struct ap_task *task = (struct ap_task *)cb_arg;
+ struct worker_thread *worker = task->worker;
+
+ assert(worker);
+
+ task->status = status;
+ spdk_thread_send_msg(worker->thread, _accel_done, task);
+}
+
+static void
+accel_perf_start(void *arg1)
+{
+ struct spdk_io_channel *accel_ch;
+
+ accel_ch = spdk_accel_engine_get_io_channel();
+ g_capabilites = spdk_accel_get_capabilities(accel_ch);
+ spdk_put_io_channel(accel_ch);
+
+ if ((g_capabilites & g_workload_selection) != g_workload_selection) {
+ SPDK_WARNLOG("The selected workload is not natively supported by the current engine\n");
+ SPDK_WARNLOG("The software engine will be used instead.\n\n");
+ }
+
+ g_tsc_rate = spdk_get_ticks_hz();
+ g_tsc_us_rate = g_tsc_rate / (1000 * 1000);
+ g_tsc_end = spdk_get_ticks() + g_time_in_sec * g_tsc_rate;
+
+ printf("Running for %d seconds...\n", g_time_in_sec);
+ fflush(stdout);
+
+ spdk_for_each_thread(_init_thread, NULL, _init_thread_done);
+}
+
+int
+main(int argc, char **argv)
+{
+ struct spdk_app_opts opts = {};
+ struct worker_thread *worker, *tmp;
+ int rc = 0;
+
+ pthread_mutex_init(&g_workers_lock, NULL);
+ spdk_app_opts_init(&opts);
+ opts.reactor_mask = "0x1";
+ if ((rc = spdk_app_parse_args(argc, argv, &opts, "o:q:t:yw:P:f:", NULL, parse_args,
+ usage)) != SPDK_APP_PARSE_ARGS_SUCCESS) {
+ rc = -1;
+ goto cleanup;
+ }
+
+ if ((g_workload_selection != ACCEL_COPY) &&
+ (g_workload_selection != ACCEL_FILL) &&
+ (g_workload_selection != ACCEL_CRC32C) &&
+ (g_workload_selection != ACCEL_COMPARE) &&
+ (g_workload_selection != ACCEL_DUALCAST)) {
+ usage();
+ rc = -1;
+ goto cleanup;
+ }
+
+ dump_user_config(&opts);
+ rc = spdk_app_start(&opts, accel_perf_start, NULL);
+ if (rc) {
+ SPDK_ERRLOG("ERROR starting application\n");
+ } else {
+ dump_result();
+ }
+
+ pthread_mutex_destroy(&g_workers_lock);
+
+ worker = g_workers;
+ while (worker) {
+ tmp = worker->next;
+ free(worker);
+ worker = tmp;
+ }
+cleanup:
+ spdk_app_fini();
+ return rc;
+}
diff --git a/src/spdk/examples/bdev/Makefile b/src/spdk/examples/bdev/Makefile
new file mode 100644
index 000000000..dc1f52213
--- /dev/null
+++ b/src/spdk/examples/bdev/Makefile
@@ -0,0 +1,48 @@
+#
+# BSD LICENSE
+#
+# Copyright (c) Intel Corporation.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Intel Corporation nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+DIRS-$(CONFIG_FIO_PLUGIN) = fio_plugin
+DIRS-y += hello_world
+
+.PHONY: all clean $(DIRS-y)
+
+all: $(DIRS-y)
+ @:
+
+clean: $(DIRS-y)
+ @:
+
+include $(SPDK_ROOT_DIR)/mk/spdk.subdirs.mk
diff --git a/src/spdk/examples/bdev/fio_plugin/.gitignore b/src/spdk/examples/bdev/fio_plugin/.gitignore
new file mode 100644
index 000000000..1b0b36ac4
--- /dev/null
+++ b/src/spdk/examples/bdev/fio_plugin/.gitignore
@@ -0,0 +1 @@
+fio_plugin
diff --git a/src/spdk/examples/bdev/fio_plugin/Makefile b/src/spdk/examples/bdev/fio_plugin/Makefile
new file mode 100644
index 000000000..e50498d24
--- /dev/null
+++ b/src/spdk/examples/bdev/fio_plugin/Makefile
@@ -0,0 +1,48 @@
+#
+# BSD LICENSE
+#
+# Copyright (c) Intel Corporation.
+# Copyright (c) 2015-2016, Micron Technology, Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Intel Corporation nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+include $(SPDK_ROOT_DIR)/mk/spdk.modules.mk
+
+FIO_PLUGIN := spdk_bdev
+
+C_SRCS = fio_plugin.c
+
+# Unable to combine the FIO plugin and the VPP socket abstraction (license incompatibility)
+SPDK_LIB_LIST = $(filter-out sock_vpp,$(ALL_MODULES_LIST))
+SPDK_LIB_LIST += thread util bdev bdev_rpc conf accel rpc jsonrpc json log sock trace notify
+SPDK_LIB_LIST += event $(EVENT_BDEV_SUBSYSTEM)
+
+include $(SPDK_ROOT_DIR)/mk/spdk.fio.mk
diff --git a/src/spdk/examples/bdev/fio_plugin/README.md b/src/spdk/examples/bdev/fio_plugin/README.md
new file mode 100644
index 000000000..369756fe3
--- /dev/null
+++ b/src/spdk/examples/bdev/fio_plugin/README.md
@@ -0,0 +1,74 @@
+# Introduction
+
+This directory contains a plug-in module for fio to enable use
+with SPDK. Fio is free software published under version 2 of
+the GPL license.
+
+# Compiling fio
+
+Clone the fio source repository from https://github.com/axboe/fio
+
+ git clone https://github.com/axboe/fio
+ cd fio
+
+Compile the fio code and install:
+
+ make
+ make install
+
+# Compiling SPDK
+
+Clone the SPDK source repository from https://github.com/spdk/spdk
+
+ git clone https://github.com/spdk/spdk
+ cd spdk
+ git submodule update --init
+
+Then, run the SPDK configure script to enable fio (point it to the root of the fio repository):
+
+ cd spdk
+ ./configure --with-fio=/path/to/fio/repo <other configuration options>
+
+Finally, build SPDK:
+
+ make
+
+**Note to advanced users**: These steps assume you're using the DPDK submodule. If you are using your
+own version of DPDK, the fio plugin requires that DPDK be compiled with -fPIC. You can compile DPDK
+with -fPIC by modifying your DPDK configuration file and adding the line:
+
+ EXTRA_CFLAGS=-fPIC
+
+# Usage
+
+To use the SPDK fio plugin with fio, specify the plugin binary using LD_PRELOAD when running
+fio and set ioengine=spdk_bdev in the fio configuration file (see example_config.fio in the same
+directory as this README).
+
+ LD_PRELOAD=<path to spdk repo>/build/fio/spdk_bdev fio
+
+The fio configuration file must contain one new parameter:
+
+ spdk_conf=./examples/bdev/fio_plugin/bdev.conf
+
+This must point at an SPDK configuration file. There are a number of example configuration
+files in the SPDK repository under etc/spdk.
+
+You can specify which block device to run against by setting the filename parameter
+to the block device name:
+
+ filename=Malloc0
+
+Or for NVMe devices:
+
+ filename=Nvme0n1
+
+Currently the SPDK fio plugin is limited to the thread usage model, so fio jobs must also specify thread=1
+when using the SPDK fio plugin.
+
+fio also currently has a race condition on shutdown if dynamically loading the ioengine by specifying the
+engine's full path via the ioengine parameter - LD_PRELOAD is recommended to avoid this race condition.
+
+When testing random workloads, it is recommended to set norandommap=1. fio's random map
+processing consumes extra CPU cycles which will degrade performance over time with
+the fio_plugin since all I/O are submitted and completed on a single CPU core.
diff --git a/src/spdk/examples/bdev/fio_plugin/bdev.conf.in b/src/spdk/examples/bdev/fio_plugin/bdev.conf.in
new file mode 100644
index 000000000..948cebe33
--- /dev/null
+++ b/src/spdk/examples/bdev/fio_plugin/bdev.conf.in
@@ -0,0 +1,3 @@
+[Malloc]
+ NumberOfLuns 1
+ LunSizeInMB 128
diff --git a/src/spdk/examples/bdev/fio_plugin/example_config.fio b/src/spdk/examples/bdev/fio_plugin/example_config.fio
new file mode 100644
index 000000000..3a35432e9
--- /dev/null
+++ b/src/spdk/examples/bdev/fio_plugin/example_config.fio
@@ -0,0 +1,16 @@
+[global]
+ioengine=spdk_bdev
+spdk_conf=./examples/bdev/fio_plugin/bdev.conf.in
+thread=1
+group_reporting=1
+direct=1
+verify=0
+time_based=1
+ramp_time=0
+runtime=2
+iodepth=128
+rw=randrw
+bs=4k
+
+[test]
+numjobs=1
diff --git a/src/spdk/examples/bdev/fio_plugin/fio_plugin.c b/src/spdk/examples/bdev/fio_plugin/fio_plugin.c
new file mode 100644
index 000000000..4f50cfb01
--- /dev/null
+++ b/src/spdk/examples/bdev/fio_plugin/fio_plugin.c
@@ -0,0 +1,838 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+
+#include "spdk/bdev.h"
+#include "spdk/accel_engine.h"
+#include "spdk/conf.h"
+#include "spdk/env.h"
+#include "spdk/thread.h"
+#include "spdk/log.h"
+#include "spdk/string.h"
+#include "spdk/queue.h"
+#include "spdk/util.h"
+
+#include "spdk_internal/thread.h"
+#include "spdk_internal/event.h"
+
+#include "config-host.h"
+#include "fio.h"
+#include "optgroup.h"
+
+/* FreeBSD is missing CLOCK_MONOTONIC_RAW,
+ * so alternative is provided. */
+#ifndef CLOCK_MONOTONIC_RAW /* Defined in glibc bits/time.h */
+#define CLOCK_MONOTONIC_RAW CLOCK_MONOTONIC
+#endif
+
+struct spdk_fio_options {
+ void *pad;
+ char *conf;
+ char *json_conf;
+ unsigned mem_mb;
+ bool mem_single_seg;
+};
+
+struct spdk_fio_request {
+ struct io_u *io;
+ struct thread_data *td;
+};
+
+struct spdk_fio_target {
+ struct spdk_bdev *bdev;
+ struct spdk_bdev_desc *desc;
+ struct spdk_io_channel *ch;
+
+ TAILQ_ENTRY(spdk_fio_target) link;
+};
+
+struct spdk_fio_thread {
+ struct thread_data *td; /* fio thread context */
+ struct spdk_thread *thread; /* spdk thread context */
+
+ TAILQ_HEAD(, spdk_fio_target) targets;
+ bool failed; /* true if the thread failed to initialize */
+
+ struct io_u **iocq; /* io completion queue */
+ unsigned int iocq_count; /* number of iocq entries filled by last getevents */
+ unsigned int iocq_size; /* number of iocq entries allocated */
+};
+
+static bool g_spdk_env_initialized = false;
+static const char *g_json_config_file = NULL;
+
+static int spdk_fio_init(struct thread_data *td);
+static void spdk_fio_cleanup(struct thread_data *td);
+static size_t spdk_fio_poll_thread(struct spdk_fio_thread *fio_thread);
+
+/* Default polling timeout (ns) */
+#define SPDK_FIO_POLLING_TIMEOUT 1000000000ULL
+
+static int
+spdk_fio_init_thread(struct thread_data *td)
+{
+ struct spdk_fio_thread *fio_thread;
+
+ fio_thread = calloc(1, sizeof(*fio_thread));
+ if (!fio_thread) {
+ SPDK_ERRLOG("failed to allocate thread local context\n");
+ return -1;
+ }
+
+ fio_thread->td = td;
+ td->io_ops_data = fio_thread;
+
+ fio_thread->thread = spdk_thread_create("fio_thread", NULL);
+ if (!fio_thread->thread) {
+ free(fio_thread);
+ SPDK_ERRLOG("failed to allocate thread\n");
+ return -1;
+ }
+ spdk_set_thread(fio_thread->thread);
+
+ fio_thread->iocq_size = td->o.iodepth;
+ fio_thread->iocq = calloc(fio_thread->iocq_size, sizeof(struct io_u *));
+ assert(fio_thread->iocq != NULL);
+
+ TAILQ_INIT(&fio_thread->targets);
+
+ return 0;
+}
+
+static void
+spdk_fio_bdev_close_targets(void *arg)
+{
+ struct spdk_fio_thread *fio_thread = arg;
+ struct spdk_fio_target *target, *tmp;
+
+ TAILQ_FOREACH_SAFE(target, &fio_thread->targets, link, tmp) {
+ TAILQ_REMOVE(&fio_thread->targets, target, link);
+ spdk_put_io_channel(target->ch);
+ spdk_bdev_close(target->desc);
+ free(target);
+ }
+}
+
+static void
+spdk_fio_cleanup_thread(struct spdk_fio_thread *fio_thread)
+{
+ spdk_thread_send_msg(fio_thread->thread, spdk_fio_bdev_close_targets, fio_thread);
+
+ while (!spdk_thread_is_idle(fio_thread->thread)) {
+ spdk_fio_poll_thread(fio_thread);
+ }
+
+ spdk_set_thread(fio_thread->thread);
+
+ spdk_thread_exit(fio_thread->thread);
+ while (!spdk_thread_is_exited(fio_thread->thread)) {
+ spdk_thread_poll(fio_thread->thread, 0, 0);
+ }
+ spdk_thread_destroy(fio_thread->thread);
+ free(fio_thread->iocq);
+ free(fio_thread);
+}
+
+static void
+spdk_fio_calc_timeout(struct spdk_fio_thread *fio_thread, struct timespec *ts)
+{
+ uint64_t timeout, now;
+
+ if (spdk_thread_has_active_pollers(fio_thread->thread)) {
+ return;
+ }
+
+ timeout = spdk_thread_next_poller_expiration(fio_thread->thread);
+ now = spdk_get_ticks();
+
+ if (timeout == 0) {
+ timeout = now + (SPDK_FIO_POLLING_TIMEOUT * spdk_get_ticks_hz()) / SPDK_SEC_TO_NSEC;
+ }
+
+ if (timeout > now) {
+ timeout = ((timeout - now) * SPDK_SEC_TO_NSEC) / spdk_get_ticks_hz() +
+ ts->tv_sec * SPDK_SEC_TO_NSEC + ts->tv_nsec;
+
+ ts->tv_sec = timeout / SPDK_SEC_TO_NSEC;
+ ts->tv_nsec = timeout % SPDK_SEC_TO_NSEC;
+ }
+}
+
+static pthread_t g_init_thread_id = 0;
+static pthread_mutex_t g_init_mtx = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t g_init_cond;
+static bool g_poll_loop = true;
+
+static void
+spdk_fio_bdev_init_done(int rc, void *cb_arg)
+{
+ *(bool *)cb_arg = true;
+}
+
+static void
+spdk_fio_bdev_init_start(void *arg)
+{
+ bool *done = arg;
+
+ if (g_json_config_file != NULL) {
+ spdk_app_json_config_load(g_json_config_file, SPDK_DEFAULT_RPC_ADDR,
+ spdk_fio_bdev_init_done, done, true);
+ } else {
+ spdk_subsystem_init(spdk_fio_bdev_init_done, done);
+ }
+}
+
+static void
+spdk_fio_bdev_fini_done(void *cb_arg)
+{
+ *(bool *)cb_arg = true;
+}
+
+static void
+spdk_fio_bdev_fini_start(void *arg)
+{
+ bool *done = arg;
+
+ spdk_subsystem_fini(spdk_fio_bdev_fini_done, done);
+}
+
+static void *
+spdk_init_thread_poll(void *arg)
+{
+ struct spdk_fio_options *eo = arg;
+ struct spdk_fio_thread *fio_thread;
+ struct spdk_conf *config = NULL;
+ struct spdk_env_opts opts;
+ bool done;
+ int rc;
+ struct timespec ts;
+ struct thread_data td = {};
+
+ /* Create a dummy thread data for use on the initialization thread. */
+ td.o.iodepth = 32;
+ td.eo = eo;
+
+ /* Parse the SPDK configuration file */
+ eo = arg;
+
+ if (eo->conf && eo->json_conf) {
+ SPDK_ERRLOG("Cannot provide two types of configuration files\n");
+ rc = EINVAL;
+ goto err_exit;
+ } else if (eo->conf && strlen(eo->conf)) {
+ config = spdk_conf_allocate();
+ if (!config) {
+ SPDK_ERRLOG("Unable to allocate configuration file\n");
+ rc = ENOMEM;
+ goto err_exit;
+ }
+
+ rc = spdk_conf_read(config, eo->conf);
+ if (rc != 0) {
+ SPDK_ERRLOG("Invalid configuration file format\n");
+ spdk_conf_free(config);
+ goto err_exit;
+ }
+ if (spdk_conf_first_section(config) == NULL) {
+ SPDK_ERRLOG("Invalid configuration file format\n");
+ spdk_conf_free(config);
+ rc = EINVAL;
+ goto err_exit;
+ }
+ spdk_conf_set_as_default(config);
+ } else if (eo->json_conf && strlen(eo->json_conf)) {
+ g_json_config_file = eo->json_conf;
+ } else {
+ SPDK_ERRLOG("No configuration file provided\n");
+ rc = EINVAL;
+ goto err_exit;
+ }
+
+ /* Initialize the environment library */
+ spdk_env_opts_init(&opts);
+ opts.name = "fio";
+
+ if (eo->mem_mb) {
+ opts.mem_size = eo->mem_mb;
+ }
+ opts.hugepage_single_segments = eo->mem_single_seg;
+
+ if (spdk_env_init(&opts) < 0) {
+ SPDK_ERRLOG("Unable to initialize SPDK env\n");
+ spdk_conf_free(config);
+ rc = EINVAL;
+ goto err_exit;
+ }
+ spdk_unaffinitize_thread();
+
+ spdk_thread_lib_init(NULL, 0);
+
+ /* Create an SPDK thread temporarily */
+ rc = spdk_fio_init_thread(&td);
+ if (rc < 0) {
+ SPDK_ERRLOG("Failed to create initialization thread\n");
+ goto err_exit;
+ }
+
+ fio_thread = td.io_ops_data;
+
+ /* Initialize the bdev layer */
+ done = false;
+ spdk_thread_send_msg(fio_thread->thread, spdk_fio_bdev_init_start, &done);
+
+ do {
+ spdk_fio_poll_thread(fio_thread);
+ } while (!done);
+
+ /*
+ * Continue polling until there are no more events.
+ * This handles any final events posted by pollers.
+ */
+ while (spdk_fio_poll_thread(fio_thread) > 0) {};
+
+ /* Set condition variable */
+ pthread_mutex_lock(&g_init_mtx);
+ pthread_cond_signal(&g_init_cond);
+
+ while (g_poll_loop) {
+ spdk_fio_poll_thread(fio_thread);
+
+ clock_gettime(CLOCK_MONOTONIC, &ts);
+ spdk_fio_calc_timeout(fio_thread, &ts);
+
+ rc = pthread_cond_timedwait(&g_init_cond, &g_init_mtx, &ts);
+ if (rc != ETIMEDOUT) {
+ break;
+ }
+ }
+
+ pthread_mutex_unlock(&g_init_mtx);
+
+ /* Finalize the bdev layer */
+ done = false;
+ spdk_thread_send_msg(fio_thread->thread, spdk_fio_bdev_fini_start, &done);
+
+ do {
+ spdk_fio_poll_thread(fio_thread);
+ } while (!done && !spdk_thread_is_idle(fio_thread->thread));
+
+ spdk_fio_cleanup_thread(fio_thread);
+
+ pthread_exit(NULL);
+
+err_exit:
+ exit(rc);
+ return NULL;
+}
+
+static int
+spdk_fio_init_env(struct thread_data *td)
+{
+ pthread_condattr_t attr;
+ int rc = -1;
+
+ if (pthread_condattr_init(&attr)) {
+ SPDK_ERRLOG("Unable to initialize condition variable\n");
+ return -1;
+ }
+
+ if (pthread_condattr_setclock(&attr, CLOCK_MONOTONIC)) {
+ SPDK_ERRLOG("Unable to initialize condition variable\n");
+ goto out;
+ }
+
+ if (pthread_cond_init(&g_init_cond, &attr)) {
+ SPDK_ERRLOG("Unable to initialize condition variable\n");
+ goto out;
+ }
+
+ /*
+ * Spawn a thread to handle initialization operations and to poll things
+ * like the admin queues periodically.
+ */
+ rc = pthread_create(&g_init_thread_id, NULL, &spdk_init_thread_poll, td->eo);
+ if (rc != 0) {
+ SPDK_ERRLOG("Unable to spawn thread to poll admin queue. It won't be polled.\n");
+ }
+
+ /* Wait for background thread to advance past the initialization */
+ pthread_mutex_lock(&g_init_mtx);
+ pthread_cond_wait(&g_init_cond, &g_init_mtx);
+ pthread_mutex_unlock(&g_init_mtx);
+out:
+ pthread_condattr_destroy(&attr);
+ return rc;
+}
+
+/* Called for each thread to fill in the 'real_file_size' member for
+ * each file associated with this thread. This is called prior to
+ * the init operation (spdk_fio_init()) below. This call will occur
+ * on the initial start up thread if 'create_serialize' is true, or
+ * on the thread actually associated with 'thread_data' if 'create_serialize'
+ * is false.
+ */
+static int
+spdk_fio_setup(struct thread_data *td)
+{
+ unsigned int i;
+ struct fio_file *f;
+
+ /* we might be running in a daemonized FIO instance where standard
+ * input and output were closed and fds 0, 1, and 2 are reused
+ * for something important by FIO. We can't ensure we won't print
+ * anything (and so will our dependencies, e.g. DPDK), so abort early.
+ * (is_backend is an fio global variable)
+ */
+ if (is_backend) {
+ char buf[1024];
+ snprintf(buf, sizeof(buf),
+ "SPDK FIO plugin won't work with daemonized FIO server.");
+ fio_server_text_output(FIO_LOG_ERR, buf, sizeof(buf));
+ return -1;
+ }
+
+ if (!td->o.use_thread) {
+ SPDK_ERRLOG("must set thread=1 when using spdk plugin\n");
+ return -1;
+ }
+
+ if (!g_spdk_env_initialized) {
+ if (spdk_fio_init_env(td)) {
+ SPDK_ERRLOG("failed to initialize\n");
+ return -1;
+ }
+
+ g_spdk_env_initialized = true;
+ }
+
+ if (td->o.nr_files == 1 && strcmp(td->files[0]->file_name, "*") == 0) {
+ struct spdk_bdev *bdev;
+
+ /* add all available bdevs as fio targets */
+ for (bdev = spdk_bdev_first_leaf(); bdev; bdev = spdk_bdev_next_leaf(bdev)) {
+ add_file(td, spdk_bdev_get_name(bdev), 0, 1);
+ }
+ }
+
+ for_each_file(td, f, i) {
+ struct spdk_bdev *bdev;
+
+ if (strcmp(f->file_name, "*") == 0) {
+ continue;
+ }
+
+ bdev = spdk_bdev_get_by_name(f->file_name);
+ if (!bdev) {
+ SPDK_ERRLOG("Unable to find bdev with name %s\n", f->file_name);
+ return -1;
+ }
+
+ f->real_file_size = spdk_bdev_get_num_blocks(bdev) *
+ spdk_bdev_get_block_size(bdev);
+
+ }
+
+ return 0;
+}
+
+static void
+spdk_fio_bdev_open(void *arg)
+{
+ struct thread_data *td = arg;
+ struct spdk_fio_thread *fio_thread;
+ unsigned int i;
+ struct fio_file *f;
+ int rc;
+
+ fio_thread = td->io_ops_data;
+
+ for_each_file(td, f, i) {
+ struct spdk_fio_target *target;
+
+ if (strcmp(f->file_name, "*") == 0) {
+ continue;
+ }
+
+ target = calloc(1, sizeof(*target));
+ if (!target) {
+ SPDK_ERRLOG("Unable to allocate memory for I/O target.\n");
+ fio_thread->failed = true;
+ return;
+ }
+
+ target->bdev = spdk_bdev_get_by_name(f->file_name);
+ if (!target->bdev) {
+ SPDK_ERRLOG("Unable to find bdev with name %s\n", f->file_name);
+ free(target);
+ fio_thread->failed = true;
+ return;
+ }
+
+ rc = spdk_bdev_open(target->bdev, true, NULL, NULL, &target->desc);
+ if (rc) {
+ SPDK_ERRLOG("Unable to open bdev %s\n", f->file_name);
+ free(target);
+ fio_thread->failed = true;
+ return;
+ }
+
+ target->ch = spdk_bdev_get_io_channel(target->desc);
+ if (!target->ch) {
+ SPDK_ERRLOG("Unable to get I/O channel for bdev.\n");
+ spdk_bdev_close(target->desc);
+ free(target);
+ fio_thread->failed = true;
+ return;
+ }
+
+ f->engine_data = target;
+
+ TAILQ_INSERT_TAIL(&fio_thread->targets, target, link);
+ }
+}
+
+/* Called for each thread, on that thread, shortly after the thread
+ * starts.
+ */
+static int
+spdk_fio_init(struct thread_data *td)
+{
+ struct spdk_fio_thread *fio_thread;
+
+ spdk_fio_init_thread(td);
+
+ fio_thread = td->io_ops_data;
+ fio_thread->failed = false;
+
+ spdk_thread_send_msg(fio_thread->thread, spdk_fio_bdev_open, td);
+
+ while (spdk_fio_poll_thread(fio_thread) > 0) {}
+
+ if (fio_thread->failed) {
+ return -1;
+ }
+
+ return 0;
+}
+
+static void
+spdk_fio_cleanup(struct thread_data *td)
+{
+ struct spdk_fio_thread *fio_thread = td->io_ops_data;
+
+ spdk_fio_cleanup_thread(fio_thread);
+ td->io_ops_data = NULL;
+}
+
+static int
+spdk_fio_open(struct thread_data *td, struct fio_file *f)
+{
+
+ return 0;
+}
+
+static int
+spdk_fio_close(struct thread_data *td, struct fio_file *f)
+{
+ return 0;
+}
+
+static int
+spdk_fio_iomem_alloc(struct thread_data *td, size_t total_mem)
+{
+ td->orig_buffer = spdk_dma_zmalloc(total_mem, 0x1000, NULL);
+ return td->orig_buffer == NULL;
+}
+
+static void
+spdk_fio_iomem_free(struct thread_data *td)
+{
+ spdk_dma_free(td->orig_buffer);
+}
+
+static int
+spdk_fio_io_u_init(struct thread_data *td, struct io_u *io_u)
+{
+ struct spdk_fio_request *fio_req;
+
+ io_u->engine_data = NULL;
+
+ fio_req = calloc(1, sizeof(*fio_req));
+ if (fio_req == NULL) {
+ return 1;
+ }
+ fio_req->io = io_u;
+ fio_req->td = td;
+
+ io_u->engine_data = fio_req;
+
+ return 0;
+}
+
+static void
+spdk_fio_io_u_free(struct thread_data *td, struct io_u *io_u)
+{
+ struct spdk_fio_request *fio_req = io_u->engine_data;
+
+ if (fio_req) {
+ assert(fio_req->io == io_u);
+ free(fio_req);
+ io_u->engine_data = NULL;
+ }
+}
+
+static void
+spdk_fio_completion_cb(struct spdk_bdev_io *bdev_io,
+ bool success,
+ void *cb_arg)
+{
+ struct spdk_fio_request *fio_req = cb_arg;
+ struct thread_data *td = fio_req->td;
+ struct spdk_fio_thread *fio_thread = td->io_ops_data;
+
+ assert(fio_thread->iocq_count < fio_thread->iocq_size);
+ fio_req->io->error = success ? 0 : EIO;
+ fio_thread->iocq[fio_thread->iocq_count++] = fio_req->io;
+
+ spdk_bdev_free_io(bdev_io);
+}
+
+#if FIO_IOOPS_VERSION >= 24
+typedef enum fio_q_status fio_q_status_t;
+#else
+typedef int fio_q_status_t;
+#endif
+
+static fio_q_status_t
+spdk_fio_queue(struct thread_data *td, struct io_u *io_u)
+{
+ int rc = 1;
+ struct spdk_fio_request *fio_req = io_u->engine_data;
+ struct spdk_fio_target *target = io_u->file->engine_data;
+
+ assert(fio_req->td == td);
+
+ if (!target) {
+ SPDK_ERRLOG("Unable to look up correct I/O target.\n");
+ fio_req->io->error = ENODEV;
+ return FIO_Q_COMPLETED;
+ }
+
+ switch (io_u->ddir) {
+ case DDIR_READ:
+ rc = spdk_bdev_read(target->desc, target->ch,
+ io_u->buf, io_u->offset, io_u->xfer_buflen,
+ spdk_fio_completion_cb, fio_req);
+ break;
+ case DDIR_WRITE:
+ rc = spdk_bdev_write(target->desc, target->ch,
+ io_u->buf, io_u->offset, io_u->xfer_buflen,
+ spdk_fio_completion_cb, fio_req);
+ break;
+ case DDIR_TRIM:
+ rc = spdk_bdev_unmap(target->desc, target->ch,
+ io_u->offset, io_u->xfer_buflen,
+ spdk_fio_completion_cb, fio_req);
+ break;
+ default:
+ assert(false);
+ break;
+ }
+
+ if (rc == -ENOMEM) {
+ return FIO_Q_BUSY;
+ }
+
+ if (rc != 0) {
+ fio_req->io->error = abs(rc);
+ return FIO_Q_COMPLETED;
+ }
+
+ return FIO_Q_QUEUED;
+}
+
+static struct io_u *
+spdk_fio_event(struct thread_data *td, int event)
+{
+ struct spdk_fio_thread *fio_thread = td->io_ops_data;
+
+ assert(event >= 0);
+ assert((unsigned)event < fio_thread->iocq_count);
+ return fio_thread->iocq[event];
+}
+
+static size_t
+spdk_fio_poll_thread(struct spdk_fio_thread *fio_thread)
+{
+ return spdk_thread_poll(fio_thread->thread, 0, 0);
+}
+
+static int
+spdk_fio_getevents(struct thread_data *td, unsigned int min,
+ unsigned int max, const struct timespec *t)
+{
+ struct spdk_fio_thread *fio_thread = td->io_ops_data;
+ struct timespec t0, t1;
+ uint64_t timeout = 0;
+
+ if (t) {
+ timeout = t->tv_sec * SPDK_SEC_TO_NSEC + t->tv_nsec;
+ clock_gettime(CLOCK_MONOTONIC_RAW, &t0);
+ }
+
+ fio_thread->iocq_count = 0;
+
+ for (;;) {
+ spdk_fio_poll_thread(fio_thread);
+
+ if (fio_thread->iocq_count >= min) {
+ return fio_thread->iocq_count;
+ }
+
+ if (t) {
+ clock_gettime(CLOCK_MONOTONIC_RAW, &t1);
+ uint64_t elapse = ((t1.tv_sec - t0.tv_sec) * SPDK_SEC_TO_NSEC)
+ + t1.tv_nsec - t0.tv_nsec;
+ if (elapse > timeout) {
+ break;
+ }
+ }
+ }
+
+ return fio_thread->iocq_count;
+}
+
+static int
+spdk_fio_invalidate(struct thread_data *td, struct fio_file *f)
+{
+ /* TODO: This should probably send a flush to the device, but for now just return successful. */
+ return 0;
+}
+
+static struct fio_option options[] = {
+ {
+ .name = "spdk_conf",
+ .lname = "SPDK configuration file",
+ .type = FIO_OPT_STR_STORE,
+ .off1 = offsetof(struct spdk_fio_options, conf),
+ .help = "A SPDK configuration file",
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_INVALID,
+ },
+ {
+ .name = "spdk_json_conf",
+ .lname = "SPDK JSON configuration file",
+ .type = FIO_OPT_STR_STORE,
+ .off1 = offsetof(struct spdk_fio_options, json_conf),
+ .help = "A SPDK JSON configuration file",
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_INVALID,
+ },
+ {
+ .name = "spdk_mem",
+ .lname = "SPDK memory in MB",
+ .type = FIO_OPT_INT,
+ .off1 = offsetof(struct spdk_fio_options, mem_mb),
+ .help = "Amount of memory in MB to allocate for SPDK",
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_INVALID,
+ },
+ {
+ .name = "spdk_single_seg",
+ .lname = "SPDK switch to create just a single hugetlbfs file",
+ .type = FIO_OPT_BOOL,
+ .off1 = offsetof(struct spdk_fio_options, mem_single_seg),
+ .help = "If set to 1, SPDK will use just a single hugetlbfs file",
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_INVALID,
+ },
+ {
+ .name = NULL,
+ },
+};
+
+/* FIO imports this structure using dlsym */
+struct ioengine_ops ioengine = {
+ .name = "spdk_bdev",
+ .version = FIO_IOOPS_VERSION,
+ .flags = FIO_RAWIO | FIO_NOEXTEND | FIO_NODISKUTIL | FIO_MEMALIGN,
+ .setup = spdk_fio_setup,
+ .init = spdk_fio_init,
+ /* .prep = unused, */
+ .queue = spdk_fio_queue,
+ /* .commit = unused, */
+ .getevents = spdk_fio_getevents,
+ .event = spdk_fio_event,
+ /* .errdetails = unused, */
+ /* .cancel = unused, */
+ .cleanup = spdk_fio_cleanup,
+ .open_file = spdk_fio_open,
+ .close_file = spdk_fio_close,
+ .invalidate = spdk_fio_invalidate,
+ /* .unlink_file = unused, */
+ /* .get_file_size = unused, */
+ /* .terminate = unused, */
+ .iomem_alloc = spdk_fio_iomem_alloc,
+ .iomem_free = spdk_fio_iomem_free,
+ .io_u_init = spdk_fio_io_u_init,
+ .io_u_free = spdk_fio_io_u_free,
+ .option_struct_size = sizeof(struct spdk_fio_options),
+ .options = options,
+};
+
+static void fio_init spdk_fio_register(void)
+{
+ register_ioengine(&ioengine);
+}
+
+static void
+spdk_fio_finish_env(void)
+{
+ pthread_mutex_lock(&g_init_mtx);
+ g_poll_loop = false;
+ pthread_cond_signal(&g_init_cond);
+ pthread_mutex_unlock(&g_init_mtx);
+ pthread_join(g_init_thread_id, NULL);
+
+ spdk_thread_lib_fini();
+}
+
+static void fio_exit spdk_fio_unregister(void)
+{
+ if (g_spdk_env_initialized) {
+ spdk_fio_finish_env();
+ g_spdk_env_initialized = false;
+ }
+ unregister_ioengine(&ioengine);
+}
diff --git a/src/spdk/examples/bdev/fio_plugin/full_bench.fio b/src/spdk/examples/bdev/fio_plugin/full_bench.fio
new file mode 100644
index 000000000..f76da18db
--- /dev/null
+++ b/src/spdk/examples/bdev/fio_plugin/full_bench.fio
@@ -0,0 +1,20 @@
+[global]
+ioengine=spdk_bdev
+spdk_conf=./examples/bdev/fio_plugin/bdev.conf.in
+thread=1
+group_reporting=1
+direct=1
+verify=0
+norandommap=1
+cpumask=1
+percentile_list=50:99:99.9:99.99:99.999
+
+[4k_randread_qd1]
+filename=Malloc0
+description="4KiB Random Read QD=1"
+bs=4k
+rw=randread
+iodepth=1
+time_based=1
+ramp_time=0
+runtime=10
diff --git a/src/spdk/examples/bdev/hello_world/.gitignore b/src/spdk/examples/bdev/hello_world/.gitignore
new file mode 100644
index 000000000..7bdf93936
--- /dev/null
+++ b/src/spdk/examples/bdev/hello_world/.gitignore
@@ -0,0 +1 @@
+hello_bdev
diff --git a/src/spdk/examples/bdev/hello_world/Makefile b/src/spdk/examples/bdev/hello_world/Makefile
new file mode 100644
index 000000000..f4a5a5b69
--- /dev/null
+++ b/src/spdk/examples/bdev/hello_world/Makefile
@@ -0,0 +1,44 @@
+#
+# Copyright (c) Intel Corporation.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Intel Corporation nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+include $(SPDK_ROOT_DIR)/mk/spdk.modules.mk
+
+APP = hello_bdev
+
+C_SRCS := hello_bdev.c
+
+SPDK_LIB_LIST = $(ALL_MODULES_LIST)
+SPDK_LIB_LIST += $(EVENT_BDEV_SUBSYSTEM)
+SPDK_LIB_LIST += bdev_rpc bdev accel event thread util conf trace log jsonrpc json rpc sock notify
+
+include $(SPDK_ROOT_DIR)/mk/spdk.app.mk
diff --git a/src/spdk/examples/bdev/hello_world/bdev.conf b/src/spdk/examples/bdev/hello_world/bdev.conf
new file mode 100644
index 000000000..80af878f8
--- /dev/null
+++ b/src/spdk/examples/bdev/hello_world/bdev.conf
@@ -0,0 +1,3 @@
+[Malloc]
+ NumberOfLuns 1
+ LunSizeInMB 32
diff --git a/src/spdk/examples/bdev/hello_world/hello_bdev.c b/src/spdk/examples/bdev/hello_world/hello_bdev.c
new file mode 100644
index 000000000..565005212
--- /dev/null
+++ b/src/spdk/examples/bdev/hello_world/hello_bdev.c
@@ -0,0 +1,295 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+#include "spdk/thread.h"
+#include "spdk/bdev.h"
+#include "spdk/env.h"
+#include "spdk/event.h"
+#include "spdk/log.h"
+#include "spdk/string.h"
+#include "spdk/bdev_module.h"
+
+static char *g_bdev_name = "Malloc0";
+
+/*
+ * We'll use this struct to gather housekeeping hello_context to pass between
+ * our events and callbacks.
+ */
+struct hello_context_t {
+ struct spdk_bdev *bdev;
+ struct spdk_bdev_desc *bdev_desc;
+ struct spdk_io_channel *bdev_io_channel;
+ char *buff;
+ char *bdev_name;
+ struct spdk_bdev_io_wait_entry bdev_io_wait;
+};
+
+/*
+ * Usage function for printing parameters that are specific to this application
+ */
+static void
+hello_bdev_usage(void)
+{
+ printf(" -b <bdev> name of the bdev to use\n");
+}
+
+/*
+ * This function is called to parse the parameters that are specific to this application
+ */
+static int hello_bdev_parse_arg(int ch, char *arg)
+{
+ switch (ch) {
+ case 'b':
+ g_bdev_name = arg;
+ break;
+ default:
+ return -EINVAL;
+ }
+ return 0;
+}
+
+/*
+ * Callback function for read io completion.
+ */
+static void
+read_complete(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
+{
+ struct hello_context_t *hello_context = cb_arg;
+
+ if (success) {
+ SPDK_NOTICELOG("Read string from bdev : %s\n", hello_context->buff);
+ } else {
+ SPDK_ERRLOG("bdev io read error\n");
+ }
+
+ /* Complete the bdev io and close the channel */
+ spdk_bdev_free_io(bdev_io);
+ spdk_put_io_channel(hello_context->bdev_io_channel);
+ spdk_bdev_close(hello_context->bdev_desc);
+ SPDK_NOTICELOG("Stopping app\n");
+ spdk_app_stop(success ? 0 : -1);
+}
+
+static void
+hello_read(void *arg)
+{
+ struct hello_context_t *hello_context = arg;
+ int rc = 0;
+ uint32_t length = spdk_bdev_get_block_size(hello_context->bdev);
+
+ SPDK_NOTICELOG("Reading io\n");
+ rc = spdk_bdev_read(hello_context->bdev_desc, hello_context->bdev_io_channel,
+ hello_context->buff, 0, length, read_complete, hello_context);
+
+ if (rc == -ENOMEM) {
+ SPDK_NOTICELOG("Queueing io\n");
+ /* In case we cannot perform I/O now, queue I/O */
+ hello_context->bdev_io_wait.bdev = hello_context->bdev;
+ hello_context->bdev_io_wait.cb_fn = hello_read;
+ hello_context->bdev_io_wait.cb_arg = hello_context;
+ spdk_bdev_queue_io_wait(hello_context->bdev, hello_context->bdev_io_channel,
+ &hello_context->bdev_io_wait);
+ } else if (rc) {
+ SPDK_ERRLOG("%s error while reading from bdev: %d\n", spdk_strerror(-rc), rc);
+ spdk_put_io_channel(hello_context->bdev_io_channel);
+ spdk_bdev_close(hello_context->bdev_desc);
+ spdk_app_stop(-1);
+ }
+}
+
+/*
+ * Callback function for write io completion.
+ */
+static void
+write_complete(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
+{
+ struct hello_context_t *hello_context = cb_arg;
+ uint32_t length;
+
+ /* Complete the I/O */
+ spdk_bdev_free_io(bdev_io);
+
+ if (success) {
+ SPDK_NOTICELOG("bdev io write completed successfully\n");
+ } else {
+ SPDK_ERRLOG("bdev io write error: %d\n", EIO);
+ spdk_put_io_channel(hello_context->bdev_io_channel);
+ spdk_bdev_close(hello_context->bdev_desc);
+ spdk_app_stop(-1);
+ return;
+ }
+
+ /* Zero the buffer so that we can use it for reading */
+ length = spdk_bdev_get_block_size(hello_context->bdev);
+ memset(hello_context->buff, 0, length);
+
+ hello_read(hello_context);
+}
+
+static void
+hello_write(void *arg)
+{
+ struct hello_context_t *hello_context = arg;
+ int rc = 0;
+ uint32_t length = spdk_bdev_get_block_size(hello_context->bdev);
+
+ SPDK_NOTICELOG("Writing to the bdev\n");
+ rc = spdk_bdev_write(hello_context->bdev_desc, hello_context->bdev_io_channel,
+ hello_context->buff, 0, length, write_complete, hello_context);
+
+ if (rc == -ENOMEM) {
+ SPDK_NOTICELOG("Queueing io\n");
+ /* In case we cannot perform I/O now, queue I/O */
+ hello_context->bdev_io_wait.bdev = hello_context->bdev;
+ hello_context->bdev_io_wait.cb_fn = hello_write;
+ hello_context->bdev_io_wait.cb_arg = hello_context;
+ spdk_bdev_queue_io_wait(hello_context->bdev, hello_context->bdev_io_channel,
+ &hello_context->bdev_io_wait);
+ } else if (rc) {
+ SPDK_ERRLOG("%s error while writing to bdev: %d\n", spdk_strerror(-rc), rc);
+ spdk_put_io_channel(hello_context->bdev_io_channel);
+ spdk_bdev_close(hello_context->bdev_desc);
+ spdk_app_stop(-1);
+ }
+}
+
+/*
+ * Our initial event that kicks off everything from main().
+ */
+static void
+hello_start(void *arg1)
+{
+ struct hello_context_t *hello_context = arg1;
+ uint32_t blk_size, buf_align;
+ int rc = 0;
+ hello_context->bdev = NULL;
+ hello_context->bdev_desc = NULL;
+
+ SPDK_NOTICELOG("Successfully started the application\n");
+
+ /*
+ * Get the bdev. There can be many bdevs configured, but this
+ * application will only use the one input by the user at runtime so
+ * we get it via its name.
+ */
+ hello_context->bdev = spdk_bdev_get_by_name(hello_context->bdev_name);
+ if (hello_context->bdev == NULL) {
+ SPDK_ERRLOG("Could not find the bdev: %s\n", hello_context->bdev_name);
+ spdk_app_stop(-1);
+ return;
+ }
+
+ /*
+ * Open the bdev by calling spdk_bdev_open()
+ * The function will return a descriptor
+ */
+ SPDK_NOTICELOG("Opening the bdev %s\n", hello_context->bdev_name);
+ rc = spdk_bdev_open(hello_context->bdev, true, NULL, NULL, &hello_context->bdev_desc);
+ if (rc) {
+ SPDK_ERRLOG("Could not open bdev: %s\n", hello_context->bdev_name);
+ spdk_app_stop(-1);
+ return;
+ }
+
+ SPDK_NOTICELOG("Opening io channel\n");
+ /* Open I/O channel */
+ hello_context->bdev_io_channel = spdk_bdev_get_io_channel(hello_context->bdev_desc);
+ if (hello_context->bdev_io_channel == NULL) {
+ SPDK_ERRLOG("Could not create bdev I/O channel!!\n");
+ spdk_bdev_close(hello_context->bdev_desc);
+ spdk_app_stop(-1);
+ return;
+ }
+
+ /* Allocate memory for the write buffer.
+ * Initialize the write buffer with the string "Hello World!"
+ */
+ blk_size = spdk_bdev_get_block_size(hello_context->bdev);
+ buf_align = spdk_bdev_get_buf_align(hello_context->bdev);
+ hello_context->buff = spdk_dma_zmalloc(blk_size, buf_align, NULL);
+ if (!hello_context->buff) {
+ SPDK_ERRLOG("Failed to allocate buffer\n");
+ spdk_put_io_channel(hello_context->bdev_io_channel);
+ spdk_bdev_close(hello_context->bdev_desc);
+ spdk_app_stop(-1);
+ return;
+ }
+ snprintf(hello_context->buff, blk_size, "%s", "Hello World!\n");
+
+ hello_write(hello_context);
+}
+
+int
+main(int argc, char **argv)
+{
+ struct spdk_app_opts opts = {};
+ int rc = 0;
+ struct hello_context_t hello_context = {};
+
+ /* Set default values in opts structure. */
+ spdk_app_opts_init(&opts);
+ opts.name = "hello_bdev";
+
+ /*
+ * Parse built-in SPDK command line parameters as well
+ * as our custom one(s).
+ */
+ if ((rc = spdk_app_parse_args(argc, argv, &opts, "b:", NULL, hello_bdev_parse_arg,
+ hello_bdev_usage)) != SPDK_APP_PARSE_ARGS_SUCCESS) {
+ exit(rc);
+ }
+ hello_context.bdev_name = g_bdev_name;
+
+ /*
+ * spdk_app_start() will initialize the SPDK framework, call hello_start(),
+ * and then block until spdk_app_stop() is called (or if an initialization
+ * error occurs, spdk_app_start() will return with rc even without calling
+ * hello_start().
+ */
+ rc = spdk_app_start(&opts, hello_start, &hello_context);
+ if (rc) {
+ SPDK_ERRLOG("ERROR starting application\n");
+ }
+
+ /* At this point either spdk_app_stop() was called, or spdk_app_start()
+ * failed because of internal error.
+ */
+
+ /* When the app stops, free up memory that we allocated. */
+ spdk_dma_free(hello_context.buff);
+
+ /* Gracefully close out all of the SPDK subsystems. */
+ spdk_app_fini();
+ return rc;
+}
diff --git a/src/spdk/examples/blob/Makefile b/src/spdk/examples/blob/Makefile
new file mode 100644
index 000000000..a297ddb26
--- /dev/null
+++ b/src/spdk/examples/blob/Makefile
@@ -0,0 +1,44 @@
+#
+# BSD LICENSE
+#
+# Copyright (c) Intel Corporation.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Intel Corporation nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+DIRS-y += hello_world cli
+
+.PHONY: all clean $(DIRS-y)
+
+all: $(DIRS-y)
+clean: $(DIRS-y)
+
+include $(SPDK_ROOT_DIR)/mk/spdk.subdirs.mk
diff --git a/src/spdk/examples/blob/cli/.gitignore b/src/spdk/examples/blob/cli/.gitignore
new file mode 100644
index 000000000..6c895d79b
--- /dev/null
+++ b/src/spdk/examples/blob/cli/.gitignore
@@ -0,0 +1 @@
+blobcli
diff --git a/src/spdk/examples/blob/cli/Makefile b/src/spdk/examples/blob/cli/Makefile
new file mode 100644
index 000000000..3c3ff1f26
--- /dev/null
+++ b/src/spdk/examples/blob/cli/Makefile
@@ -0,0 +1,46 @@
+#
+# Copyright (c) Intel Corporation.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Intel Corporation nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+include $(SPDK_ROOT_DIR)/mk/spdk.modules.mk
+
+APP = blobcli
+
+C_SRCS := blobcli.c
+
+# Don't link bdev_lvol in blobcli - otherwise this utility cannot operate on an lvolstore
+SPDK_LIB_LIST = $(filter-out bdev_lvol,$(ALL_MODULES_LIST))
+SPDK_LIB_LIST += $(EVENT_BDEV_SUBSYSTEM)
+SPDK_LIB_LIST += bdev accel event thread util conf trace \
+ log jsonrpc json rpc sock notify
+
+include $(SPDK_ROOT_DIR)/mk/spdk.app.mk
diff --git a/src/spdk/examples/blob/cli/README.md b/src/spdk/examples/blob/cli/README.md
new file mode 100644
index 000000000..9c87dbdd4
--- /dev/null
+++ b/src/spdk/examples/blob/cli/README.md
@@ -0,0 +1,65 @@
+The blobcli tool has several options that are listed by using the -h command
+however the three operating modes are covered in more detail here:
+
+Command Mode
+------------
+
+This is the default and will just execute one command at a time. It's simple
+but the downside is that if you are going to interact quite a bit with the
+blobstore, the startup time for the application can be cumbersome.
+
+Shell Mode
+----------
+
+You startup shell mode by using the -S command. At that point you will get
+a "blob>" prompt where you can enter any of the commands, including -h,
+to execute them. You can stil enter just one at a time but the initial
+startup time for the application will not get in the way between commands
+anymore so it is much more usable.
+
+Script (aka test) Mode
+----------------------
+
+In script mode you just supply one command with a filename when you start
+the cli, for example `blobcli -T test.bs` will feed the tool the file
+called test.bs which contains a series of commands that will all run
+automatically and, like shell mode, will only initialize one time so is
+quick.
+
+The script file format (example) is shown below. Comments are allowed and
+each line should contain one valid command (and its parameters) only. In
+order to operate on blobs via their ID value, use the token $Bn where n
+represents the instance of the blob created in the script.
+
+For example, the line `-s $B0` will operate on the blobid of the first
+blob created in the script (0 index based). `$B2` represents the third
+blob created in the script.
+
+If you start test mode with the additional "ignore" option, any invalid
+script lines will simply be skipped, otherwise the tool will exit if
+it runs into an invalid line (ie './blobcli -T test.bs ignore`).
+
+Sample test/bs file:
+
+~~~{.sh}
+# this is a comment
+-i
+-s bs
+-l bdevs
+-n 1
+-s bs
+-s $B0
+-n 2
+-s $B1
+-m $B0 Makefile
+-d $B0 M.blob
+-f $B1 65
+-d $B1 65.blob
+-s bs
+-x $B0 b0key boval
+-x $B1 b1key b1val
+-r $B0 b0key
+-s $B0
+-s $B1
+-s bs
+~~~
diff --git a/src/spdk/examples/blob/cli/blobcli.c b/src/spdk/examples/blob/cli/blobcli.c
new file mode 100644
index 000000000..edb071847
--- /dev/null
+++ b/src/spdk/examples/blob/cli/blobcli.c
@@ -0,0 +1,1572 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+
+#include "spdk/bdev.h"
+#include "spdk/env.h"
+#include "spdk/event.h"
+#include "spdk/blob_bdev.h"
+#include "spdk/blob.h"
+#include "spdk/log.h"
+#include "spdk/version.h"
+#include "spdk/string.h"
+#include "spdk/uuid.h"
+
+/*
+ * The following is not a public header file, but the CLI does expose
+ * some internals of blobstore for dev/debug puposes so we
+ * include it here.
+ */
+#include "../lib/blob/blobstore.h"
+static void cli_start(void *arg1);
+
+static const char *program_name = "blobcli";
+/* default name for .conf file, any name can be used however with -c switch */
+static const char *program_conf = "blobcli.conf";
+
+/*
+ * CMD mode runs one command at a time which can be annoying as the init takes
+ * a few seconds, so the shell mode, invoked with -S, does the init once and gives
+ * the user an interactive shell instead. With script mode init is also done just
+ * once.
+ */
+enum cli_mode_type {
+ CLI_MODE_CMD,
+ CLI_MODE_SHELL,
+ CLI_MODE_SCRIPT
+};
+
+enum cli_action_type {
+ CLI_NONE,
+ CLI_IMPORT_BLOB,
+ CLI_DUMP_BLOB,
+ CLI_FILL,
+ CLI_REM_XATTR,
+ CLI_SET_XATTR,
+ CLI_SET_SUPER,
+ CLI_SHOW_BS,
+ CLI_SHOW_BLOB,
+ CLI_CREATE_BLOB,
+ CLI_LIST_BDEVS,
+ CLI_LIST_BLOBS,
+ CLI_INIT_BS,
+ CLI_DUMP_BS,
+ CLI_SHELL_EXIT,
+ CLI_HELP,
+};
+
+#define BUFSIZE 255
+#define MAX_ARGS 16
+#define ALIGN_4K 4096
+#define STARTING_IO_UNIT 0
+#define NUM_IO_UNITS 1
+
+/*
+ * The CLI uses the SPDK app framework so is async and callback driven. A
+ * pointer to this structure is passed to SPDK calls and returned in the
+ * callbacks for easy access to all the info we may need.
+ */
+struct cli_context_t {
+ struct spdk_blob_store *bs;
+ struct spdk_blob *blob;
+ struct spdk_bs_dev *bs_dev;
+ spdk_blob_id blobid;
+ spdk_blob_id superid;
+ struct spdk_io_channel *channel;
+ uint8_t *buff;
+ uint64_t page_size;
+ uint64_t io_unit_size;
+ uint64_t io_unit_count;
+ uint64_t blob_io_units;
+ uint64_t bytes_so_far;
+ FILE *fp;
+ enum cli_action_type action;
+ char key[BUFSIZE + 1];
+ char value[BUFSIZE + 1];
+ char file[BUFSIZE + 1];
+ uint64_t filesize;
+ int fill_value;
+ char bdev_name[BUFSIZE];
+ int rc;
+ int num_clusters;
+ enum cli_mode_type cli_mode;
+ const char *config_file;
+ int argc;
+ char *argv[MAX_ARGS];
+ bool app_started;
+ char script_file[BUFSIZE + 1];
+};
+
+/* we store a bunch of stuff in a global struct for use by scripting mode */
+#define MAX_SCRIPT_LINES 64
+#define MAX_SCRIPT_BLOBS 16
+struct cli_script_t {
+ spdk_blob_id blobid[MAX_SCRIPT_BLOBS];
+ int blobid_idx;
+ int max_index;
+ int cmdline_idx;
+ bool ignore_errors;
+ char *cmdline[MAX_SCRIPT_LINES];
+};
+struct cli_script_t g_script;
+
+/*
+ * Common printing of commands for CLI and shell modes.
+ */
+static void
+print_cmds(void)
+{
+ printf("\nCommands include:\n");
+ printf("\t-b bdev - name of the block device to use (example: Nvme0n1)\n");
+ printf("\t-d <blobid> filename - dump contents of a blob to a file\n");
+ printf("\t-D - dump metadata contents of an existing blobstore\n");
+ printf("\t-f <blobid> value - fill a blob with a decimal value\n");
+ printf("\t-h - this help screen\n");
+ printf("\t-i - initialize a blobstore\n");
+ printf("\t-l bdevs | blobs - list either available bdevs or existing blobs\n");
+ printf("\t-m <blobid> filename - import contents of a file to a blob\n");
+ printf("\t-n <# clusters> - create new blob\n");
+ printf("\t-p <blobid> - set the superblob to the ID provided\n");
+ printf("\t-r <blobid> name - remove xattr name/value pair\n");
+ printf("\t-s <blobid> | bs - show blob info or blobstore info\n");
+ printf("\t-x <blobid> name value - set xattr name/value pair\n");
+ printf("\t-X - exit when in interactive shell mode\n");
+ printf("\t-S - enter interactive shell mode\n");
+ printf("\t-T <filename> - automated script mode\n");
+ printf("\n");
+}
+
+/*
+ * Prints usage and relevant error message.
+ */
+static void
+usage(struct cli_context_t *cli_context, char *msg)
+{
+ if (msg) {
+ printf("%s", msg);
+ }
+
+ if (!cli_context || cli_context->cli_mode == CLI_MODE_CMD) {
+ printf("Version %s\n", SPDK_VERSION_STRING);
+ printf("Usage: %s [-c SPDK config_file] Command\n", program_name);
+ printf("\n%s is a command line tool for interacting with blobstore\n",
+ program_name);
+ printf("on the underlying device specified in the conf file passed\n");
+ printf("in as a command line option.\n");
+ }
+ if (!cli_context || cli_context->cli_mode != CLI_MODE_SCRIPT) {
+ print_cmds();
+ }
+}
+
+/*
+ * Free up memory that we allocated.
+ */
+static void
+cli_cleanup(struct cli_context_t *cli_context)
+{
+ if (cli_context->buff) {
+ spdk_free(cli_context->buff);
+ }
+ if (cli_context->cli_mode == CLI_MODE_SCRIPT) {
+ int i;
+
+ for (i = 0; i <= g_script.max_index; i++) {
+ free(g_script.cmdline[i]);
+ }
+ }
+ free(cli_context);
+}
+
+/*
+ * Callback routine for the blobstore unload.
+ */
+static void
+unload_complete(void *cb_arg, int bserrno)
+{
+ struct cli_context_t *cli_context = cb_arg;
+
+ if (bserrno) {
+ printf("Error %d unloading the bobstore\n", bserrno);
+ cli_context->rc = bserrno;
+ }
+
+ /*
+ * Quit if we're in cmd mode or exiting shell mode, otherwise
+ * clear the action field and start the main function again.
+ */
+ if (cli_context->cli_mode == CLI_MODE_CMD ||
+ cli_context->action == CLI_SHELL_EXIT) {
+ spdk_app_stop(cli_context->rc);
+ } else {
+ /* when action is CLI_NONE, we know we need to remain in the shell */
+ cli_context->bs = NULL;
+ cli_context->action = CLI_NONE;
+ cli_start(cli_context);
+ }
+}
+
+/*
+ * Unload the blobstore.
+ */
+static void
+unload_bs(struct cli_context_t *cli_context, char *msg, int bserrno)
+{
+ if (bserrno) {
+ printf("%s (err %d)\n", msg, bserrno);
+ cli_context->rc = bserrno;
+ }
+
+ if (cli_context->bs) {
+ if (cli_context->channel) {
+ spdk_bs_free_io_channel(cli_context->channel);
+ cli_context->channel = NULL;
+ }
+ spdk_bs_unload(cli_context->bs, unload_complete, cli_context);
+ } else if (cli_context->cli_mode != CLI_MODE_SCRIPT) {
+ spdk_app_stop(bserrno);
+
+ }
+}
+
+/*
+ * Callback for closing a blob.
+ */
+static void
+close_cb(void *arg1, int bserrno)
+{
+ struct cli_context_t *cli_context = arg1;
+
+ if (bserrno) {
+ unload_bs(cli_context, "Error in close callback",
+ bserrno);
+ return;
+ }
+ unload_bs(cli_context, "", 0);
+}
+
+/*
+ * Callback function for sync'ing metadata.
+ */
+static void
+sync_cb(void *arg1, int bserrno)
+{
+ struct cli_context_t *cli_context = arg1;
+
+ if (bserrno) {
+ unload_bs(cli_context, "Error in sync callback",
+ bserrno);
+ return;
+ }
+
+ spdk_blob_close(cli_context->blob, close_cb, cli_context);
+}
+
+static void
+resize_cb(void *cb_arg, int bserrno)
+{
+ struct cli_context_t *cli_context = cb_arg;
+ uint64_t total = 0;
+
+ if (bserrno) {
+ unload_bs(cli_context, "Error in blob resize",
+ bserrno);
+ return;
+ }
+
+ total = spdk_blob_get_num_clusters(cli_context->blob);
+ printf("blob now has USED clusters of %" PRIu64 "\n",
+ total);
+
+ /*
+ * Always a good idea to sync after MD changes or the changes
+ * may be lost if things aren't closed cleanly.
+ */
+ spdk_blob_sync_md(cli_context->blob, sync_cb, cli_context);
+}
+
+/*
+ * Callback function for opening a blob after creating.
+ */
+static void
+open_now_resize_cb(void *cb_arg, struct spdk_blob *blob, int bserrno)
+{
+ struct cli_context_t *cli_context = cb_arg;
+
+ if (bserrno) {
+ unload_bs(cli_context, "Error in open completion",
+ bserrno);
+ return;
+ }
+ cli_context->blob = blob;
+
+ spdk_blob_resize(cli_context->blob, cli_context->num_clusters,
+ resize_cb, cli_context);
+}
+
+/*
+ * Callback function for creating a blob.
+ */
+static void
+blob_create_cb(void *arg1, spdk_blob_id blobid, int bserrno)
+{
+ struct cli_context_t *cli_context = arg1;
+
+ if (bserrno) {
+ unload_bs(cli_context, "Error in blob create callback",
+ bserrno);
+ return;
+ }
+
+ cli_context->blobid = blobid;
+ printf("New blob id %" PRIu64 "\n", cli_context->blobid);
+
+ /* if we're in script mode, we need info on all blobids for later */
+ if (cli_context->cli_mode == CLI_MODE_SCRIPT) {
+ g_script.blobid[g_script.blobid_idx++] = blobid;
+ }
+
+ /* We have to open the blob before we can do things like resize. */
+ spdk_bs_open_blob(cli_context->bs, cli_context->blobid,
+ open_now_resize_cb, cli_context);
+}
+
+/*
+ * Callback for get_super where we'll continue on to show blobstore info.
+ */
+static void
+show_bs_cb(void *arg1, spdk_blob_id blobid, int bserrno)
+{
+ struct cli_context_t *cli_context = arg1;
+ struct spdk_bs_type bstype;
+ uint64_t val;
+ struct spdk_bdev *bdev = NULL;
+
+ if (bserrno && bserrno != -ENOENT) {
+ unload_bs(cli_context, "Error in get_super callback",
+ bserrno);
+ return;
+ }
+ cli_context->superid = blobid;
+
+ bdev = spdk_bdev_get_by_name(cli_context->bdev_name);
+ if (bdev == NULL) {
+ unload_bs(cli_context, "Error w/bdev in get_super callback",
+ bserrno);
+ return;
+ }
+
+ printf("Blobstore Public Info:\n");
+ printf("\tUsing bdev Product Name: %s\n",
+ spdk_bdev_get_product_name(bdev));
+ printf("\tAPI Version: %d\n", SPDK_BS_VERSION);
+
+ if (bserrno != -ENOENT) {
+ printf("\tsuper blob ID: %" PRIu64 "\n", cli_context->superid);
+ } else {
+ printf("\tsuper blob ID: none assigned\n");
+ }
+
+ printf("\tpage size: %" PRIu64 "\n", cli_context->page_size);
+ printf("\tio unit size: %" PRIu64 "\n", cli_context->io_unit_size);
+
+ val = spdk_bs_get_cluster_size(cli_context->bs);
+ printf("\tcluster size: %" PRIu64 "\n", val);
+
+ val = spdk_bs_free_cluster_count(cli_context->bs);
+ printf("\t# free clusters: %" PRIu64 "\n", val);
+
+ bstype = spdk_bs_get_bstype(cli_context->bs);
+ spdk_log_dump(stdout, "\tblobstore type:", &bstype, sizeof(bstype));
+
+ /*
+ * Private info isn't accessible via the public API but
+ * may be useful for debug of blobstore based applications.
+ */
+ printf("\nBlobstore Private Info:\n");
+ printf("\tMetadata start (pages): %" PRIu64 "\n",
+ cli_context->bs->md_start);
+ printf("\tMetadata length (pages): %d\n",
+ cli_context->bs->md_len);
+
+ unload_bs(cli_context, "", 0);
+}
+
+/*
+ * Show detailed info about a particular blob.
+ */
+static void
+show_blob(struct cli_context_t *cli_context)
+{
+ uint64_t val;
+ struct spdk_xattr_names *names;
+ const void *value;
+ size_t value_len;
+ unsigned int i;
+
+ printf("Blob Public Info:\n");
+
+ printf("blob ID: %" PRIu64 "\n", cli_context->blobid);
+
+ val = spdk_blob_get_num_clusters(cli_context->blob);
+ printf("# of clusters: %" PRIu64 "\n", val);
+
+ printf("# of bytes: %" PRIu64 "\n",
+ val * spdk_bs_get_cluster_size(cli_context->bs));
+
+ val = spdk_blob_get_num_pages(cli_context->blob);
+ printf("# of pages: %" PRIu64 "\n", val);
+
+ spdk_blob_get_xattr_names(cli_context->blob, &names);
+
+ printf("# of xattrs: %d\n", spdk_xattr_names_get_count(names));
+ printf("xattrs:\n");
+ for (i = 0; i < spdk_xattr_names_get_count(names); i++) {
+ spdk_blob_get_xattr_value(cli_context->blob,
+ spdk_xattr_names_get_name(names, i),
+ &value, &value_len);
+ if (value_len > BUFSIZE) {
+ printf("FYI: adjusting size of xattr due to CLI limits.\n");
+ value_len = BUFSIZE + 1;
+ }
+ printf("\n(%d) Name:%s\n", i,
+ spdk_xattr_names_get_name(names, i));
+ printf("(%d) Value:\n", i);
+ spdk_log_dump(stdout, "", value, value_len - 1);
+ }
+
+ /*
+ * Private info isn't accessible via the public API but
+ * may be useful for debug of blobstore based applications.
+ */
+ printf("\nBlob Private Info:\n");
+ switch (cli_context->blob->state) {
+ case SPDK_BLOB_STATE_DIRTY:
+ printf("state: DIRTY\n");
+ break;
+ case SPDK_BLOB_STATE_CLEAN:
+ printf("state: CLEAN\n");
+ break;
+ case SPDK_BLOB_STATE_LOADING:
+ printf("state: LOADING\n");
+ break;
+ default:
+ printf("state: UNKNOWN\n");
+ break;
+ }
+ printf("open ref count: %d\n",
+ cli_context->blob->open_ref);
+
+ spdk_xattr_names_free(names);
+}
+
+/*
+ * Callback for getting the first blob, shared with simple blob listing as well.
+ */
+static void
+blob_iter_cb(void *arg1, struct spdk_blob *blob, int bserrno)
+{
+ struct cli_context_t *cli_context = arg1;
+
+ if (bserrno) {
+ if (bserrno == -ENOENT) {
+ /* this simply means there are no more blobs */
+ unload_bs(cli_context, "", 0);
+ } else {
+ unload_bs(cli_context, "Error in blob iter callback",
+ bserrno);
+ }
+ return;
+ }
+
+ if (cli_context->action == CLI_LIST_BLOBS) {
+ printf("\nList BLOBS:\n");
+ printf("Found blob with ID# %" PRIu64 "\n",
+ spdk_blob_get_id(blob));
+ } else if (spdk_blob_get_id(blob) == cli_context->blobid) {
+ /*
+ * Found the blob we're looking for, but we need to finish
+ * iterating even after showing the info so that internally
+ * the blobstore logic will close the blob. Or we could
+ * chose to close it now, either way.
+ */
+ cli_context->blob = blob;
+ show_blob(cli_context);
+ }
+
+ spdk_bs_iter_next(cli_context->bs, blob, blob_iter_cb, cli_context);
+}
+
+/*
+ * Callback for setting the super blob ID.
+ */
+static void
+set_super_cb(void *arg1, int bserrno)
+{
+ struct cli_context_t *cli_context = arg1;
+
+ if (bserrno) {
+ unload_bs(cli_context, "Error in set_super callback",
+ bserrno);
+ return;
+ }
+
+ printf("Super Blob ID has been set.\n");
+ unload_bs(cli_context, "", 0);
+}
+
+/*
+ * Callback for set_xattr_open where we set or delete xattrs.
+ */
+static void
+set_xattr_cb(void *cb_arg, struct spdk_blob *blob, int bserrno)
+{
+ struct cli_context_t *cli_context = cb_arg;
+
+ if (bserrno) {
+ unload_bs(cli_context, "Error in blob open callback",
+ bserrno);
+ return;
+ }
+ cli_context->blob = blob;
+
+ if (cli_context->action == CLI_SET_XATTR) {
+ spdk_blob_set_xattr(cli_context->blob, cli_context->key,
+ cli_context->value, strlen(cli_context->value) + 1);
+ printf("Xattr has been set.\n");
+ } else {
+ spdk_blob_remove_xattr(cli_context->blob, cli_context->key);
+ printf("Xattr has been removed.\n");
+ }
+
+ spdk_blob_sync_md(cli_context->blob, sync_cb, cli_context);
+}
+
+/*
+ * Callback function for reading a blob for dumping to a file.
+ */
+static void
+read_dump_cb(void *arg1, int bserrno)
+{
+ struct cli_context_t *cli_context = arg1;
+ uint64_t bytes_written;
+
+ if (bserrno) {
+ fclose(cli_context->fp);
+ unload_bs(cli_context, "Error in read completion",
+ bserrno);
+ return;
+ }
+
+ bytes_written = fwrite(cli_context->buff, NUM_IO_UNITS, cli_context->io_unit_size,
+ cli_context->fp);
+ if (bytes_written != cli_context->io_unit_size) {
+ fclose(cli_context->fp);
+ unload_bs(cli_context, "Error with fwrite",
+ bserrno);
+ return;
+ }
+
+ printf(".");
+ if (++cli_context->io_unit_count < cli_context->blob_io_units) {
+ /* perform another read */
+ spdk_blob_io_read(cli_context->blob, cli_context->channel,
+ cli_context->buff, cli_context->io_unit_count,
+ NUM_IO_UNITS, read_dump_cb, cli_context);
+ } else {
+ /* done reading */
+ printf("\nFile write complete (to %s).\n", cli_context->file);
+ fclose(cli_context->fp);
+ spdk_blob_close(cli_context->blob, close_cb, cli_context);
+ }
+}
+
+/*
+ * Callback for write completion on the import of a file to a blob.
+ */
+static void
+write_imp_cb(void *arg1, int bserrno)
+{
+ struct cli_context_t *cli_context = arg1;
+ uint64_t bytes_read;
+
+ if (bserrno) {
+ fclose(cli_context->fp);
+ unload_bs(cli_context, "Error in write completion",
+ bserrno);
+ return;
+ }
+
+ if (cli_context->bytes_so_far < cli_context->filesize) {
+ /* perform another file read */
+ bytes_read = fread(cli_context->buff, 1,
+ cli_context->io_unit_size,
+ cli_context->fp);
+ cli_context->bytes_so_far += bytes_read;
+
+ /* if this read is < 1 io_unit, fill with 0s */
+ if (bytes_read < cli_context->io_unit_size) {
+ uint8_t *offset = cli_context->buff + bytes_read;
+ memset(offset, 0, cli_context->io_unit_size - bytes_read);
+ }
+ } else {
+ /*
+ * Done reading the file, fill the rest of the blob with 0s,
+ * yeah we're memsetting the same io_unit over and over here
+ */
+ memset(cli_context->buff, 0, cli_context->io_unit_size);
+ }
+ if (++cli_context->io_unit_count < cli_context->blob_io_units) {
+ printf(".");
+ spdk_blob_io_write(cli_context->blob, cli_context->channel,
+ cli_context->buff, cli_context->io_unit_count,
+ NUM_IO_UNITS, write_imp_cb, cli_context);
+ } else {
+ /* done writing */
+ printf("\nBlob import complete (from %s).\n", cli_context->file);
+ fclose(cli_context->fp);
+ spdk_blob_close(cli_context->blob, close_cb, cli_context);
+ }
+}
+
+/*
+ * Callback for open blobs where we'll continue on dump a blob to a file or
+ * import a file to a blob. For dump, the resulting file will always be the
+ * full size of the blob. For import, the blob will fill with the file
+ * contents first and then 0 out the rest of the blob.
+ */
+static void
+dump_imp_open_cb(void *cb_arg, struct spdk_blob *blob, int bserrno)
+{
+ struct cli_context_t *cli_context = cb_arg;
+
+ if (bserrno) {
+ unload_bs(cli_context, "Error in blob open callback",
+ bserrno);
+ return;
+ }
+ cli_context->blob = blob;
+
+ /*
+ * We'll transfer just one io_unit at a time to keep the buffer
+ * small. This could be bigger of course.
+ */
+ cli_context->buff = spdk_malloc(cli_context->io_unit_size, ALIGN_4K, NULL,
+ SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
+ if (cli_context->buff == NULL) {
+ printf("Error in allocating memory\n");
+ spdk_blob_close(cli_context->blob, close_cb, cli_context);
+ return;
+ }
+ printf("Working");
+ cli_context->blob_io_units = spdk_blob_get_num_io_units(cli_context->blob);
+ cli_context->io_unit_count = 0;
+ if (cli_context->action == CLI_DUMP_BLOB) {
+ cli_context->fp = fopen(cli_context->file, "w");
+ if (cli_context->fp == NULL) {
+ printf("Error in opening file\n");
+ spdk_blob_close(cli_context->blob, close_cb, cli_context);
+ return;
+ }
+
+ /* read a io_unit of data from the blob */
+ spdk_blob_io_read(cli_context->blob, cli_context->channel,
+ cli_context->buff, cli_context->io_unit_count,
+ NUM_IO_UNITS, read_dump_cb, cli_context);
+ } else {
+ cli_context->fp = fopen(cli_context->file, "r");
+ if (cli_context->fp == NULL) {
+ printf("Error in opening file: errno %d\n", errno);
+ spdk_blob_close(cli_context->blob, close_cb, cli_context);
+ return;
+ }
+
+ /* get the filesize then rewind read a io_unit of data from file */
+ fseek(cli_context->fp, 0L, SEEK_END);
+ cli_context->filesize = ftell(cli_context->fp);
+ rewind(cli_context->fp);
+ cli_context->bytes_so_far = fread(cli_context->buff, NUM_IO_UNITS,
+ cli_context->io_unit_size,
+ cli_context->fp);
+
+ /* if the file is < a io_unit, fill the rest with 0s */
+ if (cli_context->filesize < cli_context->io_unit_size) {
+ uint8_t *offset =
+ cli_context->buff + cli_context->filesize;
+
+ memset(offset, 0,
+ cli_context->io_unit_size - cli_context->filesize);
+ }
+
+ spdk_blob_io_write(cli_context->blob, cli_context->channel,
+ cli_context->buff, cli_context->io_unit_count,
+ NUM_IO_UNITS, write_imp_cb, cli_context);
+ }
+}
+
+/*
+ * Callback function for writing a specific pattern to io_unit 0.
+ */
+static void
+write_cb(void *arg1, int bserrno)
+{
+ struct cli_context_t *cli_context = arg1;
+
+ if (bserrno) {
+ unload_bs(cli_context, "Error in write completion",
+ bserrno);
+ return;
+ }
+ printf(".");
+ if (++cli_context->io_unit_count < cli_context->blob_io_units) {
+ spdk_blob_io_write(cli_context->blob, cli_context->channel,
+ cli_context->buff, cli_context->io_unit_count,
+ NUM_IO_UNITS, write_cb, cli_context);
+ } else {
+ /* done writing */
+ printf("\nBlob fill complete (with 0x%x).\n", cli_context->fill_value);
+ spdk_blob_close(cli_context->blob, close_cb, cli_context);
+ }
+}
+
+/*
+ * Callback function to fill a blob with a value, callback from open.
+ */
+static void
+fill_blob_cb(void *arg1, struct spdk_blob *blob, int bserrno)
+{
+ struct cli_context_t *cli_context = arg1;
+
+ if (bserrno) {
+ unload_bs(cli_context, "Error in open callback",
+ bserrno);
+ return;
+ }
+
+ cli_context->blob = blob;
+ cli_context->io_unit_count = 0;
+ cli_context->blob_io_units = spdk_blob_get_num_io_units(cli_context->blob);
+ cli_context->buff = spdk_malloc(cli_context->io_unit_size, ALIGN_4K, NULL,
+ SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
+ if (cli_context->buff == NULL) {
+ unload_bs(cli_context, "Error in allocating memory",
+ -ENOMEM);
+ return;
+ }
+
+ memset(cli_context->buff, cli_context->fill_value,
+ cli_context->io_unit_size);
+ printf("Working");
+ spdk_blob_io_write(cli_context->blob, cli_context->channel,
+ cli_context->buff,
+ STARTING_IO_UNIT, NUM_IO_UNITS, write_cb, cli_context);
+}
+
+/*
+ * Multiple actions require us to open the bs first so here we use
+ * a common callback to set a bunch of values and then move on to
+ * the next step saved off via function pointer.
+ */
+static void
+load_bs_cb(void *arg1, struct spdk_blob_store *bs, int bserrno)
+{
+ struct cli_context_t *cli_context = arg1;
+
+ if (bserrno) {
+ unload_bs(cli_context, "Error in load callback",
+ bserrno);
+ return;
+ }
+
+ cli_context->bs = bs;
+ cli_context->page_size = spdk_bs_get_page_size(cli_context->bs);
+ cli_context->io_unit_size = spdk_bs_get_io_unit_size(cli_context->bs);
+ cli_context->channel = spdk_bs_alloc_io_channel(cli_context->bs);
+ if (cli_context->channel == NULL) {
+ unload_bs(cli_context, "Error in allocating channel",
+ -ENOMEM);
+ return;
+ }
+
+ switch (cli_context->action) {
+ case CLI_SET_SUPER:
+ spdk_bs_set_super(cli_context->bs, cli_context->superid,
+ set_super_cb, cli_context);
+ break;
+ case CLI_SHOW_BS:
+ spdk_bs_get_super(cli_context->bs, show_bs_cb, cli_context);
+ break;
+ case CLI_CREATE_BLOB:
+ spdk_bs_create_blob(cli_context->bs, blob_create_cb, cli_context);
+ break;
+ case CLI_SET_XATTR:
+ case CLI_REM_XATTR:
+ spdk_bs_open_blob(cli_context->bs, cli_context->blobid,
+ set_xattr_cb, cli_context);
+ break;
+ case CLI_SHOW_BLOB:
+ case CLI_LIST_BLOBS:
+ spdk_bs_iter_first(cli_context->bs, blob_iter_cb, cli_context);
+
+ break;
+ case CLI_DUMP_BLOB:
+ case CLI_IMPORT_BLOB:
+ spdk_bs_open_blob(cli_context->bs, cli_context->blobid,
+ dump_imp_open_cb, cli_context);
+ break;
+ case CLI_FILL:
+ spdk_bs_open_blob(cli_context->bs, cli_context->blobid,
+ fill_blob_cb, cli_context);
+ break;
+
+ default:
+ /* should never get here */
+ exit(-1);
+ break;
+ }
+}
+
+/*
+ * Load the blobstore.
+ */
+static void
+load_bs(struct cli_context_t *cli_context)
+{
+ struct spdk_bdev *bdev = NULL;
+ struct spdk_bs_dev *bs_dev = NULL;
+
+ bdev = spdk_bdev_get_by_name(cli_context->bdev_name);
+ if (bdev == NULL) {
+ printf("Could not find a bdev\n");
+ spdk_app_stop(-1);
+ return;
+ }
+
+ bs_dev = spdk_bdev_create_bs_dev(bdev, NULL, NULL);
+ if (bs_dev == NULL) {
+ printf("Could not create blob bdev!!\n");
+ spdk_app_stop(-1);
+ return;
+ }
+
+ spdk_bs_load(bs_dev, NULL, load_bs_cb, cli_context);
+}
+
+/*
+ * Lists all the blobs on this blobstore.
+ */
+static void
+list_bdevs(struct cli_context_t *cli_context)
+{
+ struct spdk_bdev *bdev = NULL;
+
+ printf("\nList bdevs:\n");
+
+ bdev = spdk_bdev_first();
+ if (bdev == NULL) {
+ printf("Could not find a bdev\n");
+ }
+ while (bdev) {
+ printf("\tbdev Name: %s\n", spdk_bdev_get_name(bdev));
+ printf("\tbdev Product Name: %s\n",
+ spdk_bdev_get_product_name(bdev));
+ bdev = spdk_bdev_next(bdev);
+ }
+
+ printf("\n");
+ if (cli_context->cli_mode == CLI_MODE_CMD) {
+ spdk_app_stop(0);
+ } else {
+ cli_context->action = CLI_NONE;
+ cli_start(cli_context);
+ }
+}
+
+/*
+ * Callback function for initializing a blob.
+ */
+static void
+bs_init_cb(void *cb_arg, struct spdk_blob_store *bs,
+ int bserrno)
+{
+ struct cli_context_t *cli_context = cb_arg;
+
+ if (bserrno) {
+ unload_bs(cli_context, "Error in bs init callback",
+ bserrno);
+ return;
+ }
+ cli_context->bs = bs;
+ printf("blobstore init'd: (%p)\n", cli_context->bs);
+
+ unload_bs(cli_context, "", 0);
+}
+
+/*
+ * Initialize a new blobstore.
+ */
+static void
+init_bs(struct cli_context_t *cli_context)
+{
+ struct spdk_bdev *bdev = NULL;
+
+ bdev = spdk_bdev_get_by_name(cli_context->bdev_name);
+ if (bdev == NULL) {
+ printf("Could not find a bdev\n");
+ spdk_app_stop(-1);
+ return;
+ }
+ printf("Init blobstore using bdev Product Name: %s\n",
+ spdk_bdev_get_product_name(bdev));
+
+ cli_context->bs_dev = spdk_bdev_create_bs_dev(bdev, NULL, NULL);
+ if (cli_context->bs_dev == NULL) {
+ printf("Could not create blob bdev!!\n");
+ spdk_app_stop(-1);
+ return;
+ }
+
+ spdk_bs_init(cli_context->bs_dev, NULL, bs_init_cb,
+ cli_context);
+}
+
+static void
+spdk_bsdump_done(void *arg, int bserrno)
+{
+ struct cli_context_t *cli_context = arg;
+
+ if (cli_context->cli_mode == CLI_MODE_CMD) {
+ spdk_app_stop(0);
+ } else {
+ cli_context->action = CLI_NONE;
+ cli_start(cli_context);
+ }
+}
+
+static void
+bsdump_print_xattr(FILE *fp, const char *bstype, const char *name, const void *value,
+ size_t value_len)
+{
+ if (strncmp(bstype, "BLOBFS", SPDK_BLOBSTORE_TYPE_LENGTH) == 0) {
+ if (strcmp(name, "name") == 0) {
+ fprintf(fp, "%.*s", (int)value_len, (char *)value);
+ } else if (strcmp(name, "length") == 0 && value_len == sizeof(uint64_t)) {
+ uint64_t length;
+
+ memcpy(&length, value, sizeof(length));
+ fprintf(fp, "%" PRIu64, length);
+ } else {
+ fprintf(fp, "?");
+ }
+ } else if (strncmp(bstype, "LVOLSTORE", SPDK_BLOBSTORE_TYPE_LENGTH) == 0) {
+ if (strcmp(name, "name") == 0) {
+ fprintf(fp, "%s", (char *)value);
+ } else if (strcmp(name, "uuid") == 0 && value_len == sizeof(struct spdk_uuid)) {
+ char uuid[SPDK_UUID_STRING_LEN];
+
+ spdk_uuid_fmt_lower(uuid, sizeof(uuid), (struct spdk_uuid *)value);
+ fprintf(fp, "%s", uuid);
+ } else {
+ fprintf(fp, "?");
+ }
+ } else {
+ fprintf(fp, "?");
+ }
+}
+
+/*
+ * Dump metadata of an existing blobstore in a human-readable format.
+ */
+static void
+dump_bs(struct cli_context_t *cli_context)
+{
+ struct spdk_bdev *bdev = NULL;
+
+ bdev = spdk_bdev_get_by_name(cli_context->bdev_name);
+ if (bdev == NULL) {
+ printf("Could not find a bdev\n");
+ spdk_app_stop(-1);
+ return;
+ }
+ printf("Init blobstore using bdev Product Name: %s\n",
+ spdk_bdev_get_product_name(bdev));
+
+ cli_context->bs_dev = spdk_bdev_create_bs_dev(bdev, NULL, NULL);
+ if (cli_context->bs_dev == NULL) {
+ printf("Could not create blob bdev!!\n");
+ spdk_app_stop(-1);
+ return;
+ }
+
+ spdk_bs_dump(cli_context->bs_dev, stdout, bsdump_print_xattr, spdk_bsdump_done, cli_context);
+}
+
+/*
+ * Common cmd/option parser for command and shell modes.
+ */
+static bool
+cmd_parser(int argc, char **argv, struct cli_context_t *cli_context)
+{
+ int op;
+ int cmd_chosen = 0;
+ char resp;
+
+ while ((op = getopt(argc, argv, "b:c:d:f:hil:m:n:p:r:s:DST:Xx:")) != -1) {
+ switch (op) {
+ case 'b':
+ if (strcmp(cli_context->bdev_name, "") == 0) {
+ snprintf(cli_context->bdev_name, BUFSIZE, "%s", optarg);
+ } else {
+ printf("Current setting for -b is: %s\n", cli_context->bdev_name);
+ usage(cli_context, "ERROR: -b option can only be set once.\n");
+ }
+ break;
+ case 'c':
+ if (cli_context->app_started == false) {
+ cli_context->config_file = optarg;
+ } else {
+ usage(cli_context, "ERROR: -c option not valid during shell mode.\n");
+ }
+ break;
+ case 'D':
+ cmd_chosen++;
+ cli_context->action = CLI_DUMP_BS;
+ break;
+ case 'd':
+ if (argv[optind] != NULL) {
+ cmd_chosen++;
+ cli_context->action = CLI_DUMP_BLOB;
+ cli_context->blobid = spdk_strtoll(optarg, 10);
+ snprintf(cli_context->file, BUFSIZE, "%s", argv[optind]);
+ } else {
+ usage(cli_context, "ERROR: missing parameter.\n");
+ }
+ break;
+ case 'f':
+ if (argv[optind] != NULL) {
+ cmd_chosen++;
+ cli_context->action = CLI_FILL;
+ cli_context->blobid = spdk_strtoll(optarg, 10);
+ cli_context->fill_value = spdk_strtol(argv[optind], 10);
+ } else {
+ usage(cli_context, "ERROR: missing parameter.\n");
+ }
+ break;
+ case 'h':
+ cmd_chosen++;
+ cli_context->action = CLI_HELP;
+ break;
+ case 'i':
+ if (cli_context->cli_mode != CLI_MODE_SCRIPT) {
+ printf("Your entire blobstore will be destroyed. Are you sure? (y/n) ");
+ if (scanf("%c%*c", &resp)) {
+ if (resp == 'y' || resp == 'Y') {
+ cmd_chosen++;
+ cli_context->action = CLI_INIT_BS;
+ } else {
+ if (cli_context->cli_mode == CLI_MODE_CMD) {
+ spdk_app_stop(0);
+ return false;
+ }
+ }
+ }
+ } else {
+ cmd_chosen++;
+ cli_context->action = CLI_INIT_BS;
+ }
+ break;
+ case 'r':
+ if (argv[optind] != NULL) {
+ cmd_chosen++;
+ cli_context->action = CLI_REM_XATTR;
+ cli_context->blobid = spdk_strtoll(optarg, 10);
+ snprintf(cli_context->key, BUFSIZE, "%s", argv[optind]);
+ } else {
+ usage(cli_context, "ERROR: missing parameter.\n");
+ }
+ break;
+ case 'l':
+ if (strcmp("bdevs", optarg) == 0) {
+ cmd_chosen++;
+ cli_context->action = CLI_LIST_BDEVS;
+ } else if (strcmp("blobs", optarg) == 0) {
+ cmd_chosen++;
+ cli_context->action = CLI_LIST_BLOBS;
+ } else {
+ usage(cli_context, "ERROR: invalid option for list\n");
+ }
+ break;
+ case 'm':
+ if (argv[optind] != NULL) {
+ cmd_chosen++;
+ cli_context->action = CLI_IMPORT_BLOB;
+ cli_context->blobid = spdk_strtoll(optarg, 10);
+ snprintf(cli_context->file, BUFSIZE, "%s", argv[optind]);
+ } else {
+ usage(cli_context, "ERROR: missing parameter.\n");
+ }
+ break;
+ case 'n':
+ cli_context->num_clusters = spdk_strtol(optarg, 10);
+ if (cli_context->num_clusters > 0) {
+ cmd_chosen++;
+ cli_context->action = CLI_CREATE_BLOB;
+ } else {
+ usage(cli_context, "ERROR: invalid option for new\n");
+ }
+ break;
+ case 'p':
+ cmd_chosen++;
+ cli_context->action = CLI_SET_SUPER;
+ cli_context->superid = spdk_strtoll(optarg, 10);
+ break;
+ case 'S':
+ if (cli_context->cli_mode == CLI_MODE_CMD) {
+ cmd_chosen++;
+ cli_context->cli_mode = CLI_MODE_SHELL;
+ }
+ cli_context->action = CLI_NONE;
+ break;
+ case 's':
+ cmd_chosen++;
+ if (strcmp("bs", optarg) == 0) {
+ cli_context->action = CLI_SHOW_BS;
+ } else {
+ cli_context->action = CLI_SHOW_BLOB;
+ cli_context->blobid = spdk_strtoll(optarg, 10);
+ }
+ break;
+ case 'T':
+ if (cli_context->cli_mode == CLI_MODE_CMD) {
+ cmd_chosen++;
+ cli_context->cli_mode = CLI_MODE_SCRIPT;
+ if (argv[optind] && (strcmp("ignore", argv[optind]) == 0)) {
+ g_script.ignore_errors = true;
+ } else {
+ g_script.ignore_errors = false;
+ }
+ snprintf(cli_context->script_file, BUFSIZE, "%s", optarg);
+ } else {
+ cli_context->action = CLI_NONE;
+ }
+ break;
+ case 'X':
+ cmd_chosen++;
+ cli_context->action = CLI_SHELL_EXIT;
+ break;
+ case 'x':
+ if (argv[optind] != NULL || argv[optind + 1] != NULL) {
+ cmd_chosen++;
+ cli_context->action = CLI_SET_XATTR;
+ cli_context->blobid = spdk_strtoll(optarg, 10);
+ snprintf(cli_context->key, BUFSIZE, "%s", argv[optind]);
+ snprintf(cli_context->value, BUFSIZE, "%s", argv[optind + 1]);
+ } else {
+ usage(cli_context, "ERROR: missing parameter.\n");
+ }
+ break;
+ default:
+ usage(cli_context, "ERROR: invalid option\n");
+ }
+ /* only one actual command can be done at a time */
+ if (cmd_chosen > 1) {
+ usage(cli_context, "Error: Please choose only one command\n");
+ }
+ }
+
+ if (cli_context->cli_mode == CLI_MODE_CMD && cmd_chosen == 0) {
+ usage(cli_context, "Error: Please choose a command.\n");
+ }
+
+ /*
+ * We don't check the local boolean because in some modes it will have been set
+ * on and earlier command.
+ */
+ if (strcmp(cli_context->bdev_name, "") == 0) {
+ usage(cli_context, "Error: -b option is required.\n");
+ cmd_chosen = 0;
+ }
+
+ /* in shell mode we'll call getopt multiple times so need to reset its index */
+ optind = 0;
+ return (cmd_chosen == 1);
+}
+
+/*
+ * In script mode, we parsed a script file at startup and saved off a bunch of cmd
+ * lines that we now parse with each run of cli_start so we us the same cmd parser
+ * as cmd and shell modes.
+ */
+static bool
+line_parser(struct cli_context_t *cli_context)
+{
+ bool cmd_chosen;
+ char *tok = NULL;
+ int blob_num = 0;
+ int start_idx = cli_context->argc;
+ int i;
+
+ printf("\nSCRIPT NOW PROCESSING: %s\n", g_script.cmdline[g_script.cmdline_idx]);
+ tok = strtok(g_script.cmdline[g_script.cmdline_idx], " ");
+ while (tok != NULL) {
+ /*
+ * We support one replaceable token right now, a $Bn
+ * represents the blobid that was created in position n
+ * so fish this out now and use it here.
+ */
+ cli_context->argv[cli_context->argc] = strdup(tok);
+ if (tok[0] == '$' && tok[1] == 'B') {
+ tok += 2;
+ blob_num = spdk_strtol(tok, 10);
+ if (blob_num >= 0 && blob_num < MAX_SCRIPT_BLOBS) {
+ cli_context->argv[cli_context->argc] =
+ realloc(cli_context->argv[cli_context->argc], BUFSIZE);
+ if (cli_context->argv[cli_context->argc] == NULL) {
+ printf("ERROR: unable to realloc memory\n");
+ spdk_app_stop(-1);
+ }
+ if (g_script.blobid[blob_num] == 0) {
+ printf("ERROR: There is no blob for $B%d\n",
+ blob_num);
+ }
+ snprintf(cli_context->argv[cli_context->argc], BUFSIZE,
+ "%" PRIu64, g_script.blobid[blob_num]);
+ } else {
+ printf("ERROR: Invalid token or exceeded max blobs of %d\n",
+ MAX_SCRIPT_BLOBS);
+ }
+ }
+ cli_context->argc++;
+ tok = strtok(NULL, " ");
+ }
+
+ /* call parse cmd line with user input as args */
+ cmd_chosen = cmd_parser(cli_context->argc, &cli_context->argv[0], cli_context);
+
+ /* free strdup memory and reset arg count for next shell interaction */
+ for (i = start_idx; i < cli_context->argc; i++) {
+ free(cli_context->argv[i]);
+ cli_context->argv[i] = NULL;
+ }
+ cli_context->argc = 1;
+
+ g_script.cmdline_idx++;
+ assert(g_script.cmdline_idx < MAX_SCRIPT_LINES);
+
+ if (cmd_chosen == false) {
+ printf("ERROR: Invalid script line starting with: %s\n\n",
+ g_script.cmdline[g_script.cmdline_idx - 1]);
+ if (g_script.ignore_errors == false) {
+ printf("** Aborting **\n");
+ cli_context->action = CLI_SHELL_EXIT;
+ cmd_chosen = true;
+ unload_bs(cli_context, "", 0);
+ } else {
+ printf("** Skipping **\n");
+ }
+ }
+
+ return cmd_chosen;
+}
+
+/*
+ * For script mode, we read a series of commands from a text file and store them
+ * in a global struct. That, along with the cli_mode that tells us we're in
+ * script mode is what feeds the rest of the app in the same way as is it were
+ * getting commands from shell mode.
+ */
+static void
+parse_script(struct cli_context_t *cli_context)
+{
+ FILE *fp = NULL;
+ size_t bufsize = BUFSIZE;
+ int64_t bytes_in = 0;
+ int i = 0;
+
+ /* initialize global script values */
+ for (i = 0; i < MAX_SCRIPT_BLOBS; i++) {
+ g_script.blobid[i] = 0;
+ }
+ g_script.blobid_idx = 0;
+ g_script.cmdline_idx = 0;
+ i = 0;
+
+ fp = fopen(cli_context->script_file, "r");
+ if (fp == NULL) {
+ printf("ERROR: unable to open script: %s\n",
+ cli_context->script_file);
+ cli_cleanup(cli_context);
+ exit(-1);
+ }
+
+ do {
+ bytes_in = getline(&g_script.cmdline[i], &bufsize, fp);
+ if (bytes_in > 0) {
+ /* replace newline with null */
+ spdk_str_chomp(g_script.cmdline[i]);
+
+ /* ignore comments */
+ if (g_script.cmdline[i][0] != '#') {
+ i++;
+ }
+ }
+ } while (bytes_in != -1 && i < MAX_SCRIPT_LINES - 1);
+ fclose(fp);
+
+ /* add an exit cmd in case they didn't */
+ g_script.cmdline[i] = realloc(g_script.cmdline[i], BUFSIZE);
+ if (g_script.cmdline[i] == NULL) {
+ int j;
+
+ for (j = 0; j < i; j++) {
+ free(g_script.cmdline[j]);
+ g_script.cmdline[j] = NULL;
+ }
+ unload_bs(cli_context, "ERROR: unable to alloc memory.\n", 0);
+ }
+ snprintf(g_script.cmdline[i], BUFSIZE, "%s", "-X");
+ g_script.max_index = i;
+}
+
+/*
+ * Provides for a shell interface as opposed to one shot command line.
+ */
+static bool
+cli_shell(void *arg1, void *arg2)
+{
+ struct cli_context_t *cli_context = arg1;
+ char *line = NULL;
+ ssize_t buf_size = 0;
+ ssize_t bytes_in = 0;
+ ssize_t tok_len = 0;
+ char *tok = NULL;
+ bool cmd_chosen = false;
+ int start_idx = cli_context->argc;
+ int i;
+
+ printf("blob> ");
+ bytes_in = getline(&line, &buf_size, stdin);
+
+ /* If getline() failed (EOF), exit the shell. */
+ if (bytes_in < 0) {
+ free(line);
+ cli_context->action = CLI_SHELL_EXIT;
+ return true;
+ }
+
+ /* parse input and update cli_context so we can use common option parser */
+ if (bytes_in > 0) {
+ tok = strtok(line, " ");
+ }
+ while ((tok != NULL) && (cli_context->argc < MAX_ARGS)) {
+ cli_context->argv[cli_context->argc] = strdup(tok);
+ tok_len = strlen(tok);
+ cli_context->argc++;
+ tok = strtok(NULL, " ");
+ }
+
+ /* replace newline on last arg with null */
+ if (tok_len) {
+ spdk_str_chomp(cli_context->argv[cli_context->argc - 1]);
+ }
+
+ /* call parse cmd line with user input as args */
+ cmd_chosen = cmd_parser(cli_context->argc, &cli_context->argv[0], cli_context);
+
+ /* free strdup mem & reset arg count for next shell interaction */
+ for (i = start_idx; i < cli_context->argc; i++) {
+ free(cli_context->argv[i]);
+ cli_context->argv[i] = NULL;
+ }
+ cli_context->argc = 1;
+
+ free(line);
+
+ return cmd_chosen;
+}
+
+/*
+ * This is the function we pass into the SPDK framework that gets
+ * called first.
+ */
+static void
+cli_start(void *arg1)
+{
+ struct cli_context_t *cli_context = arg1;
+
+ /*
+ * If we're in script mode, we already have a list of commands so
+ * just need to pull them out one at a time and process them.
+ */
+ if (cli_context->cli_mode == CLI_MODE_SCRIPT) {
+ while (line_parser(cli_context) == false);
+ }
+
+ /*
+ * The initial cmd line options are parsed once before this function is
+ * called so if there is no action, we're in shell mode and will loop
+ * here until a a valid option is parsed and returned.
+ */
+ if (cli_context->action == CLI_NONE) {
+ while (cli_shell(cli_context, NULL) == false);
+ }
+
+ /* Decide what to do next based on cmd line parsing. */
+ switch (cli_context->action) {
+ case CLI_SET_SUPER:
+ case CLI_SHOW_BS:
+ case CLI_CREATE_BLOB:
+ case CLI_SET_XATTR:
+ case CLI_REM_XATTR:
+ case CLI_SHOW_BLOB:
+ case CLI_LIST_BLOBS:
+ case CLI_DUMP_BLOB:
+ case CLI_IMPORT_BLOB:
+ case CLI_FILL:
+ load_bs(cli_context);
+ break;
+ case CLI_INIT_BS:
+ init_bs(cli_context);
+ break;
+ case CLI_DUMP_BS:
+ dump_bs(cli_context);
+ break;
+ case CLI_LIST_BDEVS:
+ list_bdevs(cli_context);
+ break;
+ case CLI_SHELL_EXIT:
+ /*
+ * Because shell mode reuses cmd mode functions, the blobstore
+ * is loaded/unloaded with every action so we just need to
+ * stop the framework. For this app there's no need to optimize
+ * and keep the blobstore open while the app is in shell mode.
+ */
+ spdk_app_stop(0);
+ break;
+ case CLI_HELP:
+ usage(cli_context, "");
+ unload_complete(cli_context, 0);
+ break;
+ default:
+ /* should never get here */
+ exit(-1);
+ break;
+ }
+}
+
+int
+main(int argc, char **argv)
+{
+ struct spdk_app_opts opts = {};
+ struct cli_context_t *cli_context = NULL;
+ bool cmd_chosen;
+ int rc = 0;
+
+ if (argc < 2) {
+ usage(cli_context, "ERROR: Invalid option\n");
+ exit(-1);
+ }
+
+ cli_context = calloc(1, sizeof(struct cli_context_t));
+ if (cli_context == NULL) {
+ printf("ERROR: could not allocate context structure\n");
+ exit(-1);
+ }
+
+ /* default to CMD mode until we've parsed the first parms */
+ cli_context->cli_mode = CLI_MODE_CMD;
+ cli_context->argv[0] = strdup(argv[0]);
+ cli_context->argc = 1;
+
+ /* parse command line */
+ cmd_chosen = cmd_parser(argc, argv, cli_context);
+ free(cli_context->argv[0]);
+ cli_context->argv[0] = NULL;
+ if (cmd_chosen == false) {
+ cli_cleanup(cli_context);
+ exit(-1);
+ }
+
+ /* after displaying help, just exit */
+ if (cli_context->action == CLI_HELP) {
+ usage(cli_context, "");
+ cli_cleanup(cli_context);
+ exit(-1);
+ }
+
+ /* if they don't supply a conf name, use the default */
+ if (!cli_context->config_file) {
+ cli_context->config_file = program_conf;
+ }
+
+ /* if the config file doesn't exist, tell them how to make one */
+ if (access(cli_context->config_file, F_OK) == -1) {
+ printf("Error: No config file found.\n");
+ printf("To create a config file named 'blobcli.conf' for your NVMe device:\n");
+ printf(" <path to spdk>/scripts/gen_nvme.sh > blobcli.conf\n");
+ printf("and then re-run the cli tool.\n");
+ exit(-1);
+ }
+
+ /*
+ * For script mode we keep a bunch of stuff in a global since
+ * none if it is passed back and forth to SPDK.
+ */
+ if (cli_context->cli_mode == CLI_MODE_SCRIPT) {
+ /*
+ * Now we'll build up the global which will direct this run of the app
+ * as it will have a list (g_script) of all of the commands line by
+ * line as if they were typed in on the shell at cmd line.
+ */
+ parse_script(cli_context);
+ }
+
+ /* Set default values in opts struct along with name and conf file. */
+ spdk_app_opts_init(&opts);
+ opts.name = "blobcli";
+ opts.config_file = cli_context->config_file;
+
+ cli_context->app_started = true;
+ rc = spdk_app_start(&opts, cli_start, cli_context);
+ if (rc) {
+ printf("ERROR!\n");
+ }
+
+ /* Free up memory that we allocated */
+ cli_cleanup(cli_context);
+
+ /* Gracefully close out all of the SPDK subsystems. */
+ spdk_app_fini();
+ return rc;
+}
diff --git a/src/spdk/examples/blob/hello_world/.gitignore b/src/spdk/examples/blob/hello_world/.gitignore
new file mode 100644
index 000000000..683a22556
--- /dev/null
+++ b/src/spdk/examples/blob/hello_world/.gitignore
@@ -0,0 +1 @@
+hello_blob
diff --git a/src/spdk/examples/blob/hello_world/Makefile b/src/spdk/examples/blob/hello_world/Makefile
new file mode 100644
index 000000000..0b5e89396
--- /dev/null
+++ b/src/spdk/examples/blob/hello_world/Makefile
@@ -0,0 +1,45 @@
+#
+# Copyright (c) Intel Corporation.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Intel Corporation nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+include $(SPDK_ROOT_DIR)/mk/spdk.modules.mk
+
+APP = hello_blob
+
+C_SRCS := hello_blob.c
+
+SPDK_LIB_LIST = $(ALL_MODULES_LIST)
+SPDK_LIB_LIST += $(EVENT_BDEV_SUBSYSTEM)
+SPDK_LIB_LIST += bdev accel event thread util conf trace \
+ log jsonrpc json rpc sock notify
+
+include $(SPDK_ROOT_DIR)/mk/spdk.app.mk
diff --git a/src/spdk/examples/blob/hello_world/hello_blob.c b/src/spdk/examples/blob/hello_world/hello_blob.c
new file mode 100644
index 000000000..41730ce86
--- /dev/null
+++ b/src/spdk/examples/blob/hello_world/hello_blob.c
@@ -0,0 +1,498 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+
+#include "spdk/bdev.h"
+#include "spdk/env.h"
+#include "spdk/event.h"
+#include "spdk/blob_bdev.h"
+#include "spdk/blob.h"
+#include "spdk/log.h"
+
+/*
+ * We'll use this struct to gather housekeeping hello_context to pass between
+ * our events and callbacks.
+ */
+struct hello_context_t {
+ struct spdk_blob_store *bs;
+ struct spdk_blob *blob;
+ spdk_blob_id blobid;
+ struct spdk_io_channel *channel;
+ uint8_t *read_buff;
+ uint8_t *write_buff;
+ uint64_t io_unit_size;
+ int rc;
+};
+
+/*
+ * Free up memory that we allocated.
+ */
+static void
+hello_cleanup(struct hello_context_t *hello_context)
+{
+ spdk_free(hello_context->read_buff);
+ spdk_free(hello_context->write_buff);
+ free(hello_context);
+}
+
+/*
+ * Callback routine for the blobstore unload.
+ */
+static void
+unload_complete(void *cb_arg, int bserrno)
+{
+ struct hello_context_t *hello_context = cb_arg;
+
+ SPDK_NOTICELOG("entry\n");
+ if (bserrno) {
+ SPDK_ERRLOG("Error %d unloading the bobstore\n", bserrno);
+ hello_context->rc = bserrno;
+ }
+
+ spdk_app_stop(hello_context->rc);
+}
+
+/*
+ * Unload the blobstore, cleaning up as needed.
+ */
+static void
+unload_bs(struct hello_context_t *hello_context, char *msg, int bserrno)
+{
+ if (bserrno) {
+ SPDK_ERRLOG("%s (err %d)\n", msg, bserrno);
+ hello_context->rc = bserrno;
+ }
+ if (hello_context->bs) {
+ if (hello_context->channel) {
+ spdk_bs_free_io_channel(hello_context->channel);
+ }
+ spdk_bs_unload(hello_context->bs, unload_complete, hello_context);
+ } else {
+ spdk_app_stop(bserrno);
+ }
+}
+
+/*
+ * Callback routine for the deletion of a blob.
+ */
+static void
+delete_complete(void *arg1, int bserrno)
+{
+ struct hello_context_t *hello_context = arg1;
+
+ SPDK_NOTICELOG("entry\n");
+ if (bserrno) {
+ unload_bs(hello_context, "Error in delete completion",
+ bserrno);
+ return;
+ }
+
+ /* We're all done, we can unload the blobstore. */
+ unload_bs(hello_context, "", 0);
+}
+
+/*
+ * Function for deleting a blob.
+ */
+static void
+delete_blob(void *arg1, int bserrno)
+{
+ struct hello_context_t *hello_context = arg1;
+
+ SPDK_NOTICELOG("entry\n");
+ if (bserrno) {
+ unload_bs(hello_context, "Error in close completion",
+ bserrno);
+ return;
+ }
+
+ spdk_bs_delete_blob(hello_context->bs, hello_context->blobid,
+ delete_complete, hello_context);
+}
+
+/*
+ * Callback function for reading a blob.
+ */
+static void
+read_complete(void *arg1, int bserrno)
+{
+ struct hello_context_t *hello_context = arg1;
+ int match_res = -1;
+
+ SPDK_NOTICELOG("entry\n");
+ if (bserrno) {
+ unload_bs(hello_context, "Error in read completion",
+ bserrno);
+ return;
+ }
+
+ /* Now let's make sure things match. */
+ match_res = memcmp(hello_context->write_buff, hello_context->read_buff,
+ hello_context->io_unit_size);
+ if (match_res) {
+ unload_bs(hello_context, "Error in data compare", -1);
+ return;
+ } else {
+ SPDK_NOTICELOG("read SUCCESS and data matches!\n");
+ }
+
+ /* Now let's close it and delete the blob in the callback. */
+ spdk_blob_close(hello_context->blob, delete_blob, hello_context);
+}
+
+/*
+ * Function for reading a blob.
+ */
+static void
+read_blob(struct hello_context_t *hello_context)
+{
+ SPDK_NOTICELOG("entry\n");
+
+ hello_context->read_buff = spdk_malloc(hello_context->io_unit_size,
+ 0x1000, NULL, SPDK_ENV_LCORE_ID_ANY,
+ SPDK_MALLOC_DMA);
+ if (hello_context->read_buff == NULL) {
+ unload_bs(hello_context, "Error in memory allocation",
+ -ENOMEM);
+ return;
+ }
+
+ /* Issue the read and compare the results in the callback. */
+ spdk_blob_io_read(hello_context->blob, hello_context->channel,
+ hello_context->read_buff, 0, 1, read_complete,
+ hello_context);
+}
+
+/*
+ * Callback function for writing a blob.
+ */
+static void
+write_complete(void *arg1, int bserrno)
+{
+ struct hello_context_t *hello_context = arg1;
+
+ SPDK_NOTICELOG("entry\n");
+ if (bserrno) {
+ unload_bs(hello_context, "Error in write completion",
+ bserrno);
+ return;
+ }
+
+ /* Now let's read back what we wrote and make sure it matches. */
+ read_blob(hello_context);
+}
+
+/*
+ * Function for writing to a blob.
+ */
+static void
+blob_write(struct hello_context_t *hello_context)
+{
+ SPDK_NOTICELOG("entry\n");
+
+ /*
+ * Buffers for data transfer need to be allocated via SPDK. We will
+ * transfer 1 io_unit of 4K aligned data at offset 0 in the blob.
+ */
+ hello_context->write_buff = spdk_malloc(hello_context->io_unit_size,
+ 0x1000, NULL, SPDK_ENV_LCORE_ID_ANY,
+ SPDK_MALLOC_DMA);
+ if (hello_context->write_buff == NULL) {
+ unload_bs(hello_context, "Error in allocating memory",
+ -ENOMEM);
+ return;
+ }
+ memset(hello_context->write_buff, 0x5a, hello_context->io_unit_size);
+
+ /* Now we have to allocate a channel. */
+ hello_context->channel = spdk_bs_alloc_io_channel(hello_context->bs);
+ if (hello_context->channel == NULL) {
+ unload_bs(hello_context, "Error in allocating channel",
+ -ENOMEM);
+ return;
+ }
+
+ /* Let's perform the write, 1 io_unit at offset 0. */
+ spdk_blob_io_write(hello_context->blob, hello_context->channel,
+ hello_context->write_buff,
+ 0, 1, write_complete, hello_context);
+}
+
+/*
+ * Callback function for sync'ing metadata.
+ */
+static void
+sync_complete(void *arg1, int bserrno)
+{
+ struct hello_context_t *hello_context = arg1;
+
+ SPDK_NOTICELOG("entry\n");
+ if (bserrno) {
+ unload_bs(hello_context, "Error in sync callback",
+ bserrno);
+ return;
+ }
+
+ /* Blob has been created & sized & MD sync'd, let's write to it. */
+ blob_write(hello_context);
+}
+
+static void
+resize_complete(void *cb_arg, int bserrno)
+{
+ struct hello_context_t *hello_context = cb_arg;
+ uint64_t total = 0;
+
+ if (bserrno) {
+ unload_bs(hello_context, "Error in blob resize", bserrno);
+ return;
+ }
+
+ total = spdk_blob_get_num_clusters(hello_context->blob);
+ SPDK_NOTICELOG("resized blob now has USED clusters of %" PRIu64 "\n",
+ total);
+
+ /*
+ * Metadata is stored in volatile memory for performance
+ * reasons and therefore needs to be synchronized with
+ * non-volatile storage to make it persistent. This can be
+ * done manually, as shown here, or if not it will be done
+ * automatically when the blob is closed. It is always a
+ * good idea to sync after making metadata changes unless
+ * it has an unacceptable impact on application performance.
+ */
+ spdk_blob_sync_md(hello_context->blob, sync_complete, hello_context);
+}
+
+/*
+ * Callback function for opening a blob.
+ */
+static void
+open_complete(void *cb_arg, struct spdk_blob *blob, int bserrno)
+{
+ struct hello_context_t *hello_context = cb_arg;
+ uint64_t free = 0;
+
+ SPDK_NOTICELOG("entry\n");
+ if (bserrno) {
+ unload_bs(hello_context, "Error in open completion",
+ bserrno);
+ return;
+ }
+
+
+ hello_context->blob = blob;
+ free = spdk_bs_free_cluster_count(hello_context->bs);
+ SPDK_NOTICELOG("blobstore has FREE clusters of %" PRIu64 "\n",
+ free);
+
+ /*
+ * Before we can use our new blob, we have to resize it
+ * as the initial size is 0. For this example we'll use the
+ * full size of the blobstore but it would be expected that
+ * there'd usually be many blobs of various sizes. The resize
+ * unit is a cluster.
+ */
+ spdk_blob_resize(hello_context->blob, free, resize_complete, hello_context);
+}
+
+/*
+ * Callback function for creating a blob.
+ */
+static void
+blob_create_complete(void *arg1, spdk_blob_id blobid, int bserrno)
+{
+ struct hello_context_t *hello_context = arg1;
+
+ SPDK_NOTICELOG("entry\n");
+ if (bserrno) {
+ unload_bs(hello_context, "Error in blob create callback",
+ bserrno);
+ return;
+ }
+
+ hello_context->blobid = blobid;
+ SPDK_NOTICELOG("new blob id %" PRIu64 "\n", hello_context->blobid);
+
+ /* We have to open the blob before we can do things like resize. */
+ spdk_bs_open_blob(hello_context->bs, hello_context->blobid,
+ open_complete, hello_context);
+}
+
+/*
+ * Function for creating a blob.
+ */
+static void
+create_blob(struct hello_context_t *hello_context)
+{
+ SPDK_NOTICELOG("entry\n");
+ spdk_bs_create_blob(hello_context->bs, blob_create_complete, hello_context);
+}
+
+/*
+ * Callback function for initializing the blobstore.
+ */
+static void
+bs_init_complete(void *cb_arg, struct spdk_blob_store *bs,
+ int bserrno)
+{
+ struct hello_context_t *hello_context = cb_arg;
+
+ SPDK_NOTICELOG("entry\n");
+ if (bserrno) {
+ unload_bs(hello_context, "Error init'ing the blobstore",
+ bserrno);
+ return;
+ }
+
+ hello_context->bs = bs;
+ SPDK_NOTICELOG("blobstore: %p\n", hello_context->bs);
+ /*
+ * We will use the io_unit size in allocating buffers, etc., later
+ * so we'll just save it in out context buffer here.
+ */
+ hello_context->io_unit_size = spdk_bs_get_io_unit_size(hello_context->bs);
+
+ /*
+ * The blobstore has been initialized, let's create a blob.
+ * Note that we could pass a message back to ourselves using
+ * spdk_thread_send_msg() if we wanted to keep our processing
+ * time limited.
+ */
+ create_blob(hello_context);
+}
+
+/*
+ * Our initial event that kicks off everything from main().
+ */
+static void
+hello_start(void *arg1)
+{
+ struct hello_context_t *hello_context = arg1;
+ struct spdk_bdev *bdev = NULL;
+ struct spdk_bs_dev *bs_dev = NULL;
+
+ SPDK_NOTICELOG("entry\n");
+ /*
+ * Get the bdev. For this example it is our malloc (RAM)
+ * disk configured via hello_blob.conf that was passed
+ * in when we started the SPDK app framework so we can
+ * get it via its name.
+ */
+ bdev = spdk_bdev_get_by_name("Malloc0");
+ if (bdev == NULL) {
+ SPDK_ERRLOG("Could not find a bdev\n");
+ spdk_app_stop(-1);
+ return;
+ }
+
+ /*
+ * spdk_bs_init() requires us to fill out the structure
+ * spdk_bs_dev with a set of callbacks. These callbacks
+ * implement read, write, and other operations on the
+ * underlying disks. As a convenience, a utility function
+ * is provided that creates an spdk_bs_dev that implements
+ * all of the callbacks by forwarding the I/O to the
+ * SPDK bdev layer. Other helper functions are also
+ * available in the blob lib in blob_bdev.c that simply
+ * make it easier to layer blobstore on top of a bdev.
+ * However blobstore can be more tightly integrated into
+ * any lower layer, such as NVMe for example.
+ */
+ bs_dev = spdk_bdev_create_bs_dev(bdev, NULL, NULL);
+ if (bs_dev == NULL) {
+ SPDK_ERRLOG("Could not create blob bdev!!\n");
+ spdk_app_stop(-1);
+ return;
+ }
+
+ spdk_bs_init(bs_dev, NULL, bs_init_complete, hello_context);
+}
+
+int
+main(int argc, char **argv)
+{
+ struct spdk_app_opts opts = {};
+ int rc = 0;
+ struct hello_context_t *hello_context = NULL;
+
+ SPDK_NOTICELOG("entry\n");
+
+ /* Set default values in opts structure. */
+ spdk_app_opts_init(&opts);
+
+ /*
+ * Setup a few specifics before we init, for most SPDK cmd line
+ * apps, the config file will be passed in as an arg but to make
+ * this example super simple we just hardcode it. We also need to
+ * specify a name for the app.
+ */
+ opts.name = "hello_blob";
+ opts.json_config_file = argv[1];
+
+
+ /*
+ * Now we'll allocate and initialize the blobstore itself. We
+ * can pass in an spdk_bs_opts if we want something other than
+ * the defaults (cluster size, etc), but here we'll just take the
+ * defaults. We'll also pass in a struct that we'll use for
+ * callbacks so we've got efficient bookeeping of what we're
+ * creating. This is an async operation and bs_init_complete()
+ * will be called when it is complete.
+ */
+ hello_context = calloc(1, sizeof(struct hello_context_t));
+ if (hello_context != NULL) {
+ /*
+ * spdk_app_start() will block running hello_start() until
+ * spdk_app_stop() is called by someone (not simply when
+ * hello_start() returns), or if an error occurs during
+ * spdk_app_start() before hello_start() runs.
+ */
+ rc = spdk_app_start(&opts, hello_start, hello_context);
+ if (rc) {
+ SPDK_NOTICELOG("ERROR!\n");
+ } else {
+ SPDK_NOTICELOG("SUCCESS!\n");
+ }
+ /* Free up memory that we allocated */
+ hello_cleanup(hello_context);
+ } else {
+ SPDK_ERRLOG("Could not alloc hello_context struct!!\n");
+ rc = -ENOMEM;
+ }
+
+ /* Gracefully close out all of the SPDK subsystems. */
+ spdk_app_fini();
+ return rc;
+}
diff --git a/src/spdk/examples/blob/hello_world/hello_blob.json b/src/spdk/examples/blob/hello_world/hello_blob.json
new file mode 100644
index 000000000..10ded9d47
--- /dev/null
+++ b/src/spdk/examples/blob/hello_world/hello_blob.json
@@ -0,0 +1,17 @@
+{
+ "subsystems": [
+ {
+ "subsystem": "bdev",
+ "config": [
+ {
+ "method": "bdev_malloc_create",
+ "params": {
+ "name": "Malloc0",
+ "num_blocks": 32768,
+ "block_size": 512
+ }
+ }
+ ]
+ }
+ ]
+}
diff --git a/src/spdk/examples/ioat/Makefile b/src/spdk/examples/ioat/Makefile
new file mode 100644
index 000000000..d4d62b91e
--- /dev/null
+++ b/src/spdk/examples/ioat/Makefile
@@ -0,0 +1,44 @@
+#
+# BSD LICENSE
+#
+# Copyright (c) Intel Corporation.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Intel Corporation nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+DIRS-y += perf verify
+
+.PHONY: all clean $(DIRS-y)
+
+all: $(DIRS-y)
+clean: $(DIRS-y)
+
+include $(SPDK_ROOT_DIR)/mk/spdk.subdirs.mk
diff --git a/src/spdk/examples/ioat/perf/.gitignore b/src/spdk/examples/ioat/perf/.gitignore
new file mode 100644
index 000000000..60abaee49
--- /dev/null
+++ b/src/spdk/examples/ioat/perf/.gitignore
@@ -0,0 +1 @@
+ioat_perf
diff --git a/src/spdk/examples/ioat/perf/Makefile b/src/spdk/examples/ioat/perf/Makefile
new file mode 100644
index 000000000..5586b8120
--- /dev/null
+++ b/src/spdk/examples/ioat/perf/Makefile
@@ -0,0 +1,43 @@
+#
+# BSD LICENSE
+#
+# Copyright (c) Intel Corporation.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Intel Corporation nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+APP = ioat_perf
+
+C_SRCS := perf.c
+
+SPDK_LIB_LIST = ioat thread util log
+
+include $(SPDK_ROOT_DIR)/mk/spdk.app.mk
diff --git a/src/spdk/examples/ioat/perf/perf.c b/src/spdk/examples/ioat/perf/perf.c
new file mode 100644
index 000000000..e2d94f268
--- /dev/null
+++ b/src/spdk/examples/ioat/perf/perf.c
@@ -0,0 +1,596 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+
+#include "spdk/ioat.h"
+#include "spdk/env.h"
+#include "spdk/queue.h"
+#include "spdk/string.h"
+
+struct user_config {
+ int xfer_size_bytes;
+ int queue_depth;
+ int time_in_sec;
+ bool verify;
+ char *core_mask;
+ int ioat_chan_num;
+};
+
+struct ioat_device {
+ struct spdk_ioat_chan *ioat;
+ TAILQ_ENTRY(ioat_device) tailq;
+};
+
+static TAILQ_HEAD(, ioat_device) g_devices;
+static struct ioat_device *g_next_device;
+
+static struct user_config g_user_config;
+
+struct ioat_chan_entry {
+ struct spdk_ioat_chan *chan;
+ int ioat_chan_id;
+ uint64_t xfer_completed;
+ uint64_t xfer_failed;
+ uint64_t current_queue_depth;
+ uint64_t waiting_for_flush;
+ uint64_t flush_threshold;
+ bool is_draining;
+ struct spdk_mempool *data_pool;
+ struct spdk_mempool *task_pool;
+ struct ioat_chan_entry *next;
+};
+
+struct worker_thread {
+ struct ioat_chan_entry *ctx;
+ struct worker_thread *next;
+ unsigned core;
+};
+
+struct ioat_task {
+ struct ioat_chan_entry *ioat_chan_entry;
+ void *src;
+ void *dst;
+};
+
+static struct worker_thread *g_workers = NULL;
+static int g_num_workers = 0;
+static int g_ioat_chan_num = 0;
+
+static void submit_single_xfer(struct ioat_chan_entry *ioat_chan_entry, struct ioat_task *ioat_task,
+ void *dst, void *src);
+
+static void
+construct_user_config(struct user_config *self)
+{
+ self->xfer_size_bytes = 4096;
+ self->ioat_chan_num = 1;
+ self->queue_depth = 256;
+ self->time_in_sec = 10;
+ self->verify = false;
+ self->core_mask = "0x1";
+}
+
+static void
+dump_user_config(struct user_config *self)
+{
+ printf("User configuration:\n");
+ printf("Number of channels: %u\n", self->ioat_chan_num);
+ printf("Transfer size: %u bytes\n", self->xfer_size_bytes);
+ printf("Queue depth: %u\n", self->queue_depth);
+ printf("Run time: %u seconds\n", self->time_in_sec);
+ printf("Core mask: %s\n", self->core_mask);
+ printf("Verify: %s\n\n", self->verify ? "Yes" : "No");
+}
+
+static void
+ioat_exit(void)
+{
+ struct ioat_device *dev;
+
+ while (!TAILQ_EMPTY(&g_devices)) {
+ dev = TAILQ_FIRST(&g_devices);
+ TAILQ_REMOVE(&g_devices, dev, tailq);
+ if (dev->ioat) {
+ spdk_ioat_detach(dev->ioat);
+ }
+ spdk_dma_free(dev);
+ }
+}
+
+static void
+ioat_done(void *cb_arg)
+{
+ struct ioat_task *ioat_task = (struct ioat_task *)cb_arg;
+ struct ioat_chan_entry *ioat_chan_entry = ioat_task->ioat_chan_entry;
+
+ if (g_user_config.verify && memcmp(ioat_task->src, ioat_task->dst, g_user_config.xfer_size_bytes)) {
+ ioat_chan_entry->xfer_failed++;
+ } else {
+ ioat_chan_entry->xfer_completed++;
+ }
+
+ ioat_chan_entry->current_queue_depth--;
+
+ if (ioat_chan_entry->is_draining) {
+ spdk_mempool_put(ioat_chan_entry->data_pool, ioat_task->src);
+ spdk_mempool_put(ioat_chan_entry->data_pool, ioat_task->dst);
+ spdk_mempool_put(ioat_chan_entry->task_pool, ioat_task);
+ } else {
+ submit_single_xfer(ioat_chan_entry, ioat_task, ioat_task->dst, ioat_task->src);
+ }
+}
+
+static int
+register_workers(void)
+{
+ uint32_t i;
+ struct worker_thread *worker;
+
+ g_workers = NULL;
+ g_num_workers = 0;
+
+ SPDK_ENV_FOREACH_CORE(i) {
+ worker = calloc(1, sizeof(*worker));
+ if (worker == NULL) {
+ fprintf(stderr, "Unable to allocate worker\n");
+ return 1;
+ }
+
+ worker->core = i;
+ worker->next = g_workers;
+ g_workers = worker;
+ g_num_workers++;
+ }
+
+ return 0;
+}
+
+static void
+unregister_workers(void)
+{
+ struct worker_thread *worker = g_workers;
+ struct ioat_chan_entry *entry, *entry1;
+
+ /* Free ioat_chan_entry and worker thread */
+ while (worker) {
+ struct worker_thread *next_worker = worker->next;
+ entry = worker->ctx;
+ while (entry) {
+ entry1 = entry->next;
+ spdk_mempool_free(entry->data_pool);
+ spdk_mempool_free(entry->task_pool);
+ free(entry);
+ entry = entry1;
+ }
+ free(worker);
+ worker = next_worker;
+ }
+}
+
+static bool
+probe_cb(void *cb_ctx, struct spdk_pci_device *pci_dev)
+{
+ printf(" Found matching device at %04x:%02x:%02x.%x "
+ "vendor:0x%04x device:0x%04x\n",
+ spdk_pci_device_get_domain(pci_dev),
+ spdk_pci_device_get_bus(pci_dev), spdk_pci_device_get_dev(pci_dev),
+ spdk_pci_device_get_func(pci_dev),
+ spdk_pci_device_get_vendor_id(pci_dev), spdk_pci_device_get_device_id(pci_dev));
+
+ return true;
+}
+
+static void
+attach_cb(void *cb_ctx, struct spdk_pci_device *pci_dev, struct spdk_ioat_chan *ioat)
+{
+ struct ioat_device *dev;
+
+ if (g_ioat_chan_num >= g_user_config.ioat_chan_num) {
+ return;
+ }
+
+ dev = spdk_dma_zmalloc(sizeof(*dev), 0, NULL);
+ if (dev == NULL) {
+ printf("Failed to allocate device struct\n");
+ return;
+ }
+
+ dev->ioat = ioat;
+ g_ioat_chan_num++;
+ TAILQ_INSERT_TAIL(&g_devices, dev, tailq);
+}
+
+static int
+ioat_init(void)
+{
+ TAILQ_INIT(&g_devices);
+
+ if (spdk_ioat_probe(NULL, probe_cb, attach_cb) != 0) {
+ fprintf(stderr, "ioat_probe() failed\n");
+ return 1;
+ }
+
+ return 0;
+}
+
+static void
+usage(char *program_name)
+{
+ printf("%s options\n", program_name);
+ printf("\t[-h help message]\n");
+ printf("\t[-c core mask for distributing I/O submission/completion work]\n");
+ printf("\t[-q queue depth]\n");
+ printf("\t[-n number of channels]\n");
+ printf("\t[-o transfer size in bytes]\n");
+ printf("\t[-t time in seconds]\n");
+ printf("\t[-v verify copy result if this switch is on]\n");
+}
+
+static int
+parse_args(int argc, char **argv)
+{
+ int op;
+
+ construct_user_config(&g_user_config);
+ while ((op = getopt(argc, argv, "c:hn:o:q:t:v")) != -1) {
+ switch (op) {
+ case 'o':
+ g_user_config.xfer_size_bytes = spdk_strtol(optarg, 10);
+ break;
+ case 'n':
+ g_user_config.ioat_chan_num = spdk_strtol(optarg, 10);
+ break;
+ case 'q':
+ g_user_config.queue_depth = spdk_strtol(optarg, 10);
+ break;
+ case 't':
+ g_user_config.time_in_sec = spdk_strtol(optarg, 10);
+ break;
+ case 'c':
+ g_user_config.core_mask = optarg;
+ break;
+ case 'v':
+ g_user_config.verify = true;
+ break;
+ case 'h':
+ usage(argv[0]);
+ exit(0);
+ default:
+ usage(argv[0]);
+ return 1;
+ }
+ }
+ if (g_user_config.xfer_size_bytes <= 0 || g_user_config.queue_depth <= 0 ||
+ g_user_config.time_in_sec <= 0 || !g_user_config.core_mask ||
+ g_user_config.ioat_chan_num <= 0) {
+ usage(argv[0]);
+ return 1;
+ }
+
+ return 0;
+}
+
+static void
+drain_io(struct ioat_chan_entry *ioat_chan_entry)
+{
+ spdk_ioat_flush(ioat_chan_entry->chan);
+ while (ioat_chan_entry->current_queue_depth > 0) {
+ spdk_ioat_process_events(ioat_chan_entry->chan);
+ }
+}
+
+static void
+submit_single_xfer(struct ioat_chan_entry *ioat_chan_entry, struct ioat_task *ioat_task, void *dst,
+ void *src)
+{
+ ioat_task->ioat_chan_entry = ioat_chan_entry;
+ ioat_task->src = src;
+ ioat_task->dst = dst;
+
+ spdk_ioat_build_copy(ioat_chan_entry->chan, ioat_task, ioat_done, dst, src,
+ g_user_config.xfer_size_bytes);
+ ioat_chan_entry->waiting_for_flush++;
+ if (ioat_chan_entry->waiting_for_flush >= ioat_chan_entry->flush_threshold) {
+ spdk_ioat_flush(ioat_chan_entry->chan);
+ ioat_chan_entry->waiting_for_flush = 0;
+ }
+
+ ioat_chan_entry->current_queue_depth++;
+}
+
+static int
+submit_xfers(struct ioat_chan_entry *ioat_chan_entry, uint64_t queue_depth)
+{
+ while (queue_depth-- > 0) {
+ void *src = NULL, *dst = NULL;
+ struct ioat_task *ioat_task = NULL;
+
+ src = spdk_mempool_get(ioat_chan_entry->data_pool);
+ dst = spdk_mempool_get(ioat_chan_entry->data_pool);
+ ioat_task = spdk_mempool_get(ioat_chan_entry->task_pool);
+ if (!ioat_task) {
+ fprintf(stderr, "Unable to get ioat_task\n");
+ return 1;
+ }
+
+ submit_single_xfer(ioat_chan_entry, ioat_task, dst, src);
+ }
+ return 0;
+}
+
+static int
+work_fn(void *arg)
+{
+ uint64_t tsc_end;
+ struct worker_thread *worker = (struct worker_thread *)arg;
+ struct ioat_chan_entry *t = NULL;
+
+ printf("Starting thread on core %u\n", worker->core);
+
+ tsc_end = spdk_get_ticks() + g_user_config.time_in_sec * spdk_get_ticks_hz();
+
+ t = worker->ctx;
+ while (t != NULL) {
+ /* begin to submit transfers */
+ t->waiting_for_flush = 0;
+ t->flush_threshold = g_user_config.queue_depth / 2;
+ if (submit_xfers(t, g_user_config.queue_depth) != 0) {
+ return 1;
+ }
+ t = t->next;
+ }
+
+ while (1) {
+ t = worker->ctx;
+ while (t != NULL) {
+ spdk_ioat_process_events(t->chan);
+ t = t->next;
+ }
+
+ if (spdk_get_ticks() > tsc_end) {
+ break;
+ }
+ }
+
+ t = worker->ctx;
+ while (t != NULL) {
+ /* begin to drain io */
+ t->is_draining = true;
+ drain_io(t);
+ t = t->next;
+ }
+
+ return 0;
+}
+
+static int
+init(void)
+{
+ struct spdk_env_opts opts;
+
+ spdk_env_opts_init(&opts);
+ opts.name = "ioat_perf";
+ opts.core_mask = g_user_config.core_mask;
+ if (spdk_env_init(&opts) < 0) {
+ return 1;
+ }
+
+ return 0;
+}
+
+static int
+dump_result(void)
+{
+ uint64_t total_completed = 0;
+ uint64_t total_failed = 0;
+ uint64_t total_xfer_per_sec, total_bw_in_MiBps;
+ struct worker_thread *worker = g_workers;
+
+ printf("Channel_ID Core Transfers Bandwidth Failed\n");
+ printf("-----------------------------------------------------------\n");
+ while (worker != NULL) {
+ struct ioat_chan_entry *t = worker->ctx;
+ while (t) {
+ uint64_t xfer_per_sec = t->xfer_completed / g_user_config.time_in_sec;
+ uint64_t bw_in_MiBps = (t->xfer_completed * g_user_config.xfer_size_bytes) /
+ (g_user_config.time_in_sec * 1024 * 1024);
+
+ total_completed += t->xfer_completed;
+ total_failed += t->xfer_failed;
+
+ if (xfer_per_sec) {
+ printf("%10d%10d%12" PRIu64 "/s%8" PRIu64 " MiB/s%11" PRIu64 "\n",
+ t->ioat_chan_id, worker->core, xfer_per_sec,
+ bw_in_MiBps, t->xfer_failed);
+ }
+ t = t->next;
+ }
+ worker = worker->next;
+ }
+
+ total_xfer_per_sec = total_completed / g_user_config.time_in_sec;
+ total_bw_in_MiBps = (total_completed * g_user_config.xfer_size_bytes) /
+ (g_user_config.time_in_sec * 1024 * 1024);
+
+ printf("===========================================================\n");
+ printf("Total:%26" PRIu64 "/s%8" PRIu64 " MiB/s%11" PRIu64 "\n",
+ total_xfer_per_sec, total_bw_in_MiBps, total_failed);
+
+ return total_failed ? 1 : 0;
+}
+
+static struct spdk_ioat_chan *
+get_next_chan(void)
+{
+ struct spdk_ioat_chan *chan;
+
+ if (g_next_device == NULL) {
+ return NULL;
+ }
+
+ chan = g_next_device->ioat;
+
+ g_next_device = TAILQ_NEXT(g_next_device, tailq);
+
+ return chan;
+}
+
+static int
+associate_workers_with_chan(void)
+{
+ struct spdk_ioat_chan *chan = get_next_chan();
+ struct worker_thread *worker = g_workers;
+ struct ioat_chan_entry *t;
+ char buf_pool_name[30], task_pool_name[30];
+ int i = 0;
+
+ while (chan != NULL) {
+ t = calloc(1, sizeof(struct ioat_chan_entry));
+ if (!t) {
+ return 1;
+ }
+
+ t->ioat_chan_id = i;
+ snprintf(buf_pool_name, sizeof(buf_pool_name), "buf_pool_%d", i);
+ snprintf(task_pool_name, sizeof(task_pool_name), "task_pool_%d", i);
+ t->data_pool = spdk_mempool_create(buf_pool_name,
+ g_user_config.queue_depth * 2, /* src + dst */
+ g_user_config.xfer_size_bytes,
+ SPDK_MEMPOOL_DEFAULT_CACHE_SIZE,
+ SPDK_ENV_SOCKET_ID_ANY);
+ t->task_pool = spdk_mempool_create(task_pool_name,
+ g_user_config.queue_depth,
+ sizeof(struct ioat_task),
+ SPDK_MEMPOOL_DEFAULT_CACHE_SIZE,
+ SPDK_ENV_SOCKET_ID_ANY);
+ if (!t->data_pool || !t->task_pool) {
+ fprintf(stderr, "Could not allocate buffer pool.\n");
+ spdk_mempool_free(t->data_pool);
+ spdk_mempool_free(t->task_pool);
+ free(t);
+ return 1;
+ }
+ printf("Associating ioat_channel %d with core %d\n", i, worker->core);
+ t->chan = chan;
+ t->next = worker->ctx;
+ worker->ctx = t;
+
+ worker = worker->next;
+ if (worker == NULL) {
+ worker = g_workers;
+ }
+
+ chan = get_next_chan();
+ i++;
+ }
+
+ return 0;
+}
+
+int
+main(int argc, char **argv)
+{
+ int rc;
+ struct worker_thread *worker, *master_worker;
+ unsigned master_core;
+
+ if (parse_args(argc, argv) != 0) {
+ return 1;
+ }
+
+ if (init() != 0) {
+ return 1;
+ }
+
+ if (register_workers() != 0) {
+ rc = 1;
+ goto cleanup;
+ }
+
+ if (ioat_init() != 0) {
+ rc = 1;
+ goto cleanup;
+ }
+
+ if (g_ioat_chan_num == 0) {
+ printf("No channels found\n");
+ rc = 1;
+ goto cleanup;
+ }
+
+ if (g_user_config.ioat_chan_num > g_ioat_chan_num) {
+ printf("%d channels are requested, but only %d are found,"
+ "so only test %d channels\n", g_user_config.ioat_chan_num,
+ g_ioat_chan_num, g_ioat_chan_num);
+ g_user_config.ioat_chan_num = g_ioat_chan_num;
+ }
+
+ g_next_device = TAILQ_FIRST(&g_devices);
+ dump_user_config(&g_user_config);
+
+ if (associate_workers_with_chan() != 0) {
+ rc = 1;
+ goto cleanup;
+ }
+
+ /* Launch all of the slave workers */
+ master_core = spdk_env_get_current_core();
+ master_worker = NULL;
+ worker = g_workers;
+ while (worker != NULL) {
+ if (worker->core != master_core) {
+ spdk_env_thread_launch_pinned(worker->core, work_fn, worker);
+ } else {
+ assert(master_worker == NULL);
+ master_worker = worker;
+ }
+ worker = worker->next;
+ }
+
+ assert(master_worker != NULL);
+ rc = work_fn(master_worker);
+ if (rc != 0) {
+ goto cleanup;
+ }
+
+ spdk_env_thread_wait_all();
+
+ rc = dump_result();
+
+cleanup:
+ unregister_workers();
+ ioat_exit();
+
+ return rc;
+}
diff --git a/src/spdk/examples/ioat/verify/.gitignore b/src/spdk/examples/ioat/verify/.gitignore
new file mode 100644
index 000000000..0b5987362
--- /dev/null
+++ b/src/spdk/examples/ioat/verify/.gitignore
@@ -0,0 +1 @@
+verify
diff --git a/src/spdk/examples/ioat/verify/Makefile b/src/spdk/examples/ioat/verify/Makefile
new file mode 100644
index 000000000..50cfc6665
--- /dev/null
+++ b/src/spdk/examples/ioat/verify/Makefile
@@ -0,0 +1,43 @@
+#
+# BSD LICENSE
+#
+# Copyright (c) Intel Corporation.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Intel Corporation nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+APP = verify
+
+C_SRCS := verify.c
+
+SPDK_LIB_LIST = ioat thread util log
+
+include $(SPDK_ROOT_DIR)/mk/spdk.app.mk
diff --git a/src/spdk/examples/ioat/verify/verify.c b/src/spdk/examples/ioat/verify/verify.c
new file mode 100644
index 000000000..0df41f69b
--- /dev/null
+++ b/src/spdk/examples/ioat/verify/verify.c
@@ -0,0 +1,521 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+
+#include "spdk/ioat.h"
+#include "spdk/env.h"
+#include "spdk/queue.h"
+#include "spdk/string.h"
+#include "spdk/util.h"
+
+#define SRC_BUFFER_SIZE (512*1024)
+
+enum ioat_task_type {
+ IOAT_COPY_TYPE,
+ IOAT_FILL_TYPE,
+};
+
+struct user_config {
+ int queue_depth;
+ int time_in_sec;
+ char *core_mask;
+};
+
+struct ioat_device {
+ struct spdk_ioat_chan *ioat;
+ TAILQ_ENTRY(ioat_device) tailq;
+};
+
+static TAILQ_HEAD(, ioat_device) g_devices;
+static struct ioat_device *g_next_device;
+
+static struct user_config g_user_config;
+
+struct thread_entry {
+ struct spdk_ioat_chan *chan;
+ uint64_t xfer_completed;
+ uint64_t xfer_failed;
+ uint64_t fill_completed;
+ uint64_t fill_failed;
+ uint64_t current_queue_depth;
+ unsigned lcore_id;
+ bool is_draining;
+ bool init_failed;
+ struct spdk_mempool *data_pool;
+ struct spdk_mempool *task_pool;
+};
+
+struct ioat_task {
+ enum ioat_task_type type;
+ struct thread_entry *thread_entry;
+ void *buffer;
+ int len;
+ uint64_t fill_pattern;
+ void *src;
+ void *dst;
+};
+
+static __thread unsigned int seed = 0;
+
+static unsigned char *g_src;
+
+static void submit_single_xfer(struct ioat_task *ioat_task);
+
+static void
+construct_user_config(struct user_config *self)
+{
+ self->queue_depth = 32;
+ self->time_in_sec = 10;
+ self->core_mask = "0x1";
+}
+
+static void
+dump_user_config(struct user_config *self)
+{
+ printf("User configuration:\n");
+ printf("Run time: %u seconds\n", self->time_in_sec);
+ printf("Core mask: %s\n", self->core_mask);
+ printf("Queue depth: %u\n", self->queue_depth);
+}
+
+static void
+ioat_exit(void)
+{
+ struct ioat_device *dev;
+
+ while (!TAILQ_EMPTY(&g_devices)) {
+ dev = TAILQ_FIRST(&g_devices);
+ TAILQ_REMOVE(&g_devices, dev, tailq);
+ if (dev->ioat) {
+ spdk_ioat_detach(dev->ioat);
+ }
+ free(dev);
+ }
+}
+static void prepare_ioat_task(struct thread_entry *thread_entry, struct ioat_task *ioat_task)
+{
+ int len;
+ uintptr_t src_offset;
+ uintptr_t dst_offset;
+ uint64_t fill_pattern;
+
+ if (ioat_task->type == IOAT_FILL_TYPE) {
+ fill_pattern = rand_r(&seed);
+ fill_pattern = fill_pattern << 32 | rand_r(&seed);
+
+ /* Ensure that the length of memset block is 8 Bytes aligned.
+ * In case the buffer crosses hugepage boundary and must be split,
+ * we also need to ensure 8 byte address alignment. We do it
+ * unconditionally to keep things simple.
+ */
+ len = 8 + ((rand_r(&seed) % (SRC_BUFFER_SIZE - 16)) & ~0x7);
+ dst_offset = 8 + rand_r(&seed) % (SRC_BUFFER_SIZE - 8 - len);
+ ioat_task->fill_pattern = fill_pattern;
+ ioat_task->dst = (void *)(((uintptr_t)ioat_task->buffer + dst_offset) & ~0x7);
+ } else {
+ src_offset = rand_r(&seed) % SRC_BUFFER_SIZE;
+ len = rand_r(&seed) % (SRC_BUFFER_SIZE - src_offset);
+ dst_offset = rand_r(&seed) % (SRC_BUFFER_SIZE - len);
+
+ memset(ioat_task->buffer, 0, SRC_BUFFER_SIZE);
+ ioat_task->src = (void *)((uintptr_t)g_src + src_offset);
+ ioat_task->dst = (void *)((uintptr_t)ioat_task->buffer + dst_offset);
+ }
+ ioat_task->len = len;
+ ioat_task->thread_entry = thread_entry;
+}
+
+static void
+ioat_done(void *cb_arg)
+{
+ char *value;
+ int i, failed = 0;
+ struct ioat_task *ioat_task = (struct ioat_task *)cb_arg;
+ struct thread_entry *thread_entry = ioat_task->thread_entry;
+
+ if (ioat_task->type == IOAT_FILL_TYPE) {
+ value = ioat_task->dst;
+ for (i = 0; i < ioat_task->len / 8; i++) {
+ if (memcmp(value, &ioat_task->fill_pattern, 8) != 0) {
+ thread_entry->fill_failed++;
+ failed = 1;
+ break;
+ }
+ value += 8;
+ }
+ if (!failed) {
+ thread_entry->fill_completed++;
+ }
+ } else {
+ if (memcmp(ioat_task->src, ioat_task->dst, ioat_task->len)) {
+ thread_entry->xfer_failed++;
+ } else {
+ thread_entry->xfer_completed++;
+ }
+ }
+
+ thread_entry->current_queue_depth--;
+ if (thread_entry->is_draining) {
+ spdk_mempool_put(thread_entry->data_pool, ioat_task->buffer);
+ spdk_mempool_put(thread_entry->task_pool, ioat_task);
+ } else {
+ prepare_ioat_task(thread_entry, ioat_task);
+ submit_single_xfer(ioat_task);
+ }
+}
+
+static bool
+probe_cb(void *cb_ctx, struct spdk_pci_device *pci_dev)
+{
+ printf(" Found matching device at %04x:%02x:%02x.%x "
+ "vendor:0x%04x device:0x%04x\n",
+ spdk_pci_device_get_domain(pci_dev),
+ spdk_pci_device_get_bus(pci_dev), spdk_pci_device_get_dev(pci_dev),
+ spdk_pci_device_get_func(pci_dev),
+ spdk_pci_device_get_vendor_id(pci_dev), spdk_pci_device_get_device_id(pci_dev));
+
+ return true;
+}
+
+static void
+attach_cb(void *cb_ctx, struct spdk_pci_device *pci_dev, struct spdk_ioat_chan *ioat)
+{
+ struct ioat_device *dev;
+
+ dev = malloc(sizeof(*dev));
+ if (dev == NULL) {
+ printf("Failed to allocate device struct\n");
+ return;
+ }
+ memset(dev, 0, sizeof(*dev));
+
+ dev->ioat = ioat;
+ TAILQ_INSERT_TAIL(&g_devices, dev, tailq);
+}
+
+static int
+ioat_init(void)
+{
+ TAILQ_INIT(&g_devices);
+
+ if (spdk_ioat_probe(NULL, probe_cb, attach_cb) != 0) {
+ fprintf(stderr, "ioat_probe() failed\n");
+ return 1;
+ }
+
+ return 0;
+}
+
+static void
+usage(char *program_name)
+{
+ printf("%s options\n", program_name);
+ printf("\t[-h help message]\n");
+ printf("\t[-c core mask for distributing I/O submission/completion work]\n");
+ printf("\t[-t time in seconds]\n");
+ printf("\t[-q queue depth]\n");
+}
+
+static int
+parse_args(int argc, char **argv)
+{
+ int op;
+
+ construct_user_config(&g_user_config);
+ while ((op = getopt(argc, argv, "c:ht:q:")) != -1) {
+ switch (op) {
+ case 't':
+ g_user_config.time_in_sec = spdk_strtol(optarg, 10);
+ break;
+ case 'c':
+ g_user_config.core_mask = optarg;
+ break;
+ case 'q':
+ g_user_config.queue_depth = spdk_strtol(optarg, 10);
+ break;
+ case 'h':
+ usage(argv[0]);
+ exit(0);
+ default:
+ usage(argv[0]);
+ return 1;
+ }
+ }
+ if (g_user_config.time_in_sec <= 0 || !g_user_config.core_mask ||
+ g_user_config.queue_depth <= 0) {
+ usage(argv[0]);
+ return 1;
+ }
+
+ return 0;
+}
+
+static void
+drain_xfers(struct thread_entry *thread_entry)
+{
+ while (thread_entry->current_queue_depth > 0) {
+ spdk_ioat_process_events(thread_entry->chan);
+ }
+}
+
+static void
+submit_single_xfer(struct ioat_task *ioat_task)
+{
+ if (ioat_task->type == IOAT_FILL_TYPE)
+ spdk_ioat_submit_fill(ioat_task->thread_entry->chan, ioat_task, ioat_done,
+ ioat_task->dst, ioat_task->fill_pattern, ioat_task->len);
+ else
+ spdk_ioat_submit_copy(ioat_task->thread_entry->chan, ioat_task, ioat_done,
+ ioat_task->dst, ioat_task->src, ioat_task->len);
+ ioat_task->thread_entry->current_queue_depth++;
+}
+
+static void
+submit_xfers(struct thread_entry *thread_entry, uint64_t queue_depth)
+{
+ while (queue_depth-- > 0) {
+ struct ioat_task *ioat_task = NULL;
+ ioat_task = spdk_mempool_get(thread_entry->task_pool);
+ ioat_task->buffer = spdk_mempool_get(thread_entry->data_pool);
+
+ ioat_task->type = IOAT_COPY_TYPE;
+ if (spdk_ioat_get_dma_capabilities(thread_entry->chan) & SPDK_IOAT_ENGINE_FILL_SUPPORTED) {
+ if (queue_depth % 2) {
+ ioat_task->type = IOAT_FILL_TYPE;
+ }
+ }
+ prepare_ioat_task(thread_entry, ioat_task);
+ submit_single_xfer(ioat_task);
+ }
+}
+
+static int
+work_fn(void *arg)
+{
+ uint64_t tsc_end;
+ char buf_pool_name[20], task_pool_name[20];
+ struct thread_entry *t = (struct thread_entry *)arg;
+
+ if (!t->chan) {
+ return 1;
+ }
+
+ t->lcore_id = spdk_env_get_current_core();
+
+ snprintf(buf_pool_name, sizeof(buf_pool_name), "buf_pool_%u", t->lcore_id);
+ snprintf(task_pool_name, sizeof(task_pool_name), "task_pool_%u", t->lcore_id);
+ t->data_pool = spdk_mempool_create(buf_pool_name, g_user_config.queue_depth, SRC_BUFFER_SIZE,
+ SPDK_MEMPOOL_DEFAULT_CACHE_SIZE,
+ SPDK_ENV_SOCKET_ID_ANY);
+ t->task_pool = spdk_mempool_create(task_pool_name, g_user_config.queue_depth,
+ sizeof(struct ioat_task),
+ SPDK_MEMPOOL_DEFAULT_CACHE_SIZE,
+ SPDK_ENV_SOCKET_ID_ANY);
+ if (!t->data_pool || !t->task_pool) {
+ fprintf(stderr, "Could not allocate buffer pool.\n");
+ t->init_failed = true;
+ return 1;
+ }
+
+ tsc_end = spdk_get_ticks() + g_user_config.time_in_sec * spdk_get_ticks_hz();
+
+ submit_xfers(t, g_user_config.queue_depth);
+ while (spdk_get_ticks() < tsc_end) {
+ spdk_ioat_process_events(t->chan);
+ }
+
+ t->is_draining = true;
+ drain_xfers(t);
+
+ return 0;
+}
+
+static int
+init_src_buffer(void)
+{
+ int i;
+
+ g_src = spdk_dma_zmalloc(SRC_BUFFER_SIZE, 512, NULL);
+ if (g_src == NULL) {
+ fprintf(stderr, "Allocate src buffer failed\n");
+ return 1;
+ }
+
+ for (i = 0; i < SRC_BUFFER_SIZE / 4; i++) {
+ memset((g_src + (4 * i)), i, 4);
+ }
+
+ return 0;
+}
+
+static int
+init(void)
+{
+ struct spdk_env_opts opts;
+
+ spdk_env_opts_init(&opts);
+ opts.name = "verify";
+ opts.core_mask = g_user_config.core_mask;
+ if (spdk_env_init(&opts) < 0) {
+ fprintf(stderr, "Unable to initialize SPDK env\n");
+ return 1;
+ }
+
+ if (init_src_buffer() != 0) {
+ fprintf(stderr, "Could not init src buffer\n");
+ return 1;
+ }
+ if (ioat_init() != 0) {
+ fprintf(stderr, "Could not init ioat\n");
+ return 1;
+ }
+
+ return 0;
+}
+
+static int
+dump_result(struct thread_entry *threads, uint32_t num_threads)
+{
+ uint32_t i;
+ uint64_t total_completed = 0;
+ uint64_t total_failed = 0;
+
+ for (i = 0; i < num_threads; i++) {
+ struct thread_entry *t = &threads[i];
+
+ if (!t->chan) {
+ continue;
+ }
+
+ if (t->init_failed) {
+ total_failed++;
+ continue;
+ }
+
+ total_completed += t->xfer_completed;
+ total_completed += t->fill_completed;
+ total_failed += t->xfer_failed;
+ total_failed += t->fill_failed;
+ if (total_completed || total_failed)
+ printf("lcore = %d, copy success = %ld, copy failed = %ld, fill success = %ld, fill failed = %ld\n",
+ t->lcore_id, t->xfer_completed, t->xfer_failed, t->fill_completed, t->fill_failed);
+ }
+ return total_failed ? 1 : 0;
+}
+
+static struct spdk_ioat_chan *
+get_next_chan(void)
+{
+ struct spdk_ioat_chan *chan;
+
+ if (g_next_device == NULL) {
+ fprintf(stderr, "Not enough ioat channels found. Check that ioat channels are bound\n");
+ fprintf(stderr, "to uio_pci_generic or vfio-pci. scripts/setup.sh can help with this.\n");
+ return NULL;
+ }
+
+ chan = g_next_device->ioat;
+
+ g_next_device = TAILQ_NEXT(g_next_device, tailq);
+
+ return chan;
+}
+
+static uint32_t
+get_max_core(void)
+{
+ uint32_t i;
+ uint32_t max_core = 0;
+
+ SPDK_ENV_FOREACH_CORE(i) {
+ if (i > max_core) {
+ max_core = i;
+ }
+ }
+
+ return max_core;
+}
+
+int
+main(int argc, char **argv)
+{
+ uint32_t i, current_core;
+ struct thread_entry *threads;
+ uint32_t num_threads;
+ int rc;
+
+ if (parse_args(argc, argv) != 0) {
+ return 1;
+ }
+
+ if (init() != 0) {
+ return 1;
+ }
+
+ dump_user_config(&g_user_config);
+
+ g_next_device = TAILQ_FIRST(&g_devices);
+
+ num_threads = get_max_core() + 1;
+ threads = calloc(num_threads, sizeof(*threads));
+ if (!threads) {
+ fprintf(stderr, "Thread memory allocation failed\n");
+ rc = 1;
+ goto cleanup;
+ }
+
+ current_core = spdk_env_get_current_core();
+ SPDK_ENV_FOREACH_CORE(i) {
+ if (i != current_core) {
+ threads[i].chan = get_next_chan();
+ spdk_env_thread_launch_pinned(i, work_fn, &threads[i]);
+ }
+ }
+
+ threads[current_core].chan = get_next_chan();
+ if (work_fn(&threads[current_core]) != 0) {
+ rc = 1;
+ goto cleanup;
+ }
+
+ spdk_env_thread_wait_all();
+ rc = dump_result(threads, num_threads);
+
+cleanup:
+ spdk_dma_free(g_src);
+ ioat_exit();
+ free(threads);
+
+ return rc;
+}
diff --git a/src/spdk/examples/nvme/Makefile b/src/spdk/examples/nvme/Makefile
new file mode 100644
index 000000000..14eeb9be7
--- /dev/null
+++ b/src/spdk/examples/nvme/Makefile
@@ -0,0 +1,47 @@
+#
+# BSD LICENSE
+#
+# Copyright (c) Intel Corporation.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Intel Corporation nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+DIRS-y += hello_world identify perf reconnect nvme_manage arbitration \
+ hotplug cmb_copy abort
+
+DIRS-$(CONFIG_FIO_PLUGIN) += fio_plugin
+
+.PHONY: all clean $(DIRS-y)
+
+all: $(DIRS-y)
+clean: $(DIRS-y)
+
+include $(SPDK_ROOT_DIR)/mk/spdk.subdirs.mk
diff --git a/src/spdk/examples/nvme/abort/.gitignore b/src/spdk/examples/nvme/abort/.gitignore
new file mode 100644
index 000000000..f7d13fd04
--- /dev/null
+++ b/src/spdk/examples/nvme/abort/.gitignore
@@ -0,0 +1 @@
+abort
diff --git a/src/spdk/examples/nvme/abort/Makefile b/src/spdk/examples/nvme/abort/Makefile
new file mode 100644
index 000000000..5073a842d
--- /dev/null
+++ b/src/spdk/examples/nvme/abort/Makefile
@@ -0,0 +1,38 @@
+#
+# BSD LICENSE
+#
+# Copyright (c) Intel Corporation.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Intel Corporation nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
+
+APP = abort
+
+include $(SPDK_ROOT_DIR)/mk/nvme.libtest.mk
diff --git a/src/spdk/examples/nvme/abort/abort.c b/src/spdk/examples/nvme/abort/abort.c
new file mode 100644
index 000000000..728790513
--- /dev/null
+++ b/src/spdk/examples/nvme/abort/abort.c
@@ -0,0 +1,1144 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+
+#include "spdk/env.h"
+#include "spdk/log.h"
+#include "spdk/nvme.h"
+#include "spdk/queue.h"
+#include "spdk/string.h"
+#include "spdk/util.h"
+#include "spdk/likely.h"
+
+struct ctrlr_entry {
+ struct spdk_nvme_ctrlr *ctrlr;
+ enum spdk_nvme_transport_type trtype;
+
+ struct ctrlr_entry *next;
+ char name[1024];
+};
+
+struct ns_entry {
+ struct spdk_nvme_ctrlr *ctrlr;
+ struct spdk_nvme_ns *ns;
+
+ struct ns_entry *next;
+ uint32_t io_size_blocks;
+ uint32_t num_io_requests;
+ uint64_t size_in_ios;
+ uint32_t block_size;
+ char name[1024];
+};
+
+struct ctrlr_worker_ctx {
+ pthread_mutex_t mutex;
+ struct ctrlr_entry *entry;
+ uint64_t abort_submitted;
+ uint64_t abort_submit_failed;
+ uint64_t successful_abort;
+ uint64_t unsuccessful_abort;
+ uint64_t abort_failed;
+ uint64_t current_queue_depth;
+ struct spdk_nvme_ctrlr *ctrlr;
+ struct ctrlr_worker_ctx *next;
+};
+
+struct ns_worker_ctx {
+ struct ns_entry *entry;
+ uint64_t io_submitted;
+ uint64_t io_completed;
+ uint64_t io_aborted;
+ uint64_t io_failed;
+ uint64_t current_queue_depth;
+ uint64_t offset_in_ios;
+ bool is_draining;
+ struct spdk_nvme_qpair *qpair;
+ struct ctrlr_worker_ctx *ctrlr_ctx;
+ struct ns_worker_ctx *next;
+};
+
+struct perf_task {
+ struct ns_worker_ctx *ns_ctx;
+ void *buf;
+};
+
+struct worker_thread {
+ struct ns_worker_ctx *ns_ctx;
+ struct ctrlr_worker_ctx *ctrlr_ctx;
+ struct worker_thread *next;
+ unsigned lcore;
+};
+
+static const char *g_workload_type = "read";
+static struct ctrlr_entry *g_controllers;
+static struct ns_entry *g_namespaces;
+static int g_num_namespaces;
+static struct worker_thread *g_workers;
+static int g_num_workers;
+static uint32_t g_master_core;
+
+static int g_abort_interval = 1;
+
+static uint64_t g_tsc_rate;
+
+static uint32_t g_io_size_bytes = 131072;
+static uint32_t g_max_io_size_blocks;
+static int g_rw_percentage = -1;
+static int g_is_random;
+static int g_queue_depth = 128;
+static int g_time_in_sec = 3;
+static int g_dpdk_mem;
+static int g_shm_id = -1;
+static bool g_no_pci;
+static bool g_warn;
+static bool g_mix_specified;
+
+static const char *g_core_mask;
+
+struct trid_entry {
+ struct spdk_nvme_transport_id trid;
+ uint16_t nsid;
+ TAILQ_ENTRY(trid_entry) tailq;
+};
+
+static TAILQ_HEAD(, trid_entry) g_trid_list = TAILQ_HEAD_INITIALIZER(g_trid_list);
+
+static void io_complete(void *ctx, const struct spdk_nvme_cpl *cpl);
+
+static int
+build_nvme_name(char *name, size_t length, struct spdk_nvme_ctrlr *ctrlr)
+{
+ const struct spdk_nvme_transport_id *trid;
+ int res = 0;
+
+ trid = spdk_nvme_ctrlr_get_transport_id(ctrlr);
+
+ switch (trid->trtype) {
+ case SPDK_NVME_TRANSPORT_PCIE:
+ res = snprintf(name, length, "PCIE (%s)", trid->traddr);
+ break;
+ case SPDK_NVME_TRANSPORT_RDMA:
+ res = snprintf(name, length, "RDMA (addr:%s subnqn:%s)", trid->traddr, trid->subnqn);
+ break;
+ case SPDK_NVME_TRANSPORT_TCP:
+ res = snprintf(name, length, "TCP (addr:%s subnqn:%s)", trid->traddr, trid->subnqn);
+ break;
+
+ default:
+ fprintf(stderr, "Unknown transport type %d\n", trid->trtype);
+ break;
+ }
+ return res;
+}
+
+static void
+build_nvme_ns_name(char *name, size_t length, struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid)
+{
+ int res = 0;
+
+ res = build_nvme_name(name, length, ctrlr);
+ if (res > 0) {
+ snprintf(name + res, length - res, " NSID %u", nsid);
+ }
+
+}
+
+static void
+register_ns(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns *ns)
+{
+ struct ns_entry *entry;
+ const struct spdk_nvme_ctrlr_data *cdata;
+ uint32_t max_xfer_size, entries, sector_size;
+ uint64_t ns_size;
+ struct spdk_nvme_io_qpair_opts opts;
+
+ cdata = spdk_nvme_ctrlr_get_data(ctrlr);
+
+ if (!spdk_nvme_ns_is_active(ns)) {
+ printf("Controller %-20.20s (%-20.20s): Skipping inactive NS %u\n",
+ cdata->mn, cdata->sn,
+ spdk_nvme_ns_get_id(ns));
+ g_warn = true;
+ return;
+ }
+
+ ns_size = spdk_nvme_ns_get_size(ns);
+ sector_size = spdk_nvme_ns_get_sector_size(ns);
+
+ if (ns_size < g_io_size_bytes || sector_size > g_io_size_bytes) {
+ printf("WARNING: controller %-20.20s (%-20.20s) ns %u has invalid "
+ "ns size %" PRIu64 " / block size %u for I/O size %u\n",
+ cdata->mn, cdata->sn, spdk_nvme_ns_get_id(ns),
+ ns_size, spdk_nvme_ns_get_sector_size(ns), g_io_size_bytes);
+ g_warn = true;
+ return;
+ }
+
+ max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns);
+ spdk_nvme_ctrlr_get_default_io_qpair_opts(ctrlr, &opts, sizeof(opts));
+ /* NVMe driver may add additional entries based on
+ * stripe size and maximum transfer size, we assume
+ * 1 more entry be used for stripe.
+ */
+ entries = (g_io_size_bytes - 1) / max_xfer_size + 2;
+ if ((g_queue_depth * entries) > opts.io_queue_size) {
+ printf("controller IO queue size %u less than required\n",
+ opts.io_queue_size);
+ printf("Consider using lower queue depth or small IO size because "
+ "IO requests may be queued at the NVMe driver.\n");
+ }
+ /* For requests which have children requests, parent request itself
+ * will also occupy 1 entry.
+ */
+ entries += 1;
+
+ entry = calloc(1, sizeof(struct ns_entry));
+ if (entry == NULL) {
+ perror("ns_entry malloc");
+ exit(1);
+ }
+
+ entry->ctrlr = ctrlr;
+ entry->ns = ns;
+ entry->num_io_requests = g_queue_depth * entries;
+
+ entry->size_in_ios = ns_size / g_io_size_bytes;
+ entry->io_size_blocks = g_io_size_bytes / sector_size;
+
+ entry->block_size = spdk_nvme_ns_get_sector_size(ns);
+
+ if (g_max_io_size_blocks < entry->io_size_blocks) {
+ g_max_io_size_blocks = entry->io_size_blocks;
+ }
+
+ build_nvme_ns_name(entry->name, sizeof(entry->name), ctrlr, spdk_nvme_ns_get_id(ns));
+
+ g_num_namespaces++;
+ entry->next = g_namespaces;
+ g_namespaces = entry;
+}
+
+static void
+unregister_namespaces(void)
+{
+ struct ns_entry *entry = g_namespaces;
+
+ while (entry) {
+ struct ns_entry *next = entry->next;
+ free(entry);
+ entry = next;
+ }
+}
+
+static void
+register_ctrlr(struct spdk_nvme_ctrlr *ctrlr, struct trid_entry *trid_entry)
+{
+ struct spdk_nvme_ns *ns;
+ struct ctrlr_entry *entry = malloc(sizeof(struct ctrlr_entry));
+ uint32_t nsid;
+
+ if (entry == NULL) {
+ perror("ctrlr_entry malloc");
+ exit(1);
+ }
+
+ build_nvme_name(entry->name, sizeof(entry->name), ctrlr);
+
+ entry->ctrlr = ctrlr;
+ entry->trtype = trid_entry->trid.trtype;
+ entry->next = g_controllers;
+ g_controllers = entry;
+
+ if (trid_entry->nsid == 0) {
+ for (nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr);
+ nsid != 0; nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid)) {
+ ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid);
+ if (ns == NULL) {
+ continue;
+ }
+ register_ns(ctrlr, ns);
+ }
+ } else {
+ ns = spdk_nvme_ctrlr_get_ns(ctrlr, trid_entry->nsid);
+ if (!ns) {
+ perror("Namespace does not exist.");
+ exit(1);
+ }
+
+ register_ns(ctrlr, ns);
+ }
+}
+
+static void
+abort_complete(void *ctx, const struct spdk_nvme_cpl *cpl)
+{
+ struct ctrlr_worker_ctx *ctrlr_ctx = ctx;
+
+ ctrlr_ctx->current_queue_depth--;
+ if (spdk_unlikely(spdk_nvme_cpl_is_error(cpl))) {
+ ctrlr_ctx->abort_failed++;
+ } else if ((cpl->cdw0 & 0x1) == 0) {
+ ctrlr_ctx->successful_abort++;
+ } else {
+ ctrlr_ctx->unsuccessful_abort++;
+ }
+}
+
+static void
+abort_task(struct perf_task *task)
+{
+ struct ns_worker_ctx *ns_ctx = task->ns_ctx;
+ struct ctrlr_worker_ctx *ctrlr_ctx = ns_ctx->ctrlr_ctx;
+ int rc;
+
+ /* Hold mutex to guard ctrlr_ctx->current_queue_depth. */
+ pthread_mutex_lock(&ctrlr_ctx->mutex);
+
+ rc = spdk_nvme_ctrlr_cmd_abort_ext(ctrlr_ctx->ctrlr, ns_ctx->qpair, task, abort_complete,
+ ctrlr_ctx);
+
+ if (spdk_unlikely(rc != 0)) {
+ ctrlr_ctx->abort_submit_failed++;
+ } else {
+ ctrlr_ctx->current_queue_depth++;
+ ctrlr_ctx->abort_submitted++;
+ }
+
+ pthread_mutex_unlock(&ctrlr_ctx->mutex);
+}
+
+static __thread unsigned int seed = 0;
+
+static inline void
+submit_single_io(struct perf_task *task)
+{
+ uint64_t offset_in_ios, lba;
+ int rc;
+ struct ns_worker_ctx *ns_ctx = task->ns_ctx;
+ struct ns_entry *entry = ns_ctx->entry;
+
+ if (g_is_random) {
+ offset_in_ios = rand_r(&seed) % entry->size_in_ios;
+ } else {
+ offset_in_ios = ns_ctx->offset_in_ios++;
+ if (ns_ctx->offset_in_ios == entry->size_in_ios) {
+ ns_ctx->offset_in_ios = 0;
+ }
+ }
+
+ lba = offset_in_ios * entry->io_size_blocks;
+
+ if ((g_rw_percentage == 100) ||
+ (g_rw_percentage != 0 && (rand_r(&seed) % 100) < g_rw_percentage)) {
+ rc = spdk_nvme_ns_cmd_read(entry->ns, ns_ctx->qpair, task->buf,
+ lba, entry->io_size_blocks, io_complete, task, 0);
+ } else {
+ rc = spdk_nvme_ns_cmd_write(entry->ns, ns_ctx->qpair, task->buf,
+ lba, entry->io_size_blocks, io_complete, task, 0);
+ }
+
+ if (spdk_unlikely(rc != 0)) {
+ fprintf(stderr, "I/O submission failed\n");
+ } else {
+ ns_ctx->current_queue_depth++;
+ ns_ctx->io_submitted++;
+
+ if ((ns_ctx->io_submitted % g_abort_interval) == 0) {
+ abort_task(task);
+ }
+ }
+
+}
+
+static void
+io_complete(void *ctx, const struct spdk_nvme_cpl *cpl)
+{
+ struct perf_task *task = ctx;
+ struct ns_worker_ctx *ns_ctx = task->ns_ctx;
+
+ ns_ctx->current_queue_depth--;
+ if (spdk_unlikely(spdk_nvme_cpl_is_error(cpl))) {
+ ns_ctx->io_failed++;
+ } else {
+ ns_ctx->io_completed++;
+ }
+
+ /* is_draining indicates when time has expired for the test run and we are
+ * just waiting for the previously submitted I/O to complete. In this case,
+ * do not submit a new I/O to replace the one just completed.
+ */
+ if (spdk_unlikely(ns_ctx->is_draining)) {
+ spdk_dma_free(task->buf);
+ free(task);
+ } else {
+ submit_single_io(task);
+ }
+}
+
+static struct perf_task *
+allocate_task(struct ns_worker_ctx *ns_ctx)
+{
+ struct perf_task *task;
+
+ task = calloc(1, sizeof(*task));
+ if (task == NULL) {
+ fprintf(stderr, "Failed to allocate task\n");
+ exit(1);
+ }
+
+ task->buf = spdk_dma_zmalloc(g_io_size_bytes, 0x200, NULL);
+ if (task->buf == NULL) {
+ free(task);
+ fprintf(stderr, "Failed to allocate task->buf\n");
+ exit(1);
+ }
+
+ task->ns_ctx = ns_ctx;
+
+ return task;
+}
+
+static void
+submit_io(struct ns_worker_ctx *ns_ctx, int queue_depth)
+{
+ struct perf_task *task;
+
+ while (queue_depth-- > 0) {
+ task = allocate_task(ns_ctx);
+ submit_single_io(task);
+ }
+}
+
+static int
+work_fn(void *arg)
+{
+ struct worker_thread *worker = (struct worker_thread *)arg;
+ struct ns_worker_ctx *ns_ctx;
+ struct ctrlr_worker_ctx *ctrlr_ctx;
+ struct ns_entry *ns_entry;
+ struct spdk_nvme_io_qpair_opts opts;
+ uint64_t tsc_end;
+ uint32_t unfinished_ctx;
+
+ /* Allocate queue pair for each namespace. */
+ ns_ctx = worker->ns_ctx;
+ while (ns_ctx != NULL) {
+ ns_entry = ns_ctx->entry;
+
+ spdk_nvme_ctrlr_get_default_io_qpair_opts(ns_entry->ctrlr, &opts, sizeof(opts));
+ if (opts.io_queue_requests < ns_entry->num_io_requests) {
+ opts.io_queue_requests = ns_entry->num_io_requests;
+ }
+
+ ns_ctx->qpair = spdk_nvme_ctrlr_alloc_io_qpair(ns_entry->ctrlr, &opts, sizeof(opts));
+ if (ns_ctx->qpair == NULL) {
+ fprintf(stderr, "spdk_nvme_ctrlr_alloc_io_qpair failed\n");
+ return 1;
+ }
+
+ ns_ctx = ns_ctx->next;
+ }
+
+ tsc_end = spdk_get_ticks() + g_time_in_sec * g_tsc_rate;
+
+ /* Submit initial I/O for each namespace. */
+ ns_ctx = worker->ns_ctx;
+ while (ns_ctx != NULL) {
+ submit_io(ns_ctx, g_queue_depth);
+ ns_ctx = ns_ctx->next;
+ }
+
+ while (1) {
+ ns_ctx = worker->ns_ctx;
+ while (ns_ctx != NULL) {
+ spdk_nvme_qpair_process_completions(ns_ctx->qpair, 0);
+ ns_ctx = ns_ctx->next;
+ }
+
+ if (worker->lcore == g_master_core) {
+ ctrlr_ctx = worker->ctrlr_ctx;
+ while (ctrlr_ctx) {
+ /* Hold mutex to guard ctrlr_ctx->current_queue_depth. */
+ pthread_mutex_lock(&ctrlr_ctx->mutex);
+ spdk_nvme_ctrlr_process_admin_completions(ctrlr_ctx->ctrlr);
+ pthread_mutex_unlock(&ctrlr_ctx->mutex);
+ ctrlr_ctx = ctrlr_ctx->next;
+ }
+ }
+
+ if (spdk_get_ticks() > tsc_end) {
+ break;
+ }
+ }
+
+ do {
+ unfinished_ctx = 0;
+
+ ns_ctx = worker->ns_ctx;
+ while (ns_ctx != NULL) {
+ if (!ns_ctx->is_draining) {
+ ns_ctx->is_draining = true;
+ }
+ if (ns_ctx->current_queue_depth > 0) {
+ spdk_nvme_qpair_process_completions(ns_ctx->qpair, 0);
+ if (ns_ctx->current_queue_depth == 0) {
+ spdk_nvme_ctrlr_free_io_qpair(ns_ctx->qpair);
+ } else {
+ unfinished_ctx++;
+ }
+ }
+ ns_ctx = ns_ctx->next;
+ }
+ } while (unfinished_ctx > 0);
+
+ if (worker->lcore == g_master_core) {
+ do {
+ unfinished_ctx = 0;
+
+ ctrlr_ctx = worker->ctrlr_ctx;
+ while (ctrlr_ctx != NULL) {
+ pthread_mutex_lock(&ctrlr_ctx->mutex);
+ if (ctrlr_ctx->current_queue_depth > 0) {
+ spdk_nvme_ctrlr_process_admin_completions(ctrlr_ctx->ctrlr);
+ if (ctrlr_ctx->current_queue_depth > 0) {
+ unfinished_ctx++;
+ }
+ }
+ pthread_mutex_unlock(&ctrlr_ctx->mutex);
+ ctrlr_ctx = ctrlr_ctx->next;
+ }
+ } while (unfinished_ctx > 0);
+ }
+
+ return 0;
+}
+
+static void
+usage(char *program_name)
+{
+ printf("%s options", program_name);
+
+ printf("\n");
+ printf("\t[-q io depth]\n");
+ printf("\t[-o io size in bytes]\n");
+ printf("\t[-w io pattern type, must be one of\n");
+ printf("\t\t(read, write, randread, randwrite, rw, randrw)]\n");
+ printf("\t[-M rwmixread (100 for reads, 0 for writes)]\n");
+ printf("\t[-t time in seconds]\n");
+ printf("\t[-c core mask for I/O submission/completion.]\n");
+ printf("\t\t(default: 1)\n");
+ printf("\t[-r Transport ID for local PCIe NVMe or NVMeoF]\n");
+ printf("\t Format: 'key:value [key:value] ...'\n");
+ printf("\t Keys:\n");
+ printf("\t trtype Transport type (e.g. PCIe, RDMA)\n");
+ printf("\t adrfam Address family (e.g. IPv4, IPv6)\n");
+ printf("\t traddr Transport address (e.g. 0000:04:00.0 for PCIe or 192.168.100.8 for RDMA)\n");
+ printf("\t trsvcid Transport service identifier (e.g. 4420)\n");
+ printf("\t subnqn Subsystem NQN (default: %s)\n", SPDK_NVMF_DISCOVERY_NQN);
+ printf("\t Example: -r 'trtype:PCIe traddr:0000:04:00.0' for PCIe or\n");
+ printf("\t -r 'trtype:RDMA adrfam:IPv4 traddr:192.168.100.8 trsvcid:4420' for NVMeoF\n");
+ printf("\t[-s DPDK huge memory size in MB.]\n");
+ printf("\t[-i shared memory group ID]\n");
+ printf("\t[-a abort interval.]\n");
+ printf("\t");
+ spdk_log_usage(stdout, "-T");
+#ifdef DEBUG
+ printf("\t[-G enable debug logging]\n");
+#else
+ printf("\t[-G enable debug logging (flag disabled, must reconfigure with --enable-debug)\n");
+#endif
+}
+
+static void
+unregister_trids(void)
+{
+ struct trid_entry *trid_entry, *tmp;
+
+ TAILQ_FOREACH_SAFE(trid_entry, &g_trid_list, tailq, tmp) {
+ TAILQ_REMOVE(&g_trid_list, trid_entry, tailq);
+ free(trid_entry);
+ }
+}
+
+static int
+add_trid(const char *trid_str)
+{
+ struct trid_entry *trid_entry;
+ struct spdk_nvme_transport_id *trid;
+ char *ns;
+
+ trid_entry = calloc(1, sizeof(*trid_entry));
+ if (trid_entry == NULL) {
+ return -1;
+ }
+
+ trid = &trid_entry->trid;
+ trid->trtype = SPDK_NVME_TRANSPORT_PCIE;
+ snprintf(trid->subnqn, sizeof(trid->subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN);
+
+ if (spdk_nvme_transport_id_parse(trid, trid_str) != 0) {
+ fprintf(stderr, "Invalid transport ID format '%s'\n", trid_str);
+ free(trid_entry);
+ return 1;
+ }
+
+ spdk_nvme_transport_id_populate_trstring(trid,
+ spdk_nvme_transport_id_trtype_str(trid->trtype));
+
+ ns = strcasestr(trid_str, "ns:");
+ if (ns) {
+ char nsid_str[6]; /* 5 digits maximum in an nsid */
+ int len;
+ int nsid;
+
+ ns += 3;
+
+ len = strcspn(ns, " \t\n");
+ if (len > 5) {
+ fprintf(stderr, "NVMe namespace IDs must be 5 digits or less\n");
+ free(trid_entry);
+ return 1;
+ }
+
+ memcpy(nsid_str, ns, len);
+ nsid_str[len] = '\0';
+
+ nsid = spdk_strtol(nsid_str, 10);
+ if (nsid <= 0 || nsid > 65535) {
+ fprintf(stderr, "NVMe namespace IDs must be less than 65536 and greater than 0\n");
+ free(trid_entry);
+ return 1;
+ }
+
+ trid_entry->nsid = (uint16_t)nsid;
+ }
+
+ TAILQ_INSERT_TAIL(&g_trid_list, trid_entry, tailq);
+ return 0;
+}
+
+static int
+parse_args(int argc, char **argv)
+{
+ int op;
+ long int val;
+ int rc;
+
+ while ((op = getopt(argc, argv, "a:c:i:o:q:r:s:t:w:M:")) != -1) {
+ switch (op) {
+ case 'a':
+ case 'i':
+ case 'o':
+ case 'q':
+ case 's':
+ case 't':
+ case 'M':
+ val = spdk_strtol(optarg, 10);
+ if (val < 0) {
+ fprintf(stderr, "Converting a string to integer failed\n");
+ return val;
+ }
+ switch (op) {
+ case 'a':
+ g_abort_interval = val;
+ break;
+ case 'i':
+ g_shm_id = val;
+ break;
+ case 'o':
+ g_io_size_bytes = val;
+ break;
+ case 'q':
+ g_queue_depth = val;
+ break;
+ case 's':
+ g_dpdk_mem = val;
+ break;
+ case 't':
+ g_time_in_sec = val;
+ break;
+ case 'M':
+ g_rw_percentage = val;
+ g_mix_specified = true;
+ break;
+ }
+ break;
+ case 'c':
+ g_core_mask = optarg;
+ break;
+ case 'r':
+ if (add_trid(optarg)) {
+ usage(argv[0]);
+ return 1;
+ }
+ break;
+ case 'w':
+ g_workload_type = optarg;
+ break;
+ case 'G':
+#ifndef DEBUG
+ fprintf(stderr, "%s must be configured with --enable-debug for -G flag\n",
+ argv[0]);
+ usage(argv[0]);
+ return 1;
+#else
+ spdk_log_set_flag("nvme");
+ spdk_log_set_print_level(SPDK_LOG_DEBUG);
+ break;
+#endif
+ case 'T':
+ rc = spdk_log_set_flag(optarg);
+ if (rc < 0) {
+ fprintf(stderr, "unknown flag\n");
+ usage(argv[0]);
+ exit(EXIT_FAILURE);
+ }
+ spdk_log_set_print_level(SPDK_LOG_DEBUG);
+#ifndef DEBUG
+ fprintf(stderr, "%s must be rebuilt with CONFIG_DEBUG=y for -T flag.\n",
+ argv[0]);
+ usage(argv[0]);
+ return 0;
+#endif
+ break;
+ default:
+ usage(argv[0]);
+ return 1;
+ }
+ }
+
+ if (!g_queue_depth) {
+ fprintf(stderr, "missing -q (queue size) operand\n");
+ usage(argv[0]);
+ return 1;
+ }
+ if (!g_io_size_bytes) {
+ fprintf(stderr, "missing -o (block size) operand\n");
+ usage(argv[0]);
+ return 1;
+ }
+ if (!g_workload_type) {
+ fprintf(stderr, "missing -t (test time in seconds) operand\n");
+ usage(argv[0]);
+ return 1;
+ }
+
+ if (!g_time_in_sec) {
+ usage(argv[0]);
+ return 1;
+ }
+
+ if (strncmp(g_workload_type, "rand", 4) == 0) {
+ g_is_random = 1;
+ g_workload_type = &g_workload_type[4];
+ }
+
+ if (strcmp(g_workload_type, "read") == 0 || strcmp(g_workload_type, "write") == 0) {
+ g_rw_percentage = strcmp(g_workload_type, "read") == 0 ? 100 : 0;
+ if (g_mix_specified) {
+ fprintf(stderr, "Ignoring -M option... Please use -M option"
+ " only when using rw or randrw.\n");
+ }
+ } else if (strcmp(g_workload_type, "rw") == 0) {
+ if (g_rw_percentage < 0 || g_rw_percentage > 100) {
+ fprintf(stderr,
+ "-M must be specified to value from 0 to 100 "
+ "for rw or randrw.\n");
+ return 1;
+ }
+ } else {
+ fprintf(stderr,
+ "io pattern type must be one of\n"
+ "(read, write, randread, randwrite, rw, randrw)\n");
+ return 1;
+ }
+
+ if (TAILQ_EMPTY(&g_trid_list)) {
+ /* If no transport IDs specified, default to enumerating all local PCIe devices */
+ add_trid("trtype:PCIe");
+ } else {
+ struct trid_entry *trid_entry, *trid_entry_tmp;
+
+ g_no_pci = true;
+ /* check whether there is local PCIe type */
+ TAILQ_FOREACH_SAFE(trid_entry, &g_trid_list, tailq, trid_entry_tmp) {
+ if (trid_entry->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) {
+ g_no_pci = false;
+ break;
+ }
+ }
+ }
+
+ return 0;
+}
+
+static int
+register_workers(void)
+{
+ uint32_t i;
+ struct worker_thread *worker;
+
+ g_workers = NULL;
+ g_num_workers = 0;
+
+ SPDK_ENV_FOREACH_CORE(i) {
+ worker = calloc(1, sizeof(*worker));
+ if (worker == NULL) {
+ fprintf(stderr, "Unable to allocate worker\n");
+ return -1;
+ }
+
+ worker->lcore = i;
+ worker->next = g_workers;
+ g_workers = worker;
+ g_num_workers++;
+ }
+
+ return 0;
+}
+
+static void
+unregister_workers(void)
+{
+ struct worker_thread *worker = g_workers;
+
+ /* Free namespace context and worker thread */
+ while (worker) {
+ struct worker_thread *next_worker = worker->next;
+ struct ns_worker_ctx *ns_ctx = worker->ns_ctx;
+
+ while (ns_ctx) {
+ struct ns_worker_ctx *next_ns_ctx = ns_ctx->next;
+
+ printf("NS: %s I/O completed: %lu, failed: %lu\n",
+ ns_ctx->entry->name, ns_ctx->io_completed, ns_ctx->io_failed);
+ free(ns_ctx);
+ ns_ctx = next_ns_ctx;
+ }
+
+ struct ctrlr_worker_ctx *ctrlr_ctx = worker->ctrlr_ctx;
+
+ while (ctrlr_ctx) {
+ struct ctrlr_worker_ctx *next_ctrlr_ctx = ctrlr_ctx->next;
+
+ printf("CTRLR: %s abort submitted %lu, failed to submit %lu\n",
+ ctrlr_ctx->entry->name, ctrlr_ctx->abort_submitted,
+ ctrlr_ctx->abort_submit_failed);
+ printf("\t success %lu, unsuccess %lu, failed %lu\n",
+ ctrlr_ctx->successful_abort, ctrlr_ctx->unsuccessful_abort,
+ ctrlr_ctx->abort_failed);
+ free(ctrlr_ctx);
+ ctrlr_ctx = next_ctrlr_ctx;
+ }
+
+ free(worker);
+ worker = next_worker;
+ }
+}
+
+static bool
+probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
+ struct spdk_nvme_ctrlr_opts *opts)
+{
+ return true;
+}
+
+static void
+attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
+ struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts)
+{
+ struct trid_entry *trid_entry = cb_ctx;
+ struct spdk_pci_addr pci_addr;
+ struct spdk_pci_device *pci_dev;
+ struct spdk_pci_id pci_id;
+
+ if (trid->trtype != SPDK_NVME_TRANSPORT_PCIE) {
+ printf("Attached to NVMe over Fabrics controller at %s:%s: %s\n",
+ trid->traddr, trid->trsvcid,
+ trid->subnqn);
+ } else {
+ if (spdk_pci_addr_parse(&pci_addr, trid->traddr)) {
+ return;
+ }
+
+ pci_dev = spdk_nvme_ctrlr_get_pci_device(ctrlr);
+ if (!pci_dev) {
+ return;
+ }
+
+ pci_id = spdk_pci_device_get_id(pci_dev);
+
+ printf("Attached to NVMe Controller at %s [%04x:%04x]\n",
+ trid->traddr,
+ pci_id.vendor_id, pci_id.device_id);
+ }
+
+ register_ctrlr(ctrlr, trid_entry);
+}
+
+static int
+register_controllers(void)
+{
+ struct trid_entry *trid_entry;
+
+ printf("Initializing NVMe Controllers\n");
+
+ TAILQ_FOREACH(trid_entry, &g_trid_list, tailq) {
+ if (spdk_nvme_probe(&trid_entry->trid, trid_entry, probe_cb, attach_cb, NULL) != 0) {
+ fprintf(stderr, "spdk_nvme_probe() failed for transport address '%s'\n",
+ trid_entry->trid.traddr);
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
+static void
+unregister_controllers(void)
+{
+ struct ctrlr_entry *entry = g_controllers;
+
+ while (entry) {
+ struct ctrlr_entry *next = entry->next;
+ spdk_nvme_detach(entry->ctrlr);
+ free(entry);
+ entry = next;
+ }
+}
+
+static int
+associate_master_worker_with_ctrlr(void)
+{
+ struct ctrlr_entry *entry = g_controllers;
+ struct worker_thread *worker = g_workers;
+ struct ctrlr_worker_ctx *ctrlr_ctx;
+
+ while (worker) {
+ if (worker->lcore == g_master_core) {
+ break;
+ }
+ worker = worker->next;
+ }
+
+ if (!worker) {
+ return -1;
+ }
+
+ while (entry) {
+ ctrlr_ctx = calloc(1, sizeof(struct ctrlr_worker_ctx));
+ if (!ctrlr_ctx) {
+ return -1;
+ }
+
+ pthread_mutex_init(&ctrlr_ctx->mutex, NULL);
+ ctrlr_ctx->entry = entry;
+ ctrlr_ctx->ctrlr = entry->ctrlr;
+ ctrlr_ctx->next = worker->ctrlr_ctx;
+ worker->ctrlr_ctx = ctrlr_ctx;
+
+ entry = entry->next;
+ }
+
+ return 0;
+}
+
+static struct ctrlr_worker_ctx *
+get_ctrlr_worker_ctx(struct spdk_nvme_ctrlr *ctrlr)
+{
+ struct worker_thread *worker = g_workers;
+ struct ctrlr_worker_ctx *ctrlr_ctx;
+
+ while (worker != NULL) {
+ if (worker->lcore == g_master_core) {
+ break;
+ }
+ worker = worker->next;
+ }
+
+ if (!worker) {
+ return NULL;
+ }
+
+ ctrlr_ctx = worker->ctrlr_ctx;
+
+ while (ctrlr_ctx != NULL) {
+ if (ctrlr_ctx->ctrlr == ctrlr) {
+ return ctrlr_ctx;
+ }
+ ctrlr_ctx = ctrlr_ctx->next;
+ }
+
+ return NULL;
+}
+
+static int
+associate_workers_with_ns(void)
+{
+ struct ns_entry *entry = g_namespaces;
+ struct worker_thread *worker = g_workers;
+ struct ns_worker_ctx *ns_ctx;
+ int i, count;
+
+ count = g_num_namespaces > g_num_workers ? g_num_namespaces : g_num_workers;
+
+ for (i = 0; i < count; i++) {
+ if (entry == NULL) {
+ break;
+ }
+
+ ns_ctx = calloc(1, sizeof(struct ns_worker_ctx));
+ if (!ns_ctx) {
+ return -1;
+ }
+
+ printf("Associating %s with lcore %d\n", entry->name, worker->lcore);
+ ns_ctx->entry = entry;
+ ns_ctx->ctrlr_ctx = get_ctrlr_worker_ctx(entry->ctrlr);
+ if (!ns_ctx->ctrlr_ctx) {
+ free(ns_ctx);
+ return -1;
+ }
+
+ ns_ctx->next = worker->ns_ctx;
+ worker->ns_ctx = ns_ctx;
+
+ worker = worker->next;
+ if (worker == NULL) {
+ worker = g_workers;
+ }
+
+ entry = entry->next;
+ if (entry == NULL) {
+ entry = g_namespaces;
+ }
+ }
+
+ return 0;
+}
+
+int main(int argc, char **argv)
+{
+ int rc;
+ struct worker_thread *worker, *master_worker;
+ struct spdk_env_opts opts;
+
+ rc = parse_args(argc, argv);
+ if (rc != 0) {
+ return rc;
+ }
+
+ spdk_env_opts_init(&opts);
+ opts.name = "abort";
+ opts.shm_id = g_shm_id;
+ if (g_core_mask) {
+ opts.core_mask = g_core_mask;
+ }
+
+ if (g_dpdk_mem) {
+ opts.mem_size = g_dpdk_mem;
+ }
+ if (g_no_pci) {
+ opts.no_pci = g_no_pci;
+ }
+ if (spdk_env_init(&opts) < 0) {
+ fprintf(stderr, "Unable to initialize SPDK env\n");
+ rc = -1;
+ goto cleanup;
+ }
+
+ g_tsc_rate = spdk_get_ticks_hz();
+
+ if (register_workers() != 0) {
+ rc = -1;
+ goto cleanup;
+ }
+
+ if (register_controllers() != 0) {
+ rc = -1;
+ goto cleanup;
+ }
+
+ if (g_warn) {
+ printf("WARNING: Some requested NVMe devices were skipped\n");
+ }
+
+ if (g_num_namespaces == 0) {
+ fprintf(stderr, "No valid NVMe controllers found\n");
+ goto cleanup;
+ }
+
+ if (associate_master_worker_with_ctrlr() != 0) {
+ rc = -1;
+ goto cleanup;
+ }
+
+ if (associate_workers_with_ns() != 0) {
+ rc = -1;
+ goto cleanup;
+ }
+
+ printf("Initialization complete. Launching workers.\n");
+
+ /* Launch all of the slave workers */
+ g_master_core = spdk_env_get_current_core();
+ master_worker = NULL;
+ worker = g_workers;
+ while (worker != NULL) {
+ if (worker->lcore != g_master_core) {
+ spdk_env_thread_launch_pinned(worker->lcore, work_fn, worker);
+ } else {
+ assert(master_worker == NULL);
+ master_worker = worker;
+ }
+ worker = worker->next;
+ }
+
+ assert(master_worker != NULL);
+ rc = work_fn(master_worker);
+
+ spdk_env_thread_wait_all();
+
+cleanup:
+ unregister_trids();
+ unregister_workers();
+ unregister_namespaces();
+ unregister_controllers();
+
+ if (rc != 0) {
+ fprintf(stderr, "%s: errors occured\n", argv[0]);
+ }
+
+ return rc;
+}
diff --git a/src/spdk/examples/nvme/arbitration/.gitignore b/src/spdk/examples/nvme/arbitration/.gitignore
new file mode 100644
index 000000000..f1d6e38dd
--- /dev/null
+++ b/src/spdk/examples/nvme/arbitration/.gitignore
@@ -0,0 +1 @@
+arbitration
diff --git a/src/spdk/examples/nvme/arbitration/Makefile b/src/spdk/examples/nvme/arbitration/Makefile
new file mode 100644
index 000000000..71cff76e6
--- /dev/null
+++ b/src/spdk/examples/nvme/arbitration/Makefile
@@ -0,0 +1,38 @@
+#
+# BSD LICENSE
+#
+# Copyright (c) Intel Corporation.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Intel Corporation nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
+
+APP = arbitration
+
+include $(SPDK_ROOT_DIR)/mk/nvme.libtest.mk
diff --git a/src/spdk/examples/nvme/arbitration/arbitration.c b/src/spdk/examples/nvme/arbitration/arbitration.c
new file mode 100644
index 000000000..444076041
--- /dev/null
+++ b/src/spdk/examples/nvme/arbitration/arbitration.c
@@ -0,0 +1,1158 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+
+#include "spdk/nvme.h"
+#include "spdk/env.h"
+#include "spdk/string.h"
+#include "spdk/nvme_intel.h"
+
+struct ctrlr_entry {
+ struct spdk_nvme_ctrlr *ctrlr;
+ struct spdk_nvme_intel_rw_latency_page latency_page;
+ struct ctrlr_entry *next;
+ char name[1024];
+};
+
+struct ns_entry {
+ struct {
+ struct spdk_nvme_ctrlr *ctrlr;
+ struct spdk_nvme_ns *ns;
+ } nvme;
+
+ struct ns_entry *next;
+ uint32_t io_size_blocks;
+ uint64_t size_in_ios;
+ char name[1024];
+};
+
+struct ns_worker_ctx {
+ struct ns_entry *entry;
+ uint64_t io_completed;
+ uint64_t current_queue_depth;
+ uint64_t offset_in_ios;
+ bool is_draining;
+ struct spdk_nvme_qpair *qpair;
+ struct ns_worker_ctx *next;
+};
+
+struct arb_task {
+ struct ns_worker_ctx *ns_ctx;
+ void *buf;
+};
+
+struct worker_thread {
+ struct ns_worker_ctx *ns_ctx;
+ struct worker_thread *next;
+ unsigned lcore;
+ enum spdk_nvme_qprio qprio;
+};
+
+struct arb_context {
+ int shm_id;
+ int outstanding_commands;
+ int num_namespaces;
+ int num_workers;
+ int rw_percentage;
+ int is_random;
+ int queue_depth;
+ int time_in_sec;
+ int io_count;
+ uint8_t latency_tracking_enable;
+ uint8_t arbitration_mechanism;
+ uint8_t arbitration_config;
+ uint32_t io_size_bytes;
+ uint32_t max_completions;
+ uint64_t tsc_rate;
+ const char *core_mask;
+ const char *workload_type;
+};
+
+struct feature {
+ uint32_t result;
+ bool valid;
+};
+
+static struct spdk_mempool *task_pool = NULL;
+
+static struct ctrlr_entry *g_controllers = NULL;
+static struct ns_entry *g_namespaces = NULL;
+static struct worker_thread *g_workers = NULL;
+
+static struct feature features[SPDK_NVME_FEAT_ARBITRATION + 1] = {};
+
+static struct arb_context g_arbitration = {
+ .shm_id = -1,
+ .outstanding_commands = 0,
+ .num_workers = 0,
+ .num_namespaces = 0,
+ .rw_percentage = 50,
+ .queue_depth = 64,
+ .time_in_sec = 60,
+ .io_count = 100000,
+ .latency_tracking_enable = 0,
+ .arbitration_mechanism = SPDK_NVME_CC_AMS_RR,
+ .arbitration_config = 0,
+ .io_size_bytes = 131072,
+ .max_completions = 0,
+ /* Default 4 cores for urgent/high/medium/low */
+ .core_mask = "0xf",
+ .workload_type = "randrw",
+};
+
+/*
+ * For weighted round robin arbitration mechanism, the smaller value between
+ * weight and burst will be picked to execute the commands in one queue.
+ */
+#define USER_SPECIFIED_HIGH_PRIORITY_WEIGHT 32
+#define USER_SPECIFIED_MEDIUM_PRIORITY_WEIGHT 16
+#define USER_SPECIFIED_LOW_PRIORITY_WEIGHT 8
+
+static void task_complete(struct arb_task *task);
+
+static void io_complete(void *ctx, const struct spdk_nvme_cpl *completion);
+
+static void get_arb_feature(struct spdk_nvme_ctrlr *ctrlr);
+
+static int set_arb_feature(struct spdk_nvme_ctrlr *ctrlr);
+
+static const char *print_qprio(enum spdk_nvme_qprio);
+
+
+static void
+register_ns(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns *ns)
+{
+ struct ns_entry *entry;
+ const struct spdk_nvme_ctrlr_data *cdata;
+
+ cdata = spdk_nvme_ctrlr_get_data(ctrlr);
+
+ if (spdk_nvme_ns_get_size(ns) < g_arbitration.io_size_bytes ||
+ spdk_nvme_ns_get_extended_sector_size(ns) > g_arbitration.io_size_bytes ||
+ g_arbitration.io_size_bytes % spdk_nvme_ns_get_extended_sector_size(ns)) {
+ printf("WARNING: controller %-20.20s (%-20.20s) ns %u has invalid "
+ "ns size %" PRIu64 " / block size %u for I/O size %u\n",
+ cdata->mn, cdata->sn, spdk_nvme_ns_get_id(ns),
+ spdk_nvme_ns_get_size(ns), spdk_nvme_ns_get_extended_sector_size(ns),
+ g_arbitration.io_size_bytes);
+ return;
+ }
+
+ entry = malloc(sizeof(struct ns_entry));
+ if (entry == NULL) {
+ perror("ns_entry malloc");
+ exit(1);
+ }
+
+ entry->nvme.ctrlr = ctrlr;
+ entry->nvme.ns = ns;
+
+ entry->size_in_ios = spdk_nvme_ns_get_size(ns) / g_arbitration.io_size_bytes;
+ entry->io_size_blocks = g_arbitration.io_size_bytes / spdk_nvme_ns_get_sector_size(ns);
+
+ snprintf(entry->name, 44, "%-20.20s (%-20.20s)", cdata->mn, cdata->sn);
+
+ g_arbitration.num_namespaces++;
+ entry->next = g_namespaces;
+ g_namespaces = entry;
+}
+
+static void
+enable_latency_tracking_complete(void *cb_arg, const struct spdk_nvme_cpl *cpl)
+{
+ if (spdk_nvme_cpl_is_error(cpl)) {
+ printf("enable_latency_tracking_complete failed\n");
+ }
+ g_arbitration.outstanding_commands--;
+}
+
+static void
+set_latency_tracking_feature(struct spdk_nvme_ctrlr *ctrlr, bool enable)
+{
+ int res;
+ union spdk_nvme_intel_feat_latency_tracking latency_tracking;
+
+ if (enable) {
+ latency_tracking.bits.enable = 0x01;
+ } else {
+ latency_tracking.bits.enable = 0x00;
+ }
+
+ res = spdk_nvme_ctrlr_cmd_set_feature(ctrlr, SPDK_NVME_INTEL_FEAT_LATENCY_TRACKING,
+ latency_tracking.raw, 0, NULL, 0, enable_latency_tracking_complete, NULL);
+ if (res) {
+ printf("fail to allocate nvme request.\n");
+ return;
+ }
+ g_arbitration.outstanding_commands++;
+
+ while (g_arbitration.outstanding_commands) {
+ spdk_nvme_ctrlr_process_admin_completions(ctrlr);
+ }
+}
+
+static void
+register_ctrlr(struct spdk_nvme_ctrlr *ctrlr)
+{
+ uint32_t nsid;
+ struct spdk_nvme_ns *ns;
+ struct ctrlr_entry *entry = calloc(1, sizeof(struct ctrlr_entry));
+ union spdk_nvme_cap_register cap = spdk_nvme_ctrlr_get_regs_cap(ctrlr);
+ const struct spdk_nvme_ctrlr_data *cdata = spdk_nvme_ctrlr_get_data(ctrlr);
+
+ if (entry == NULL) {
+ perror("ctrlr_entry malloc");
+ exit(1);
+ }
+
+ snprintf(entry->name, sizeof(entry->name), "%-20.20s (%-20.20s)", cdata->mn, cdata->sn);
+
+ entry->ctrlr = ctrlr;
+ entry->next = g_controllers;
+ g_controllers = entry;
+
+ if ((g_arbitration.latency_tracking_enable != 0) &&
+ spdk_nvme_ctrlr_is_feature_supported(ctrlr, SPDK_NVME_INTEL_FEAT_LATENCY_TRACKING)) {
+ set_latency_tracking_feature(ctrlr, true);
+ }
+
+ for (nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr); nsid != 0;
+ nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid)) {
+ ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid);
+ if (ns == NULL) {
+ continue;
+ }
+ register_ns(ctrlr, ns);
+ }
+
+ if (g_arbitration.arbitration_mechanism == SPDK_NVME_CAP_AMS_WRR &&
+ (cap.bits.ams & SPDK_NVME_CAP_AMS_WRR)) {
+ get_arb_feature(ctrlr);
+
+ if (g_arbitration.arbitration_config != 0) {
+ set_arb_feature(ctrlr);
+ get_arb_feature(ctrlr);
+ }
+ }
+}
+
+static __thread unsigned int seed = 0;
+
+static void
+submit_single_io(struct ns_worker_ctx *ns_ctx)
+{
+ struct arb_task *task = NULL;
+ uint64_t offset_in_ios;
+ int rc;
+ struct ns_entry *entry = ns_ctx->entry;
+
+ task = spdk_mempool_get(task_pool);
+ if (!task) {
+ fprintf(stderr, "Failed to get task from task_pool\n");
+ exit(1);
+ }
+
+ task->buf = spdk_dma_zmalloc(g_arbitration.io_size_bytes, 0x200, NULL);
+ if (!task->buf) {
+ spdk_mempool_put(task_pool, task);
+ fprintf(stderr, "task->buf spdk_dma_zmalloc failed\n");
+ exit(1);
+ }
+
+ task->ns_ctx = ns_ctx;
+
+ if (g_arbitration.is_random) {
+ offset_in_ios = rand_r(&seed) % entry->size_in_ios;
+ } else {
+ offset_in_ios = ns_ctx->offset_in_ios++;
+ if (ns_ctx->offset_in_ios == entry->size_in_ios) {
+ ns_ctx->offset_in_ios = 0;
+ }
+ }
+
+ if ((g_arbitration.rw_percentage == 100) ||
+ (g_arbitration.rw_percentage != 0 &&
+ ((rand_r(&seed) % 100) < g_arbitration.rw_percentage))) {
+ rc = spdk_nvme_ns_cmd_read(entry->nvme.ns, ns_ctx->qpair, task->buf,
+ offset_in_ios * entry->io_size_blocks,
+ entry->io_size_blocks, io_complete, task, 0);
+ } else {
+ rc = spdk_nvme_ns_cmd_write(entry->nvme.ns, ns_ctx->qpair, task->buf,
+ offset_in_ios * entry->io_size_blocks,
+ entry->io_size_blocks, io_complete, task, 0);
+ }
+
+ if (rc != 0) {
+ fprintf(stderr, "starting I/O failed\n");
+ } else {
+ ns_ctx->current_queue_depth++;
+ }
+}
+
+static void
+task_complete(struct arb_task *task)
+{
+ struct ns_worker_ctx *ns_ctx;
+
+ ns_ctx = task->ns_ctx;
+ ns_ctx->current_queue_depth--;
+ ns_ctx->io_completed++;
+
+ spdk_dma_free(task->buf);
+ spdk_mempool_put(task_pool, task);
+
+ /*
+ * is_draining indicates when time has expired for the test run
+ * and we are just waiting for the previously submitted I/O
+ * to complete. In this case, do not submit a new I/O to replace
+ * the one just completed.
+ */
+ if (!ns_ctx->is_draining) {
+ submit_single_io(ns_ctx);
+ }
+}
+
+static void
+io_complete(void *ctx, const struct spdk_nvme_cpl *completion)
+{
+ task_complete((struct arb_task *)ctx);
+}
+
+static void
+check_io(struct ns_worker_ctx *ns_ctx)
+{
+ spdk_nvme_qpair_process_completions(ns_ctx->qpair, g_arbitration.max_completions);
+}
+
+static void
+submit_io(struct ns_worker_ctx *ns_ctx, int queue_depth)
+{
+ while (queue_depth-- > 0) {
+ submit_single_io(ns_ctx);
+ }
+}
+
+static void
+drain_io(struct ns_worker_ctx *ns_ctx)
+{
+ ns_ctx->is_draining = true;
+ while (ns_ctx->current_queue_depth > 0) {
+ check_io(ns_ctx);
+ }
+}
+
+static int
+init_ns_worker_ctx(struct ns_worker_ctx *ns_ctx, enum spdk_nvme_qprio qprio)
+{
+ struct spdk_nvme_ctrlr *ctrlr = ns_ctx->entry->nvme.ctrlr;
+ struct spdk_nvme_io_qpair_opts opts;
+
+ spdk_nvme_ctrlr_get_default_io_qpair_opts(ctrlr, &opts, sizeof(opts));
+ opts.qprio = qprio;
+
+ ns_ctx->qpair = spdk_nvme_ctrlr_alloc_io_qpair(ctrlr, &opts, sizeof(opts));
+ if (!ns_ctx->qpair) {
+ printf("ERROR: spdk_nvme_ctrlr_alloc_io_qpair failed\n");
+ return 1;
+ }
+
+ return 0;
+}
+
+static void
+cleanup_ns_worker_ctx(struct ns_worker_ctx *ns_ctx)
+{
+ spdk_nvme_ctrlr_free_io_qpair(ns_ctx->qpair);
+}
+
+static void
+cleanup(uint32_t task_count)
+{
+ struct ns_entry *entry = g_namespaces;
+ struct ns_entry *next_entry = NULL;
+ struct worker_thread *worker = g_workers;
+ struct worker_thread *next_worker = NULL;
+
+ while (entry) {
+ next_entry = entry->next;
+ free(entry);
+ entry = next_entry;
+ };
+
+ while (worker) {
+ struct ns_worker_ctx *ns_ctx = worker->ns_ctx;
+
+ /* ns_worker_ctx is a list in the worker */
+ while (ns_ctx) {
+ struct ns_worker_ctx *next_ns_ctx = ns_ctx->next;
+ free(ns_ctx);
+ ns_ctx = next_ns_ctx;
+ }
+
+ next_worker = worker->next;
+ free(worker);
+ worker = next_worker;
+ };
+
+ if (spdk_mempool_count(task_pool) != (size_t)task_count) {
+ fprintf(stderr, "task_pool count is %zu but should be %u\n",
+ spdk_mempool_count(task_pool), task_count);
+ }
+ spdk_mempool_free(task_pool);
+}
+
+static int
+work_fn(void *arg)
+{
+ uint64_t tsc_end;
+ struct worker_thread *worker = (struct worker_thread *)arg;
+ struct ns_worker_ctx *ns_ctx = NULL;
+
+ printf("Starting thread on core %u with %s\n", worker->lcore, print_qprio(worker->qprio));
+
+ /* Allocate a queue pair for each namespace. */
+ ns_ctx = worker->ns_ctx;
+ while (ns_ctx != NULL) {
+ if (init_ns_worker_ctx(ns_ctx, worker->qprio) != 0) {
+ printf("ERROR: init_ns_worker_ctx() failed\n");
+ return 1;
+ }
+ ns_ctx = ns_ctx->next;
+ }
+
+ tsc_end = spdk_get_ticks() + g_arbitration.time_in_sec * g_arbitration.tsc_rate;
+
+ /* Submit initial I/O for each namespace. */
+ ns_ctx = worker->ns_ctx;
+
+ while (ns_ctx != NULL) {
+ submit_io(ns_ctx, g_arbitration.queue_depth);
+ ns_ctx = ns_ctx->next;
+ }
+
+ while (1) {
+ /*
+ * Check for completed I/O for each controller. A new
+ * I/O will be submitted in the io_complete callback
+ * to replace each I/O that is completed.
+ */
+ ns_ctx = worker->ns_ctx;
+ while (ns_ctx != NULL) {
+ check_io(ns_ctx);
+ ns_ctx = ns_ctx->next;
+ }
+
+ if (spdk_get_ticks() > tsc_end) {
+ break;
+ }
+ }
+
+ ns_ctx = worker->ns_ctx;
+ while (ns_ctx != NULL) {
+ drain_io(ns_ctx);
+ cleanup_ns_worker_ctx(ns_ctx);
+ ns_ctx = ns_ctx->next;
+ }
+
+ return 0;
+}
+
+static void
+usage(char *program_name)
+{
+ printf("%s options", program_name);
+ printf("\n");
+ printf("\t[-q io depth]\n");
+ printf("\t[-s io size in bytes]\n");
+ printf("\t[-w io pattern type, must be one of\n");
+ printf("\t\t(read, write, randread, randwrite, rw, randrw)]\n");
+ printf("\t[-M rwmixread (100 for reads, 0 for writes)]\n");
+ printf("\t[-l enable latency tracking, default: disabled]\n");
+ printf("\t\t(0 - disabled; 1 - enabled)\n");
+ printf("\t[-t time in seconds]\n");
+ printf("\t[-c core mask for I/O submission/completion.]\n");
+ printf("\t\t(default: 0xf - 4 cores)]\n");
+ printf("\t[-m max completions per poll]\n");
+ printf("\t\t(default: 0 - unlimited)\n");
+ printf("\t[-a arbitration mechanism, must be one of below]\n");
+ printf("\t\t(0, 1, 2)]\n");
+ printf("\t\t(0: default round robin mechanism)]\n");
+ printf("\t\t(1: weighted round robin mechanism)]\n");
+ printf("\t\t(2: vendor specific mechanism)]\n");
+ printf("\t[-b enable arbitration user configuration, default: disabled]\n");
+ printf("\t\t(0 - disabled; 1 - enabled)\n");
+ printf("\t[-n subjected IOs for performance comparison]\n");
+ printf("\t[-i shared memory group ID]\n");
+}
+
+static const char *
+print_qprio(enum spdk_nvme_qprio qprio)
+{
+ switch (qprio) {
+ case SPDK_NVME_QPRIO_URGENT:
+ return "urgent priority queue";
+ case SPDK_NVME_QPRIO_HIGH:
+ return "high priority queue";
+ case SPDK_NVME_QPRIO_MEDIUM:
+ return "medium priority queue";
+ case SPDK_NVME_QPRIO_LOW:
+ return "low priority queue";
+ default:
+ return "invalid priority queue";
+ }
+}
+
+
+static void
+print_configuration(char *program_name)
+{
+ printf("%s run with configuration:\n", program_name);
+ printf("%s -q %d -s %d -w %s -M %d -l %d -t %d -c %s -m %d -a %d -b %d -n %d -i %d\n",
+ program_name,
+ g_arbitration.queue_depth,
+ g_arbitration.io_size_bytes,
+ g_arbitration.workload_type,
+ g_arbitration.rw_percentage,
+ g_arbitration.latency_tracking_enable,
+ g_arbitration.time_in_sec,
+ g_arbitration.core_mask,
+ g_arbitration.max_completions,
+ g_arbitration.arbitration_mechanism,
+ g_arbitration.arbitration_config,
+ g_arbitration.io_count,
+ g_arbitration.shm_id);
+}
+
+
+static void
+print_performance(void)
+{
+ float io_per_second, sent_all_io_in_secs;
+ struct worker_thread *worker;
+ struct ns_worker_ctx *ns_ctx;
+
+ worker = g_workers;
+ while (worker) {
+ ns_ctx = worker->ns_ctx;
+ while (ns_ctx) {
+ io_per_second = (float)ns_ctx->io_completed / g_arbitration.time_in_sec;
+ sent_all_io_in_secs = g_arbitration.io_count / io_per_second;
+ printf("%-43.43s core %u: %8.2f IO/s %8.2f secs/%d ios\n",
+ ns_ctx->entry->name, worker->lcore,
+ io_per_second, sent_all_io_in_secs, g_arbitration.io_count);
+ ns_ctx = ns_ctx->next;
+ }
+ worker = worker->next;
+ }
+ printf("========================================================\n");
+
+ printf("\n");
+}
+
+static void
+print_latency_page(struct ctrlr_entry *entry)
+{
+ int i;
+
+ printf("\n");
+ printf("%s\n", entry->name);
+ printf("--------------------------------------------------------\n");
+
+ for (i = 0; i < 32; i++) {
+ if (entry->latency_page.buckets_32us[i])
+ printf("Bucket %dus - %dus: %d\n", i * 32, (i + 1) * 32,
+ entry->latency_page.buckets_32us[i]);
+ }
+ for (i = 0; i < 31; i++) {
+ if (entry->latency_page.buckets_1ms[i])
+ printf("Bucket %dms - %dms: %d\n", i + 1, i + 2,
+ entry->latency_page.buckets_1ms[i]);
+ }
+ for (i = 0; i < 31; i++) {
+ if (entry->latency_page.buckets_32ms[i])
+ printf("Bucket %dms - %dms: %d\n", (i + 1) * 32, (i + 2) * 32,
+ entry->latency_page.buckets_32ms[i]);
+ }
+}
+
+static void
+print_latency_statistics(const char *op_name, enum spdk_nvme_intel_log_page log_page)
+{
+ struct ctrlr_entry *ctrlr;
+
+ printf("%s Latency Statistics:\n", op_name);
+ printf("========================================================\n");
+ ctrlr = g_controllers;
+ while (ctrlr) {
+ if (spdk_nvme_ctrlr_is_log_page_supported(ctrlr->ctrlr, log_page)) {
+ if (spdk_nvme_ctrlr_cmd_get_log_page(
+ ctrlr->ctrlr, log_page,
+ SPDK_NVME_GLOBAL_NS_TAG,
+ &ctrlr->latency_page,
+ sizeof(struct spdk_nvme_intel_rw_latency_page),
+ 0,
+ enable_latency_tracking_complete,
+ NULL)) {
+ printf("nvme_ctrlr_cmd_get_log_page() failed\n");
+ exit(1);
+ }
+
+ g_arbitration.outstanding_commands++;
+ } else {
+ printf("Controller %s: %s latency statistics not supported\n",
+ ctrlr->name, op_name);
+ }
+ ctrlr = ctrlr->next;
+ }
+
+ while (g_arbitration.outstanding_commands) {
+ ctrlr = g_controllers;
+ while (ctrlr) {
+ spdk_nvme_ctrlr_process_admin_completions(ctrlr->ctrlr);
+ ctrlr = ctrlr->next;
+ }
+ }
+
+ ctrlr = g_controllers;
+ while (ctrlr) {
+ if (spdk_nvme_ctrlr_is_log_page_supported(ctrlr->ctrlr, log_page)) {
+ print_latency_page(ctrlr);
+ }
+ ctrlr = ctrlr->next;
+ }
+ printf("\n");
+}
+
+static void
+print_stats(void)
+{
+ print_performance();
+ if (g_arbitration.latency_tracking_enable) {
+ if (g_arbitration.rw_percentage != 0) {
+ print_latency_statistics("Read", SPDK_NVME_INTEL_LOG_READ_CMD_LATENCY);
+ }
+ if (g_arbitration.rw_percentage != 100) {
+ print_latency_statistics("Write", SPDK_NVME_INTEL_LOG_WRITE_CMD_LATENCY);
+ }
+ }
+}
+
+static int
+parse_args(int argc, char **argv)
+{
+ const char *workload_type = NULL;
+ int op = 0;
+ bool mix_specified = false;
+ long int val;
+
+ while ((op = getopt(argc, argv, "c:l:i:m:q:s:t:w:M:a:b:n:h")) != -1) {
+ switch (op) {
+ case 'c':
+ g_arbitration.core_mask = optarg;
+ break;
+ case 'w':
+ g_arbitration.workload_type = optarg;
+ break;
+ case 'h':
+ case '?':
+ usage(argv[0]);
+ return 1;
+ default:
+ val = spdk_strtol(optarg, 10);
+ if (val < 0) {
+ fprintf(stderr, "Converting a string to integer failed\n");
+ return val;
+ }
+ switch (op) {
+ case 'i':
+ g_arbitration.shm_id = val;
+ break;
+ case 'l':
+ g_arbitration.latency_tracking_enable = val;
+ break;
+ case 'm':
+ g_arbitration.max_completions = val;
+ break;
+ case 'q':
+ g_arbitration.queue_depth = val;
+ break;
+ case 's':
+ g_arbitration.io_size_bytes = val;
+ break;
+ case 't':
+ g_arbitration.time_in_sec = val;
+ break;
+ case 'M':
+ g_arbitration.rw_percentage = val;
+ mix_specified = true;
+ break;
+ case 'a':
+ g_arbitration.arbitration_mechanism = val;
+ break;
+ case 'b':
+ g_arbitration.arbitration_config = val;
+ break;
+ case 'n':
+ g_arbitration.io_count = val;
+ break;
+ default:
+ usage(argv[0]);
+ return -EINVAL;
+ }
+ }
+ }
+
+ workload_type = g_arbitration.workload_type;
+
+ if (strcmp(workload_type, "read") &&
+ strcmp(workload_type, "write") &&
+ strcmp(workload_type, "randread") &&
+ strcmp(workload_type, "randwrite") &&
+ strcmp(workload_type, "rw") &&
+ strcmp(workload_type, "randrw")) {
+ fprintf(stderr,
+ "io pattern type must be one of\n"
+ "(read, write, randread, randwrite, rw, randrw)\n");
+ return 1;
+ }
+
+ if (!strcmp(workload_type, "read") ||
+ !strcmp(workload_type, "randread")) {
+ g_arbitration.rw_percentage = 100;
+ }
+
+ if (!strcmp(workload_type, "write") ||
+ !strcmp(workload_type, "randwrite")) {
+ g_arbitration.rw_percentage = 0;
+ }
+
+ if (!strcmp(workload_type, "read") ||
+ !strcmp(workload_type, "randread") ||
+ !strcmp(workload_type, "write") ||
+ !strcmp(workload_type, "randwrite")) {
+ if (mix_specified) {
+ fprintf(stderr, "Ignoring -M option... Please use -M option"
+ " only when using rw or randrw.\n");
+ }
+ }
+
+ if (!strcmp(workload_type, "rw") ||
+ !strcmp(workload_type, "randrw")) {
+ if (g_arbitration.rw_percentage < 0 || g_arbitration.rw_percentage > 100) {
+ fprintf(stderr,
+ "-M must be specified to value from 0 to 100 "
+ "for rw or randrw.\n");
+ return 1;
+ }
+ }
+
+ if (!strcmp(workload_type, "read") ||
+ !strcmp(workload_type, "write") ||
+ !strcmp(workload_type, "rw")) {
+ g_arbitration.is_random = 0;
+ } else {
+ g_arbitration.is_random = 1;
+ }
+
+ if (g_arbitration.latency_tracking_enable != 0 &&
+ g_arbitration.latency_tracking_enable != 1) {
+ fprintf(stderr,
+ "-l must be specified to value 0 or 1.\n");
+ return 1;
+ }
+
+ switch (g_arbitration.arbitration_mechanism) {
+ case SPDK_NVME_CC_AMS_RR:
+ case SPDK_NVME_CC_AMS_WRR:
+ case SPDK_NVME_CC_AMS_VS:
+ break;
+ default:
+ fprintf(stderr,
+ "-a must be specified to value 0, 1, or 7.\n");
+ return 1;
+ }
+
+ if (g_arbitration.arbitration_config != 0 &&
+ g_arbitration.arbitration_config != 1) {
+ fprintf(stderr,
+ "-b must be specified to value 0 or 1.\n");
+ return 1;
+ } else if (g_arbitration.arbitration_config == 1 &&
+ g_arbitration.arbitration_mechanism != SPDK_NVME_CC_AMS_WRR) {
+ fprintf(stderr,
+ "-a must be specified to 1 (WRR) together.\n");
+ return 1;
+ }
+
+ return 0;
+}
+
+static int
+register_workers(void)
+{
+ uint32_t i;
+ struct worker_thread *worker;
+ enum spdk_nvme_qprio qprio = SPDK_NVME_QPRIO_URGENT;
+
+ g_workers = NULL;
+ g_arbitration.num_workers = 0;
+
+ SPDK_ENV_FOREACH_CORE(i) {
+ worker = calloc(1, sizeof(*worker));
+ if (worker == NULL) {
+ fprintf(stderr, "Unable to allocate worker\n");
+ return -1;
+ }
+
+ worker->lcore = i;
+ worker->next = g_workers;
+ g_workers = worker;
+ g_arbitration.num_workers++;
+
+ if (g_arbitration.arbitration_mechanism == SPDK_NVME_CAP_AMS_WRR) {
+ qprio++;
+ }
+
+ worker->qprio = qprio & SPDK_NVME_CREATE_IO_SQ_QPRIO_MASK;
+ }
+
+ return 0;
+}
+
+static bool
+probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
+ struct spdk_nvme_ctrlr_opts *opts)
+{
+ /* Update with user specified arbitration configuration */
+ opts->arb_mechanism = g_arbitration.arbitration_mechanism;
+
+ printf("Attaching to %s\n", trid->traddr);
+
+ return true;
+}
+
+static void
+attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
+ struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts)
+{
+ printf("Attached to %s\n", trid->traddr);
+
+ /* Update with actual arbitration configuration in use */
+ g_arbitration.arbitration_mechanism = opts->arb_mechanism;
+
+ register_ctrlr(ctrlr);
+}
+
+static int
+register_controllers(void)
+{
+ printf("Initializing NVMe Controllers\n");
+
+ if (spdk_nvme_probe(NULL, NULL, probe_cb, attach_cb, NULL) != 0) {
+ fprintf(stderr, "spdk_nvme_probe() failed\n");
+ return 1;
+ }
+
+ if (g_arbitration.num_namespaces == 0) {
+ fprintf(stderr, "No valid namespaces to continue IO testing\n");
+ return 1;
+ }
+
+ return 0;
+}
+
+static void
+unregister_controllers(void)
+{
+ struct ctrlr_entry *entry = g_controllers;
+
+ while (entry) {
+ struct ctrlr_entry *next = entry->next;
+ if (g_arbitration.latency_tracking_enable &&
+ spdk_nvme_ctrlr_is_feature_supported(entry->ctrlr, SPDK_NVME_INTEL_FEAT_LATENCY_TRACKING)) {
+ set_latency_tracking_feature(entry->ctrlr, false);
+ }
+ spdk_nvme_detach(entry->ctrlr);
+ free(entry);
+ entry = next;
+ }
+}
+
+static int
+associate_workers_with_ns(void)
+{
+ struct ns_entry *entry = g_namespaces;
+ struct worker_thread *worker = g_workers;
+ struct ns_worker_ctx *ns_ctx;
+ int i, count;
+
+ count = g_arbitration.num_namespaces > g_arbitration.num_workers ?
+ g_arbitration.num_namespaces : g_arbitration.num_workers;
+
+ for (i = 0; i < count; i++) {
+ if (entry == NULL) {
+ break;
+ }
+
+ ns_ctx = malloc(sizeof(struct ns_worker_ctx));
+ if (!ns_ctx) {
+ return 1;
+ }
+ memset(ns_ctx, 0, sizeof(*ns_ctx));
+
+ printf("Associating %s with lcore %d\n", entry->name, worker->lcore);
+ ns_ctx->entry = entry;
+ ns_ctx->next = worker->ns_ctx;
+ worker->ns_ctx = ns_ctx;
+
+ worker = worker->next;
+ if (worker == NULL) {
+ worker = g_workers;
+ }
+
+ entry = entry->next;
+ if (entry == NULL) {
+ entry = g_namespaces;
+ }
+
+ }
+
+ return 0;
+}
+
+static void
+get_feature_completion(void *cb_arg, const struct spdk_nvme_cpl *cpl)
+{
+ struct feature *feature = cb_arg;
+ int fid = feature - features;
+
+ if (spdk_nvme_cpl_is_error(cpl)) {
+ printf("get_feature(0x%02X) failed\n", fid);
+ } else {
+ feature->result = cpl->cdw0;
+ feature->valid = true;
+ }
+
+ g_arbitration.outstanding_commands--;
+}
+
+static int
+get_feature(struct spdk_nvme_ctrlr *ctrlr, uint8_t fid)
+{
+ struct spdk_nvme_cmd cmd = {};
+ struct feature *feature = &features[fid];
+
+ feature->valid = false;
+
+ cmd.opc = SPDK_NVME_OPC_GET_FEATURES;
+ cmd.cdw10_bits.get_features.fid = fid;
+
+ return spdk_nvme_ctrlr_cmd_admin_raw(ctrlr, &cmd, NULL, 0, get_feature_completion, feature);
+}
+
+static void
+get_arb_feature(struct spdk_nvme_ctrlr *ctrlr)
+{
+ get_feature(ctrlr, SPDK_NVME_FEAT_ARBITRATION);
+
+ g_arbitration.outstanding_commands++;
+
+ while (g_arbitration.outstanding_commands) {
+ spdk_nvme_ctrlr_process_admin_completions(ctrlr);
+ }
+
+ if (features[SPDK_NVME_FEAT_ARBITRATION].valid) {
+ union spdk_nvme_cmd_cdw11 arb;
+ arb.feat_arbitration.raw = features[SPDK_NVME_FEAT_ARBITRATION].result;
+
+ printf("Current Arbitration Configuration\n");
+ printf("===========\n");
+ printf("Arbitration Burst: ");
+ if (arb.feat_arbitration.bits.ab == SPDK_NVME_ARBITRATION_BURST_UNLIMITED) {
+ printf("no limit\n");
+ } else {
+ printf("%u\n", 1u << arb.feat_arbitration.bits.ab);
+ }
+
+ printf("Low Priority Weight: %u\n", arb.feat_arbitration.bits.lpw + 1);
+ printf("Medium Priority Weight: %u\n", arb.feat_arbitration.bits.mpw + 1);
+ printf("High Priority Weight: %u\n", arb.feat_arbitration.bits.hpw + 1);
+ printf("\n");
+ }
+}
+
+static void
+set_feature_completion(void *cb_arg, const struct spdk_nvme_cpl *cpl)
+{
+ struct feature *feature = cb_arg;
+ int fid = feature - features;
+
+ if (spdk_nvme_cpl_is_error(cpl)) {
+ printf("set_feature(0x%02X) failed\n", fid);
+ feature->valid = false;
+ } else {
+ printf("Set Arbitration Feature Successfully\n");
+ }
+
+ g_arbitration.outstanding_commands--;
+}
+
+static int
+set_arb_feature(struct spdk_nvme_ctrlr *ctrlr)
+{
+ int ret;
+ struct spdk_nvme_cmd cmd = {};
+
+ cmd.opc = SPDK_NVME_OPC_SET_FEATURES;
+ cmd.cdw10_bits.set_features.fid = SPDK_NVME_FEAT_ARBITRATION;
+
+ g_arbitration.outstanding_commands = 0;
+
+ if (features[SPDK_NVME_FEAT_ARBITRATION].valid) {
+ cmd.cdw11_bits.feat_arbitration.bits.ab = SPDK_NVME_ARBITRATION_BURST_UNLIMITED;
+ cmd.cdw11_bits.feat_arbitration.bits.lpw = USER_SPECIFIED_LOW_PRIORITY_WEIGHT;
+ cmd.cdw11_bits.feat_arbitration.bits.mpw = USER_SPECIFIED_MEDIUM_PRIORITY_WEIGHT;
+ cmd.cdw11_bits.feat_arbitration.bits.hpw = USER_SPECIFIED_HIGH_PRIORITY_WEIGHT;
+ }
+
+ ret = spdk_nvme_ctrlr_cmd_admin_raw(ctrlr, &cmd, NULL, 0,
+ set_feature_completion, &features[SPDK_NVME_FEAT_ARBITRATION]);
+ if (ret) {
+ printf("Set Arbitration Feature: Failed 0x%x\n", ret);
+ return 1;
+ }
+
+ g_arbitration.outstanding_commands++;
+
+ while (g_arbitration.outstanding_commands) {
+ spdk_nvme_ctrlr_process_admin_completions(ctrlr);
+ }
+
+ if (!features[SPDK_NVME_FEAT_ARBITRATION].valid) {
+ printf("Set Arbitration Feature failed and use default configuration\n");
+ }
+
+ return 0;
+}
+
+int
+main(int argc, char **argv)
+{
+ int rc;
+ struct worker_thread *worker, *master_worker;
+ unsigned master_core;
+ char task_pool_name[30];
+ uint32_t task_count;
+ struct spdk_env_opts opts;
+
+ rc = parse_args(argc, argv);
+ if (rc != 0) {
+ return rc;
+ }
+
+ spdk_env_opts_init(&opts);
+ opts.name = "arb";
+ opts.core_mask = g_arbitration.core_mask;
+ opts.shm_id = g_arbitration.shm_id;
+ if (spdk_env_init(&opts) < 0) {
+ return 1;
+ }
+
+ g_arbitration.tsc_rate = spdk_get_ticks_hz();
+
+ if (register_workers() != 0) {
+ return 1;
+ }
+
+ if (register_controllers() != 0) {
+ return 1;
+ }
+
+ if (associate_workers_with_ns() != 0) {
+ return 1;
+ }
+
+ snprintf(task_pool_name, sizeof(task_pool_name), "task_pool_%d", getpid());
+
+ /*
+ * The task_count will be dynamically calculated based on the
+ * number of attached active namespaces, queue depth and number
+ * of cores (workers) involved in the IO perations.
+ */
+ task_count = g_arbitration.num_namespaces > g_arbitration.num_workers ?
+ g_arbitration.num_namespaces : g_arbitration.num_workers;
+ task_count *= g_arbitration.queue_depth;
+
+ task_pool = spdk_mempool_create(task_pool_name, task_count,
+ sizeof(struct arb_task), 0, SPDK_ENV_SOCKET_ID_ANY);
+ if (task_pool == NULL) {
+ fprintf(stderr, "could not initialize task pool\n");
+ return 1;
+ }
+
+ print_configuration(argv[0]);
+
+ printf("Initialization complete. Launching workers.\n");
+
+ /* Launch all of the slave workers */
+ master_core = spdk_env_get_current_core();
+ master_worker = NULL;
+ worker = g_workers;
+ while (worker != NULL) {
+ if (worker->lcore != master_core) {
+ spdk_env_thread_launch_pinned(worker->lcore, work_fn, worker);
+ } else {
+ assert(master_worker == NULL);
+ master_worker = worker;
+ }
+ worker = worker->next;
+ }
+
+ assert(master_worker != NULL);
+ rc = work_fn(master_worker);
+
+ spdk_env_thread_wait_all();
+
+ print_stats();
+
+ unregister_controllers();
+
+ cleanup(task_count);
+
+ if (rc != 0) {
+ fprintf(stderr, "%s: errors occured\n", argv[0]);
+ }
+
+ return rc;
+}
diff --git a/src/spdk/examples/nvme/cmb_copy/.gitignore b/src/spdk/examples/nvme/cmb_copy/.gitignore
new file mode 100644
index 000000000..fce738032
--- /dev/null
+++ b/src/spdk/examples/nvme/cmb_copy/.gitignore
@@ -0,0 +1 @@
+cmb_copy
diff --git a/src/spdk/examples/nvme/cmb_copy/Makefile b/src/spdk/examples/nvme/cmb_copy/Makefile
new file mode 100644
index 000000000..77a143abb
--- /dev/null
+++ b/src/spdk/examples/nvme/cmb_copy/Makefile
@@ -0,0 +1,38 @@
+#
+# BSD LICENSE
+#
+# Copyright (c) Eideticom Inc
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Eideticom Inc nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
+
+APP = cmb_copy
+
+include $(SPDK_ROOT_DIR)/mk/nvme.libtest.mk
diff --git a/src/spdk/examples/nvme/cmb_copy/cmb_copy.c b/src/spdk/examples/nvme/cmb_copy/cmb_copy.c
new file mode 100644
index 000000000..50eedcbba
--- /dev/null
+++ b/src/spdk/examples/nvme/cmb_copy/cmb_copy.c
@@ -0,0 +1,412 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Eideticom Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Eideticom Inc, nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+
+#include "spdk/env.h"
+#include "spdk/nvme.h"
+#include "spdk/string.h"
+
+#define CMB_COPY_DELIM "-"
+#define CMB_COPY_READ 0
+#define CMB_COPY_WRITE 1
+
+struct nvme_io {
+ struct spdk_nvme_ctrlr *ctrlr;
+ struct spdk_nvme_transport_id trid;
+ struct spdk_nvme_qpair *qpair;
+ struct spdk_nvme_ns *ns;
+ unsigned nsid;
+ unsigned slba;
+ unsigned nlbas;
+ uint32_t lba_size;
+ unsigned done;
+};
+
+struct cmb_t {
+ struct spdk_nvme_transport_id trid;
+ struct spdk_nvme_ctrlr *ctrlr;
+};
+
+struct config {
+ struct nvme_io read;
+ struct nvme_io write;
+ struct cmb_t cmb;
+ size_t copy_size;
+};
+
+static struct config g_config;
+
+/* Namespaces index from 1. Return 0 to invoke an error */
+static unsigned
+get_nsid(const struct spdk_nvme_transport_id *trid)
+{
+ if (!strcmp(trid->traddr, g_config.read.trid.traddr)) {
+ return g_config.read.nsid;
+ }
+ if (!strcmp(trid->traddr, g_config.write.trid.traddr)) {
+ return g_config.write.nsid;
+ }
+ return 0;
+}
+
+static int
+get_rw(const struct spdk_nvme_transport_id *trid)
+{
+ if (!strcmp(trid->traddr, g_config.read.trid.traddr)) {
+ return CMB_COPY_READ;
+ }
+ if (!strcmp(trid->traddr, g_config.write.trid.traddr)) {
+ return CMB_COPY_WRITE;
+ }
+ return -1;
+}
+
+static void
+check_io(void *arg, const struct spdk_nvme_cpl *completion)
+{
+ int *rw = (unsigned *)arg;
+
+ if (*rw == CMB_COPY_READ) {
+ g_config.read.done = 1;
+ } else {
+ g_config.write.done = 1;
+ }
+}
+
+static int
+cmb_copy(void)
+{
+ int rc = 0, rw;
+ void *buf;
+ size_t sz;
+
+ /* Allocate QPs for the read and write controllers */
+ g_config.read.qpair = spdk_nvme_ctrlr_alloc_io_qpair(g_config.read.ctrlr, NULL, 0);
+ g_config.write.qpair = spdk_nvme_ctrlr_alloc_io_qpair(g_config.write.ctrlr, NULL, 0);
+ if (g_config.read.qpair == NULL || g_config.read.qpair == NULL) {
+ printf("ERROR: spdk_nvme_ctrlr_alloc_io_qpair() failed\n");
+ return -ENOMEM;
+ }
+
+ /* Allocate a buffer from our CMB */
+ buf = spdk_nvme_ctrlr_map_cmb(g_config.cmb.ctrlr, &sz);
+ if (buf == NULL || sz < g_config.copy_size) {
+ printf("ERROR: buffer allocation failed\n");
+ printf("Are you sure %s has a valid CMB?\n",
+ g_config.cmb.trid.traddr);
+ return -ENOMEM;
+ }
+
+ /* Clear the done flags */
+ g_config.read.done = 0;
+ g_config.write.done = 0;
+
+ rw = CMB_COPY_READ;
+ /* Do the read to the CMB IO buffer */
+ rc = spdk_nvme_ns_cmd_read(g_config.read.ns, g_config.read.qpair, buf,
+ g_config.read.slba, g_config.read.nlbas,
+ check_io, &rw, 0);
+ if (rc != 0) {
+ fprintf(stderr, "starting read I/O failed\n");
+ return -EIO;
+ }
+ while (!g_config.read.done) {
+ spdk_nvme_qpair_process_completions(g_config.read.qpair, 0);
+ }
+
+ /* Do the write from the CMB IO buffer */
+ rw = CMB_COPY_WRITE;
+ rc = spdk_nvme_ns_cmd_write(g_config.write.ns, g_config.write.qpair, buf,
+ g_config.write.slba, g_config.write.nlbas,
+ check_io, &rw, 0);
+ if (rc != 0) {
+ fprintf(stderr, "starting write I/O failed\n");
+ return -EIO;
+ }
+ while (!g_config.write.done) {
+ spdk_nvme_qpair_process_completions(g_config.write.qpair, 0);
+ }
+
+ /* Clear the done flags */
+ g_config.read.done = 0;
+ g_config.write.done = 0;
+
+ /* Free CMB buffer */
+ spdk_nvme_ctrlr_unmap_cmb(g_config.cmb.ctrlr);
+
+ /* Free the queues */
+ spdk_nvme_ctrlr_free_io_qpair(g_config.read.qpair);
+ spdk_nvme_ctrlr_free_io_qpair(g_config.write.qpair);
+
+ return rc;
+}
+
+static bool
+probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
+ struct spdk_nvme_ctrlr_opts *opts)
+{
+ /* We will only attach to the read or write controller */
+ if (strcmp(trid->traddr, g_config.read.trid.traddr) &&
+ strcmp(trid->traddr, g_config.write.trid.traddr)) {
+ printf("%s - not probed %s!\n", __func__, trid->traddr);
+ return 0;
+ }
+
+ opts->use_cmb_sqs = false;
+
+ printf("%s - probed %s!\n", __func__, trid->traddr);
+ return 1;
+}
+
+static void
+attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
+ struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts)
+{
+ struct spdk_nvme_ns *ns;
+
+ ns = spdk_nvme_ctrlr_get_ns(ctrlr, get_nsid(trid));
+ if (ns == NULL) {
+ fprintf(stderr, "Could not locate namespace %d on controller %s.\n",
+ get_nsid(trid), trid->traddr);
+ exit(-1);
+ }
+ if (get_rw(trid) == CMB_COPY_READ) {
+ g_config.read.ctrlr = ctrlr;
+ g_config.read.ns = ns;
+ g_config.read.lba_size = spdk_nvme_ns_get_sector_size(ns);
+ } else {
+ g_config.write.ctrlr = ctrlr;
+ g_config.write.ns = ns;
+ g_config.write.lba_size = spdk_nvme_ns_get_sector_size(ns);
+ }
+ printf("%s - attached %s!\n", __func__, trid->traddr);
+
+ return;
+}
+
+static void
+usage(char *program_name)
+{
+ printf("%s options (all mandatory)", program_name);
+ printf("\n");
+ printf("\t[-r NVMe read parameters]\n");
+ printf("\t[-w NVMe write parameters]\n");
+ printf("\t[-c CMB to use for data buffers]\n");
+ printf("\n");
+ printf("Read/Write params:\n");
+ printf(" <pci id>-<namespace>-<start LBA>-<number of LBAs>\n");
+}
+
+static void
+parse(char *in, struct nvme_io *io)
+{
+ char *tok = NULL;
+ long int val;
+
+ tok = strtok(in, CMB_COPY_DELIM);
+ if (tok == NULL) {
+ goto err;
+ }
+ snprintf(&io->trid.traddr[0], SPDK_NVMF_TRADDR_MAX_LEN + 1,
+ "%s", tok);
+
+ tok = strtok(NULL, CMB_COPY_DELIM);
+ if (tok == NULL) {
+ goto err;
+ }
+ val = spdk_strtol(tok, 10);
+ if (val < 0) {
+ goto err;
+ }
+ io->nsid = (unsigned)val;
+
+ tok = strtok(NULL, CMB_COPY_DELIM);
+ if (tok == NULL) {
+ goto err;
+ }
+ val = spdk_strtol(tok, 10);
+ if (val < 0) {
+ goto err;
+ }
+ io->slba = (unsigned)val;
+
+ tok = strtok(NULL, CMB_COPY_DELIM);
+ if (tok == NULL) {
+ goto err;
+ }
+ val = spdk_strtol(tok, 10);
+ if (val < 0) {
+ goto err;
+ }
+ io->nlbas = (unsigned)val;
+
+ tok = strtok(NULL, CMB_COPY_DELIM);
+ if (tok != NULL) {
+ goto err;
+ }
+ return;
+
+err:
+ fprintf(stderr, "%s: error parsing %s\n", __func__, in);
+ exit(-1);
+
+}
+
+static int
+parse_args(int argc, char **argv)
+{
+ int op;
+ unsigned read = 0, write = 0, cmb = 0;
+
+ while ((op = getopt(argc, argv, "r:w:c:")) != -1) {
+ switch (op) {
+ case 'r':
+ parse(optarg, &g_config.read);
+ read = 1;
+ break;
+ case 'w':
+ parse(optarg, &g_config.write);
+ write = 1;
+ break;
+ case 'c':
+ snprintf(g_config.cmb.trid.traddr, SPDK_NVMF_TRADDR_MAX_LEN + 1,
+ "%s", optarg);
+ cmb = 1;
+ break;
+ default:
+ usage(argv[0]);
+ return 1;
+ }
+ }
+
+ if ((!read || !write || !cmb)) {
+ usage(argv[0]);
+ return 1;
+ }
+
+ return 0;
+}
+
+int main(int argc, char **argv)
+{
+ int rc = 0;
+ struct spdk_env_opts opts;
+
+ /*
+ * Parse the input arguments. For now we use the following
+ * format list:
+ *
+ * <pci id>-<namespace>-<start LBA>-<number of LBAs>
+ *
+ */
+ rc = parse_args(argc, argv);
+ if (rc) {
+ fprintf(stderr, "Error in parse_args(): %d\n",
+ rc);
+ return -1;
+ }
+
+ /*
+ * SPDK relies on an abstraction around the local environment
+ * named env that handles memory allocation and PCI device operations.
+ * This library must be initialized first.
+ *
+ */
+ spdk_env_opts_init(&opts);
+ opts.name = "cmb_copy";
+ opts.shm_id = 0;
+ if (spdk_env_init(&opts) < 0) {
+ fprintf(stderr, "Unable to initialize SPDK env\n");
+ return 1;
+ }
+
+ /*
+ * CMBs only apply to PCIe attached NVMe controllers so we
+ * only probe the PCIe bus. This is the default when we pass
+ * in NULL for the first argument.
+ */
+
+ rc = spdk_nvme_probe(NULL, NULL, probe_cb, attach_cb, NULL);
+ if (rc) {
+ fprintf(stderr, "Error in spdk_nvme_probe(): %d\n",
+ rc);
+ return -1;
+ }
+
+ /*
+ * For now enforce that the read and write controller are not
+ * the same. This avoids an internal only DMA.
+ */
+ if (!strcmp(g_config.write.trid.traddr, g_config.read.trid.traddr)) {
+ fprintf(stderr, "Read and Write controllers must differ!\n");
+ return -1;
+ }
+
+ /*
+ * Perform a few sanity checks and set the buffer size for the
+ * CMB.
+ */
+ if (g_config.read.nlbas * g_config.read.lba_size !=
+ g_config.write.nlbas * g_config.write.lba_size) {
+ fprintf(stderr, "Read and write sizes do not match!\n");
+ return -1;
+ }
+ g_config.copy_size = g_config.read.nlbas * g_config.read.lba_size;
+
+ /*
+ * Get the ctrlr pointer for the CMB. For now we assume this
+ * is either the read or write NVMe controller though in
+ * theory that is not a necessary condition.
+ */
+
+ if (!strcmp(g_config.cmb.trid.traddr, g_config.read.trid.traddr)) {
+ g_config.cmb.ctrlr = g_config.read.ctrlr;
+ }
+ if (!strcmp(g_config.cmb.trid.traddr, g_config.write.trid.traddr)) {
+ g_config.cmb.ctrlr = g_config.write.ctrlr;
+ }
+
+ /*
+ * Call the cmb_copy() function which performs the CMB
+ * based copy or returns an error code if it fails.
+ */
+ rc = cmb_copy();
+ if (rc) {
+ fprintf(stderr, "Error in spdk_cmb_copy(): %d\n",
+ rc);
+ return -1;
+ }
+
+ return rc;
+}
diff --git a/src/spdk/examples/nvme/fio_plugin/.gitignore b/src/spdk/examples/nvme/fio_plugin/.gitignore
new file mode 100644
index 000000000..1b0b36ac4
--- /dev/null
+++ b/src/spdk/examples/nvme/fio_plugin/.gitignore
@@ -0,0 +1 @@
+fio_plugin
diff --git a/src/spdk/examples/nvme/fio_plugin/Makefile b/src/spdk/examples/nvme/fio_plugin/Makefile
new file mode 100644
index 000000000..1f71802df
--- /dev/null
+++ b/src/spdk/examples/nvme/fio_plugin/Makefile
@@ -0,0 +1,51 @@
+#
+# BSD LICENSE
+#
+# Copyright (c) Intel Corporation.
+# Copyright (c) 2015-2016, Micron Technology, Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Intel Corporation nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+include $(SPDK_ROOT_DIR)/mk/spdk.modules.mk
+
+FIO_PLUGIN := spdk_nvme
+
+C_SRCS = fio_plugin.c
+
+# Unable to combine the FIO plugin and the VPP socket abstraction (license incompatibility)
+SPDK_LIB_LIST = $(filter-out sock_vpp,$(SOCK_MODULES_LIST))
+SPDK_LIB_LIST += nvme thread util log sock vmd jsonrpc json rpc
+
+ifeq ($(CONFIG_RDMA),y)
+SPDK_LIB_LIST += rdma
+endif
+
+include $(SPDK_ROOT_DIR)/mk/spdk.fio.mk
diff --git a/src/spdk/examples/nvme/fio_plugin/README.md b/src/spdk/examples/nvme/fio_plugin/README.md
new file mode 100644
index 000000000..e7a8b7c01
--- /dev/null
+++ b/src/spdk/examples/nvme/fio_plugin/README.md
@@ -0,0 +1,107 @@
+# Compiling fio
+
+First, clone the fio source repository from https://github.com/axboe/fio
+
+ git clone https://github.com/axboe/fio
+
+Then check out the latest fio version and compile the code:
+
+ make
+
+# Compiling SPDK
+
+First, clone the SPDK source repository from https://github.com/spdk/spdk
+
+ git clone https://github.com/spdk/spdk
+ git submodule update --init
+
+Then, run the SPDK configure script to enable fio (point it to the root of the fio repository):
+
+ cd spdk
+ ./configure --with-fio=/path/to/fio/repo <other configuration options>
+
+Finally, build SPDK:
+
+ make
+
+**Note to advanced users**: These steps assume you're using the DPDK submodule. If you are using your
+own version of DPDK, the fio plugin requires that DPDK be compiled with -fPIC. You can compile DPDK
+with -fPIC by modifying your DPDK configuration file and adding the line:
+
+ EXTRA_CFLAGS=-fPIC
+
+# Usage
+
+To use the SPDK fio plugin with fio, specify the plugin binary using LD_PRELOAD when running
+fio and set ioengine=spdk in the fio configuration file (see example_config.fio in the same
+directory as this README).
+
+ LD_PRELOAD=<path to spdk repo>/build/fio/spdk_nvme fio
+
+To select NVMe devices, you pass an SPDK Transport Identifier string as the filename. These are in the
+form:
+
+ filename=key=value [key=value] ... ns=value
+
+Specifically, for local PCIe NVMe devices it will look like this:
+
+ filename=trtype=PCIe traddr=0000.04.00.0 ns=1
+
+And remote devices accessed via NVMe over Fabrics will look like this:
+
+ filename=trtype=RDMA adrfam=IPv4 traddr=192.168.100.8 trsvcid=4420 ns=1
+
+**Note**: The specification of the PCIe address should not use the normal ':'
+and instead only use '.'. This is a limitation in fio - it splits filenames on
+':'. Also, the NVMe namespaces start at 1, not 0, and the namespace must be
+specified at the end of the string.
+
+Currently the SPDK fio plugin is limited to the thread usage model, so fio jobs must also specify thread=1
+when using the SPDK fio plugin.
+
+fio also currently has a race condition on shutdown if dynamically loading the ioengine by specifying the
+engine's full path via the ioengine parameter - LD_PRELOAD is recommended to avoid this race condition.
+
+When testing random workloads, it is recommended to set norandommap=1. fio's random map
+processing consumes extra CPU cycles which will degrade performance over time with
+the fio_plugin since all I/O are submitted and completed on a single CPU core.
+
+When testing FIO on multiple NVMe SSDs with SPDK plugin, it is recommended to use multiple jobs in FIO configurion.
+It has been observed that there are some performance gap between FIO(with SPDK plugin enabled) and SPDK perf
+(examples/nvme/perf/perf) on testing multiple NVMe SSDs. If you use one job(i.e., use one CPU core) configured for
+FIO test, the performance is worse than SPDK perf (also using one CPU core) against many NVMe SSDs. But if you use
+multiple jobs for FIO test, the performance of FIO is similiar with SPDK perf. After analyzing this phenomenon, we
+think that is caused by the FIO architecture. Mainly FIO can scale with multiple threads (i.e., using CPU cores),
+but it is not good to use one thread against many I/O devices.
+
+# End-to-end Data Protection (Optional)
+
+Running with PI setting, following settings steps are required.
+First, format device namespace with proper PI setting. For example:
+
+ nvme format /dev/nvme0n1 -l 1 -i 1 -p 0 -m 1
+
+In fio configure file, add PRACT and set PRCHK by flags(GUARD|REFTAG|APPTAG) properly. For example:
+
+ pi_act=0
+ pi_chk=GUARD
+
+Blocksize should be set as the sum of data and metadata. For example, if data blocksize is 512 Byte, host generated
+PI metadata is 8 Byte, then blocksize in fio configure file should be 520 Byte:
+
+ bs=520
+
+The storage device may use a block format that requires separate metadata (DIX). In this scenario, the fio_plugin
+will automatically allocate an extra 4KiB buffer per I/O to hold this metadata. For some cases, such as 512 byte
+blocks with 32 metadata bytes per block and a 128KiB I/O size, 4KiB isn't large enough. In this case, the
+`md_per_io_size` option may be specified to increase the size of the metadata buffer.
+
+Expose two options 'apptag' and 'apptag_mask', users can change them in the configuration file when using
+application tag and application tag mask in end-to-end data protection. Application tag and application
+tag mask are set to 0x1234 and 0xFFFF by default.
+
+# VMD (Optional)
+
+To enable VMD enumeration add enable_vmd flag in fio configuration file:
+
+ enable_vmd=1
diff --git a/src/spdk/examples/nvme/fio_plugin/example_config.fio b/src/spdk/examples/nvme/fio_plugin/example_config.fio
new file mode 100644
index 000000000..a8e62ccb9
--- /dev/null
+++ b/src/spdk/examples/nvme/fio_plugin/example_config.fio
@@ -0,0 +1,15 @@
+[global]
+ioengine=spdk
+thread=1
+group_reporting=1
+direct=1
+verify=0
+time_based=1
+ramp_time=0
+runtime=2
+iodepth=128
+rw=randrw
+bs=4k
+
+[test]
+numjobs=1
diff --git a/src/spdk/examples/nvme/fio_plugin/fio_plugin.c b/src/spdk/examples/nvme/fio_plugin/fio_plugin.c
new file mode 100644
index 000000000..7aabeb8cb
--- /dev/null
+++ b/src/spdk/examples/nvme/fio_plugin/fio_plugin.c
@@ -0,0 +1,1267 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation. All rights reserved.
+ * Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+
+#include "spdk/nvme.h"
+#include "spdk/vmd.h"
+#include "spdk/env.h"
+#include "spdk/string.h"
+#include "spdk/log.h"
+#include "spdk/endian.h"
+#include "spdk/dif.h"
+#include "spdk/util.h"
+
+#include "config-host.h"
+#include "fio.h"
+#include "optgroup.h"
+
+/* FreeBSD is missing CLOCK_MONOTONIC_RAW,
+ * so alternative is provided. */
+#ifndef CLOCK_MONOTONIC_RAW /* Defined in glibc bits/time.h */
+#define CLOCK_MONOTONIC_RAW CLOCK_MONOTONIC
+#endif
+
+#define NVME_IO_ALIGN 4096
+
+static bool g_spdk_env_initialized;
+static int g_spdk_enable_sgl = 0;
+static uint32_t g_spdk_sge_size = 4096;
+static uint32_t g_spdk_bit_bucket_data_len = 0;
+static uint32_t g_spdk_pract_flag;
+static uint32_t g_spdk_prchk_flags;
+static uint32_t g_spdk_md_per_io_size = 4096;
+static uint16_t g_spdk_apptag;
+static uint16_t g_spdk_apptag_mask;
+
+struct spdk_fio_options {
+ void *pad; /* off1 used in option descriptions may not be 0 */
+ int enable_wrr;
+ int arbitration_burst;
+ int low_weight;
+ int medium_weight;
+ int high_weight;
+ int wrr_priority;
+ int mem_size;
+ int shm_id;
+ int enable_sgl;
+ int sge_size;
+ int bit_bucket_data_len;
+ char *hostnqn;
+ int pi_act;
+ char *pi_chk;
+ int md_per_io_size;
+ int apptag;
+ int apptag_mask;
+ char *digest_enable;
+ int enable_vmd;
+};
+
+struct spdk_fio_request {
+ struct io_u *io;
+ /** Offset in current iovec, fio only uses 1 vector */
+ uint32_t iov_offset;
+
+ /** Amount of data used for Bit Bucket SGL */
+ uint32_t bit_bucket_data_len;
+
+ /** Context for NVMe PI */
+ struct spdk_dif_ctx dif_ctx;
+ /** Separate metadata buffer pointer */
+ void *md_buf;
+
+ struct spdk_fio_thread *fio_thread;
+ struct spdk_fio_qpair *fio_qpair;
+};
+
+struct spdk_fio_ctrlr {
+ struct spdk_nvme_transport_id tr_id;
+ struct spdk_nvme_ctrlr_opts opts;
+ struct spdk_nvme_ctrlr *ctrlr;
+ struct spdk_fio_ctrlr *next;
+};
+
+static struct spdk_fio_ctrlr *g_ctrlr;
+static int g_td_count;
+static pthread_t g_ctrlr_thread_id = 0;
+static pthread_mutex_t g_mutex = PTHREAD_MUTEX_INITIALIZER;
+static bool g_error;
+
+struct spdk_fio_qpair {
+ struct fio_file *f;
+ struct spdk_nvme_qpair *qpair;
+ struct spdk_nvme_ns *ns;
+ uint32_t io_flags;
+ bool nvme_pi_enabled;
+ /* True for DIF and false for DIX, and this is valid only if nvme_pi_enabled is true. */
+ bool extended_lba;
+ /* True for protection info transferred at start of metadata,
+ * false for protection info transferred at end of metadata, and
+ * this is valid only if nvme_pi_enabled is true.
+ */
+ bool md_start;
+ struct spdk_fio_qpair *next;
+ struct spdk_fio_ctrlr *fio_ctrlr;
+};
+
+struct spdk_fio_thread {
+ struct thread_data *td;
+
+ struct spdk_fio_qpair *fio_qpair;
+ struct spdk_fio_qpair *fio_qpair_current; /* the current fio_qpair to be handled. */
+
+ struct io_u **iocq; /* io completion queue */
+ unsigned int iocq_count; /* number of iocq entries filled by last getevents */
+ unsigned int iocq_size; /* number of iocq entries allocated */
+ struct fio_file *current_f; /* fio_file given by user */
+
+};
+
+static void *
+spdk_fio_poll_ctrlrs(void *arg)
+{
+ struct spdk_fio_ctrlr *fio_ctrlr;
+ int oldstate;
+ int rc;
+
+ /* Loop until the thread is cancelled */
+ while (true) {
+ rc = pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate);
+ if (rc != 0) {
+ SPDK_ERRLOG("Unable to set cancel state disabled on g_init_thread (%d): %s\n",
+ rc, spdk_strerror(rc));
+ }
+
+ pthread_mutex_lock(&g_mutex);
+ fio_ctrlr = g_ctrlr;
+
+ while (fio_ctrlr) {
+ spdk_nvme_ctrlr_process_admin_completions(fio_ctrlr->ctrlr);
+ fio_ctrlr = fio_ctrlr->next;
+ }
+
+ pthread_mutex_unlock(&g_mutex);
+
+ rc = pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &oldstate);
+ if (rc != 0) {
+ SPDK_ERRLOG("Unable to set cancel state enabled on g_init_thread (%d): %s\n",
+ rc, spdk_strerror(rc));
+ }
+
+ /* This is a pthread cancellation point and cannot be removed. */
+ sleep(1);
+ }
+
+ return NULL;
+}
+
+static bool
+probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
+ struct spdk_nvme_ctrlr_opts *opts)
+{
+ struct thread_data *td = cb_ctx;
+ struct spdk_fio_options *fio_options = td->eo;
+
+ if (fio_options->hostnqn) {
+ snprintf(opts->hostnqn, sizeof(opts->hostnqn), "%s", fio_options->hostnqn);
+ }
+
+ if (fio_options->enable_wrr) {
+ opts->arb_mechanism = SPDK_NVME_CC_AMS_WRR;
+ opts->arbitration_burst = fio_options->arbitration_burst;
+ opts->low_priority_weight = fio_options->low_weight;
+ opts->medium_priority_weight = fio_options->medium_weight;
+ opts->high_priority_weight = fio_options->high_weight;
+ }
+
+ if (fio_options->digest_enable) {
+ if (strcasecmp(fio_options->digest_enable, "HEADER") == 0) {
+ opts->header_digest = true;
+ } else if (strcasecmp(fio_options->digest_enable, "DATA") == 0) {
+ opts->data_digest = true;
+ } else if (strcasecmp(fio_options->digest_enable, "BOTH") == 0) {
+ opts->header_digest = true;
+ opts->data_digest = true;
+ }
+ }
+
+ return true;
+}
+
+static struct spdk_fio_ctrlr *
+get_fio_ctrlr(const struct spdk_nvme_transport_id *trid)
+{
+ struct spdk_fio_ctrlr *fio_ctrlr = g_ctrlr;
+ while (fio_ctrlr) {
+ if (spdk_nvme_transport_id_compare(trid, &fio_ctrlr->tr_id) == 0) {
+ return fio_ctrlr;
+ }
+
+ fio_ctrlr = fio_ctrlr->next;
+ }
+
+ return NULL;
+}
+
+static void
+attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
+ struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts)
+{
+ struct thread_data *td = cb_ctx;
+ struct spdk_fio_thread *fio_thread = td->io_ops_data;
+ struct spdk_nvme_io_qpair_opts qpopts;
+ struct spdk_fio_ctrlr *fio_ctrlr;
+ struct spdk_fio_qpair *fio_qpair;
+ struct spdk_nvme_ns *ns;
+ const struct spdk_nvme_ns_data *nsdata;
+ struct fio_file *f = fio_thread->current_f;
+ uint32_t ns_id;
+ char *p;
+ long int tmp;
+ struct spdk_fio_options *fio_options = td->eo;
+
+ p = strstr(f->file_name, "ns=");
+ if (p != NULL) {
+ tmp = spdk_strtol(p + 3, 10);
+ if (tmp <= 0) {
+ SPDK_ERRLOG("namespace id should be >=1, but was invalid: %ld\n", tmp);
+ g_error = true;
+ return;
+ }
+ ns_id = (uint32_t)tmp;
+ } else {
+ ns_id = spdk_nvme_ctrlr_get_first_active_ns(ctrlr);
+ if (ns_id == 0) {
+ /* The ctrlr has no active namespaces and we didn't specify any so nothing to do. */
+ return;
+ }
+ }
+
+ pthread_mutex_lock(&g_mutex);
+ fio_ctrlr = get_fio_ctrlr(trid);
+ /* it is a new ctrlr and needs to be added */
+ if (!fio_ctrlr) {
+ /* Create an fio_ctrlr and add it to the list */
+ fio_ctrlr = calloc(1, sizeof(*fio_ctrlr));
+ if (!fio_ctrlr) {
+ SPDK_ERRLOG("Cannot allocate space for fio_ctrlr\n");
+ g_error = true;
+ pthread_mutex_unlock(&g_mutex);
+ return;
+ }
+ fio_ctrlr->opts = *opts;
+ fio_ctrlr->ctrlr = ctrlr;
+ fio_ctrlr->tr_id = *trid;
+ fio_ctrlr->next = g_ctrlr;
+ g_ctrlr = fio_ctrlr;
+ }
+ pthread_mutex_unlock(&g_mutex);
+
+ ns = spdk_nvme_ctrlr_get_ns(fio_ctrlr->ctrlr, ns_id);
+ if (ns == NULL) {
+ SPDK_ERRLOG("Cannot get namespace by ns_id=%d\n", ns_id);
+ g_error = true;
+ return;
+ }
+
+ if (!spdk_nvme_ns_is_active(ns)) {
+ SPDK_ERRLOG("Inactive namespace by ns_id=%d\n", ns_id);
+ g_error = true;
+ return;
+ }
+ nsdata = spdk_nvme_ns_get_data(ns);
+
+ fio_qpair = fio_thread->fio_qpair;
+ while (fio_qpair != NULL) {
+ if ((fio_qpair->f == f) ||
+ ((spdk_nvme_transport_id_compare(trid, &fio_qpair->fio_ctrlr->tr_id) == 0) &&
+ (spdk_nvme_ns_get_id(fio_qpair->ns) == ns_id))) {
+ /* Not the error case. Avoid duplicated connection */
+ return;
+ }
+ fio_qpair = fio_qpair->next;
+ }
+
+ /* create a new qpair */
+ fio_qpair = calloc(1, sizeof(*fio_qpair));
+ if (!fio_qpair) {
+ g_error = true;
+ SPDK_ERRLOG("Cannot allocate space for fio_qpair\n");
+ return;
+ }
+
+ spdk_nvme_ctrlr_get_default_io_qpair_opts(fio_ctrlr->ctrlr, &qpopts, sizeof(qpopts));
+ qpopts.delay_cmd_submit = true;
+ if (fio_options->enable_wrr) {
+ qpopts.qprio = fio_options->wrr_priority;
+ }
+
+ fio_qpair->qpair = spdk_nvme_ctrlr_alloc_io_qpair(fio_ctrlr->ctrlr, &qpopts, sizeof(qpopts));
+ if (!fio_qpair->qpair) {
+ SPDK_ERRLOG("Cannot allocate nvme io_qpair any more\n");
+ g_error = true;
+ free(fio_qpair);
+ return;
+ }
+
+ fio_qpair->ns = ns;
+ fio_qpair->f = f;
+ fio_qpair->fio_ctrlr = fio_ctrlr;
+ fio_qpair->next = fio_thread->fio_qpair;
+ fio_thread->fio_qpair = fio_qpair;
+
+ if (spdk_nvme_ns_get_flags(ns) & SPDK_NVME_NS_DPS_PI_SUPPORTED) {
+ assert(spdk_nvme_ns_get_pi_type(ns) != SPDK_NVME_FMT_NVM_PROTECTION_DISABLE);
+ fio_qpair->io_flags = g_spdk_pract_flag | g_spdk_prchk_flags;
+ fio_qpair->nvme_pi_enabled = true;
+ fio_qpair->md_start = nsdata->dps.md_start;
+ fio_qpair->extended_lba = spdk_nvme_ns_supports_extended_lba(ns);
+ fprintf(stdout, "PI type%u enabled with %s\n", spdk_nvme_ns_get_pi_type(ns),
+ fio_qpair->extended_lba ? "extended lba" : "separate metadata");
+ }
+
+ f->real_file_size = spdk_nvme_ns_get_size(fio_qpair->ns);
+ if (f->real_file_size <= 0) {
+ g_error = true;
+ SPDK_ERRLOG("Cannot get namespace size by ns=%p\n", ns);
+ return;
+ }
+
+ f->filetype = FIO_TYPE_BLOCK;
+ fio_file_set_size_known(f);
+}
+
+static void parse_prchk_flags(const char *prchk_str)
+{
+ if (!prchk_str) {
+ return;
+ }
+
+ if (strstr(prchk_str, "GUARD") != NULL) {
+ g_spdk_prchk_flags = SPDK_NVME_IO_FLAGS_PRCHK_GUARD;
+ }
+ if (strstr(prchk_str, "REFTAG") != NULL) {
+ g_spdk_prchk_flags |= SPDK_NVME_IO_FLAGS_PRCHK_REFTAG;
+ }
+ if (strstr(prchk_str, "APPTAG") != NULL) {
+ g_spdk_prchk_flags |= SPDK_NVME_IO_FLAGS_PRCHK_APPTAG;
+ }
+}
+
+static void parse_pract_flag(int pract)
+{
+ if (pract == 1) {
+ g_spdk_pract_flag = SPDK_NVME_IO_FLAGS_PRACT;
+ } else {
+ g_spdk_pract_flag = 0;
+ }
+}
+
+/* Called once at initialization. This is responsible for gathering the size of
+ * each "file", which in our case are in the form
+ * 'key=value [key=value] ... ns=value'
+ * For example, For local PCIe NVMe device - 'trtype=PCIe traddr=0000.04.00.0 ns=1'
+ * For remote exported by NVMe-oF target, 'trtype=RDMA adrfam=IPv4 traddr=192.168.100.8 trsvcid=4420 ns=1' */
+static int spdk_fio_setup(struct thread_data *td)
+{
+ struct spdk_fio_thread *fio_thread;
+ struct spdk_fio_options *fio_options = td->eo;
+ struct spdk_env_opts opts;
+ struct fio_file *f;
+ char *p;
+ int rc = 0;
+ struct spdk_nvme_transport_id trid;
+ struct spdk_fio_ctrlr *fio_ctrlr;
+ char *trid_info;
+ unsigned int i;
+
+ /* we might be running in a daemonized FIO instance where standard
+ * input and output were closed and fds 0, 1, and 2 are reused
+ * for something important by FIO. We can't ensure we won't print
+ * anything (and so will our dependencies, e.g. DPDK), so abort early.
+ * (is_backend is an fio global variable)
+ */
+ if (is_backend) {
+ char buf[1024];
+ snprintf(buf, sizeof(buf),
+ "SPDK FIO plugin won't work with daemonized FIO server.");
+ fio_server_text_output(FIO_LOG_ERR, buf, sizeof(buf));
+ return -1;
+ }
+
+ if (!td->o.use_thread) {
+ log_err("spdk: must set thread=1 when using spdk plugin\n");
+ return 1;
+ }
+
+ pthread_mutex_lock(&g_mutex);
+
+ fio_thread = calloc(1, sizeof(*fio_thread));
+ assert(fio_thread != NULL);
+
+ td->io_ops_data = fio_thread;
+ fio_thread->td = td;
+
+ fio_thread->iocq_size = td->o.iodepth;
+ fio_thread->iocq = calloc(fio_thread->iocq_size, sizeof(struct io_u *));
+ assert(fio_thread->iocq != NULL);
+
+ if (!g_spdk_env_initialized) {
+ spdk_env_opts_init(&opts);
+ opts.name = "fio";
+ opts.mem_size = fio_options->mem_size;
+ opts.shm_id = fio_options->shm_id;
+ g_spdk_enable_sgl = fio_options->enable_sgl;
+ g_spdk_sge_size = fio_options->sge_size;
+ g_spdk_bit_bucket_data_len = fio_options->bit_bucket_data_len;
+ parse_pract_flag(fio_options->pi_act);
+ g_spdk_md_per_io_size = spdk_max(fio_options->md_per_io_size, 4096);
+ g_spdk_apptag = (uint16_t)fio_options->apptag;
+ g_spdk_apptag_mask = (uint16_t)fio_options->apptag_mask;
+ parse_prchk_flags(fio_options->pi_chk);
+ if (spdk_env_init(&opts) < 0) {
+ SPDK_ERRLOG("Unable to initialize SPDK env\n");
+ free(fio_thread->iocq);
+ free(fio_thread);
+ fio_thread = NULL;
+ pthread_mutex_unlock(&g_mutex);
+ return 1;
+ }
+ g_spdk_env_initialized = true;
+ spdk_unaffinitize_thread();
+
+ /* Spawn a thread to continue polling the controllers */
+ rc = pthread_create(&g_ctrlr_thread_id, NULL, &spdk_fio_poll_ctrlrs, NULL);
+ if (rc != 0) {
+ SPDK_ERRLOG("Unable to spawn a thread to poll admin queues. They won't be polled.\n");
+ }
+
+ if (fio_options->enable_vmd && spdk_vmd_init()) {
+ SPDK_ERRLOG("Failed to initialize VMD. Some NVMe devices can be unavailable.\n");
+ }
+ }
+ pthread_mutex_unlock(&g_mutex);
+
+ for_each_file(td, f, i) {
+ memset(&trid, 0, sizeof(trid));
+
+ trid.trtype = SPDK_NVME_TRANSPORT_PCIE;
+
+ p = strstr(f->file_name, " ns=");
+ if (p != NULL) {
+ trid_info = strndup(f->file_name, p - f->file_name);
+ } else {
+ trid_info = strndup(f->file_name, strlen(f->file_name));
+ }
+
+ if (!trid_info) {
+ SPDK_ERRLOG("Failed to allocate space for trid_info\n");
+ continue;
+ }
+
+ rc = spdk_nvme_transport_id_parse(&trid, trid_info);
+ if (rc < 0) {
+ SPDK_ERRLOG("Failed to parse given str: %s\n", trid_info);
+ free(trid_info);
+ continue;
+ }
+ free(trid_info);
+
+ if (trid.trtype == SPDK_NVME_TRANSPORT_PCIE) {
+ struct spdk_pci_addr pci_addr;
+ if (spdk_pci_addr_parse(&pci_addr, trid.traddr) < 0) {
+ SPDK_ERRLOG("Invalid traddr=%s\n", trid.traddr);
+ continue;
+ }
+ spdk_pci_addr_fmt(trid.traddr, sizeof(trid.traddr), &pci_addr);
+ } else {
+ if (trid.subnqn[0] == '\0') {
+ snprintf(trid.subnqn, sizeof(trid.subnqn), "%s",
+ SPDK_NVMF_DISCOVERY_NQN);
+ }
+ }
+
+ fio_thread->current_f = f;
+
+ pthread_mutex_lock(&g_mutex);
+ fio_ctrlr = get_fio_ctrlr(&trid);
+ pthread_mutex_unlock(&g_mutex);
+ if (fio_ctrlr) {
+ attach_cb(td, &trid, fio_ctrlr->ctrlr, &fio_ctrlr->opts);
+ } else {
+ /* Enumerate all of the controllers */
+ if (spdk_nvme_probe(&trid, td, probe_cb, attach_cb, NULL) != 0) {
+ SPDK_ERRLOG("spdk_nvme_probe() failed\n");
+ continue;
+ }
+ }
+
+ if (g_error) {
+ log_err("Failed to initialize spdk fio plugin\n");
+ rc = 1;
+ break;
+ }
+ }
+
+ pthread_mutex_lock(&g_mutex);
+ g_td_count++;
+ pthread_mutex_unlock(&g_mutex);
+
+ return rc;
+}
+
+static int spdk_fio_open(struct thread_data *td, struct fio_file *f)
+{
+ return 0;
+}
+
+static int spdk_fio_close(struct thread_data *td, struct fio_file *f)
+{
+ return 0;
+}
+
+static int spdk_fio_iomem_alloc(struct thread_data *td, size_t total_mem)
+{
+ td->orig_buffer = spdk_dma_zmalloc(total_mem, NVME_IO_ALIGN, NULL);
+ return td->orig_buffer == NULL;
+}
+
+static void spdk_fio_iomem_free(struct thread_data *td)
+{
+ spdk_dma_free(td->orig_buffer);
+}
+
+static int spdk_fio_io_u_init(struct thread_data *td, struct io_u *io_u)
+{
+ struct spdk_fio_thread *fio_thread = td->io_ops_data;
+ struct spdk_fio_request *fio_req;
+
+ io_u->engine_data = NULL;
+
+ fio_req = calloc(1, sizeof(*fio_req));
+ if (fio_req == NULL) {
+ return 1;
+ }
+
+ fio_req->md_buf = spdk_dma_zmalloc(g_spdk_md_per_io_size, NVME_IO_ALIGN, NULL);
+ if (fio_req->md_buf == NULL) {
+ fprintf(stderr, "Allocate %u metadata failed\n", g_spdk_md_per_io_size);
+ free(fio_req);
+ return 1;
+ }
+
+ fio_req->io = io_u;
+ fio_req->fio_thread = fio_thread;
+
+ io_u->engine_data = fio_req;
+
+ return 0;
+}
+
+static void spdk_fio_io_u_free(struct thread_data *td, struct io_u *io_u)
+{
+ struct spdk_fio_request *fio_req = io_u->engine_data;
+
+ if (fio_req) {
+ assert(fio_req->io == io_u);
+ spdk_dma_free(fio_req->md_buf);
+ free(fio_req);
+ io_u->engine_data = NULL;
+ }
+}
+
+static int
+fio_extended_lba_setup_pi(struct spdk_fio_qpair *fio_qpair, struct io_u *io_u)
+{
+ struct spdk_nvme_ns *ns = fio_qpair->ns;
+ struct spdk_fio_request *fio_req = io_u->engine_data;
+ uint32_t md_size, extended_lba_size, lba_count;
+ uint64_t lba;
+ struct iovec iov;
+ int rc;
+
+ /* Set appmask and apptag when PRACT is enabled */
+ if (fio_qpair->io_flags & SPDK_NVME_IO_FLAGS_PRACT) {
+ fio_req->dif_ctx.apptag_mask = g_spdk_apptag_mask;
+ fio_req->dif_ctx.app_tag = g_spdk_apptag;
+ return 0;
+ }
+
+ extended_lba_size = spdk_nvme_ns_get_extended_sector_size(ns);
+ md_size = spdk_nvme_ns_get_md_size(ns);
+ lba = io_u->offset / extended_lba_size;
+ lba_count = io_u->xfer_buflen / extended_lba_size;
+
+ rc = spdk_dif_ctx_init(&fio_req->dif_ctx, extended_lba_size, md_size,
+ true, fio_qpair->md_start,
+ (enum spdk_dif_type)spdk_nvme_ns_get_pi_type(ns),
+ fio_qpair->io_flags, lba, g_spdk_apptag_mask, g_spdk_apptag, 0, 0);
+ if (rc != 0) {
+ fprintf(stderr, "Initialization of DIF context failed\n");
+ return rc;
+ }
+
+ if (io_u->ddir != DDIR_WRITE) {
+ return 0;
+ }
+
+ iov.iov_base = io_u->buf;
+ iov.iov_len = io_u->xfer_buflen;
+ rc = spdk_dif_generate(&iov, 1, lba_count, &fio_req->dif_ctx);
+ if (rc != 0) {
+ fprintf(stderr, "Generation of DIF failed\n");
+ }
+
+ return rc;
+}
+
+static int
+fio_separate_md_setup_pi(struct spdk_fio_qpair *fio_qpair, struct io_u *io_u)
+{
+ struct spdk_nvme_ns *ns = fio_qpair->ns;
+ struct spdk_fio_request *fio_req = io_u->engine_data;
+ uint32_t md_size, block_size, lba_count;
+ uint64_t lba;
+ struct iovec iov, md_iov;
+ int rc;
+
+ /* Set appmask and apptag when PRACT is enabled */
+ if (fio_qpair->io_flags & SPDK_NVME_IO_FLAGS_PRACT) {
+ fio_req->dif_ctx.apptag_mask = g_spdk_apptag_mask;
+ fio_req->dif_ctx.app_tag = g_spdk_apptag;
+ return 0;
+ }
+
+ block_size = spdk_nvme_ns_get_sector_size(ns);
+ md_size = spdk_nvme_ns_get_md_size(ns);
+ lba = io_u->offset / block_size;
+ lba_count = io_u->xfer_buflen / block_size;
+
+ rc = spdk_dif_ctx_init(&fio_req->dif_ctx, block_size, md_size,
+ false, fio_qpair->md_start,
+ (enum spdk_dif_type)spdk_nvme_ns_get_pi_type(ns),
+ fio_qpair->io_flags, lba, g_spdk_apptag_mask, g_spdk_apptag, 0, 0);
+ if (rc != 0) {
+ fprintf(stderr, "Initialization of DIF context failed\n");
+ return rc;
+ }
+
+ if (io_u->ddir != DDIR_WRITE) {
+ return 0;
+ }
+
+ iov.iov_base = io_u->buf;
+ iov.iov_len = io_u->xfer_buflen;
+ md_iov.iov_base = fio_req->md_buf;
+ md_iov.iov_len = spdk_min(md_size * lba_count, g_spdk_md_per_io_size);
+ rc = spdk_dix_generate(&iov, 1, &md_iov, lba_count, &fio_req->dif_ctx);
+ if (rc < 0) {
+ fprintf(stderr, "Generation of DIX failed\n");
+ }
+
+ return rc;
+}
+
+static int
+fio_extended_lba_verify_pi(struct spdk_fio_qpair *fio_qpair, struct io_u *io_u)
+{
+ struct spdk_nvme_ns *ns = fio_qpair->ns;
+ struct spdk_fio_request *fio_req = io_u->engine_data;
+ uint32_t lba_count;
+ struct iovec iov;
+ struct spdk_dif_error err_blk = {};
+ int rc;
+
+ /* Do nothing when PRACT is enabled */
+ if (fio_qpair->io_flags & SPDK_NVME_IO_FLAGS_PRACT) {
+ return 0;
+ }
+
+ iov.iov_base = io_u->buf;
+ iov.iov_len = io_u->xfer_buflen;
+ lba_count = io_u->xfer_buflen / spdk_nvme_ns_get_extended_sector_size(ns);
+
+ rc = spdk_dif_verify(&iov, 1, lba_count, &fio_req->dif_ctx, &err_blk);
+ if (rc != 0) {
+ fprintf(stderr, "DIF error detected. type=%d, offset=%" PRIu32 "\n",
+ err_blk.err_type, err_blk.err_offset);
+ }
+
+ return rc;
+}
+
+static int
+fio_separate_md_verify_pi(struct spdk_fio_qpair *fio_qpair, struct io_u *io_u)
+{
+ struct spdk_nvme_ns *ns = fio_qpair->ns;
+ struct spdk_fio_request *fio_req = io_u->engine_data;
+ uint32_t md_size, lba_count;
+ struct iovec iov, md_iov;
+ struct spdk_dif_error err_blk = {};
+ int rc;
+
+ /* Do nothing when PRACT is enabled */
+ if (fio_qpair->io_flags & SPDK_NVME_IO_FLAGS_PRACT) {
+ return 0;
+ }
+
+ iov.iov_base = io_u->buf;
+ iov.iov_len = io_u->xfer_buflen;
+ lba_count = io_u->xfer_buflen / spdk_nvme_ns_get_sector_size(ns);
+ md_size = spdk_nvme_ns_get_md_size(ns);
+ md_iov.iov_base = fio_req->md_buf;
+ md_iov.iov_len = spdk_min(md_size * lba_count, g_spdk_md_per_io_size);
+
+ rc = spdk_dix_verify(&iov, 1, &md_iov, lba_count, &fio_req->dif_ctx, &err_blk);
+ if (rc != 0) {
+ fprintf(stderr, "DIX error detected. type=%d, offset=%" PRIu32 "\n",
+ err_blk.err_type, err_blk.err_offset);
+ }
+
+ return rc;
+}
+
+static void spdk_fio_completion_cb(void *ctx, const struct spdk_nvme_cpl *cpl)
+{
+ struct spdk_fio_request *fio_req = ctx;
+ struct spdk_fio_thread *fio_thread = fio_req->fio_thread;
+ struct spdk_fio_qpair *fio_qpair = fio_req->fio_qpair;
+ int rc;
+
+ if (fio_qpair->nvme_pi_enabled && fio_req->io->ddir == DDIR_READ) {
+ if (fio_qpair->extended_lba) {
+ rc = fio_extended_lba_verify_pi(fio_qpair, fio_req->io);
+ } else {
+ rc = fio_separate_md_verify_pi(fio_qpair, fio_req->io);
+ }
+ if (rc != 0) {
+ fio_req->io->error = abs(rc);
+ }
+ }
+
+ assert(fio_thread->iocq_count < fio_thread->iocq_size);
+ fio_thread->iocq[fio_thread->iocq_count++] = fio_req->io;
+}
+
+static void
+spdk_nvme_io_reset_sgl(void *ref, uint32_t sgl_offset)
+{
+ struct spdk_fio_request *fio_req = (struct spdk_fio_request *)ref;
+
+ fio_req->iov_offset = sgl_offset;
+ fio_req->bit_bucket_data_len = 0;
+}
+
+static int
+spdk_nvme_io_next_sge(void *ref, void **address, uint32_t *length)
+{
+ struct spdk_fio_request *fio_req = (struct spdk_fio_request *)ref;
+ struct io_u *io_u = fio_req->io;
+ uint32_t iov_len;
+ uint32_t bit_bucket_len;
+
+ *address = io_u->buf;
+
+ if (fio_req->iov_offset) {
+ assert(fio_req->iov_offset <= io_u->xfer_buflen);
+ *address += fio_req->iov_offset;
+ }
+
+ iov_len = io_u->xfer_buflen - fio_req->iov_offset;
+ if (iov_len > g_spdk_sge_size) {
+ iov_len = g_spdk_sge_size;
+ }
+
+ if ((fio_req->bit_bucket_data_len < g_spdk_bit_bucket_data_len) && (io_u->ddir == DDIR_READ)) {
+ assert(g_spdk_bit_bucket_data_len < io_u->xfer_buflen);
+ *address = (void *)UINT64_MAX;
+ bit_bucket_len = g_spdk_bit_bucket_data_len - fio_req->bit_bucket_data_len;
+ if (iov_len > bit_bucket_len) {
+ iov_len = bit_bucket_len;
+ }
+ fio_req->bit_bucket_data_len += iov_len;
+ }
+
+ fio_req->iov_offset += iov_len;
+ *length = iov_len;
+
+ return 0;
+}
+
+#if FIO_IOOPS_VERSION >= 24
+typedef enum fio_q_status fio_q_status_t;
+#else
+typedef int fio_q_status_t;
+#endif
+
+static fio_q_status_t
+spdk_fio_queue(struct thread_data *td, struct io_u *io_u)
+{
+ int rc = 1;
+ struct spdk_fio_thread *fio_thread = td->io_ops_data;
+ struct spdk_fio_request *fio_req = io_u->engine_data;
+ struct spdk_fio_qpair *fio_qpair;
+ struct spdk_nvme_ns *ns = NULL;
+ void *md_buf = NULL;
+ struct spdk_dif_ctx *dif_ctx = &fio_req->dif_ctx;
+ uint32_t block_size;
+ uint64_t lba;
+ uint32_t lba_count;
+
+ /* Find the namespace that corresponds to the file in the io_u */
+ fio_qpair = fio_thread->fio_qpair;
+ while (fio_qpair != NULL) {
+ if (fio_qpair->f == io_u->file) {
+ ns = fio_qpair->ns;
+ break;
+ }
+ fio_qpair = fio_qpair->next;
+ }
+ if (fio_qpair == NULL || ns == NULL) {
+ return -ENXIO;
+ }
+ if (fio_qpair->nvme_pi_enabled && !fio_qpair->extended_lba) {
+ md_buf = fio_req->md_buf;
+ }
+ fio_req->fio_qpair = fio_qpair;
+
+ block_size = spdk_nvme_ns_get_extended_sector_size(ns);
+ if ((fio_qpair->io_flags & g_spdk_pract_flag) && (spdk_nvme_ns_get_md_size(ns) == 8)) {
+ /* If metadata size = 8 bytes, PI is stripped (read) or inserted (write), and
+ * so reduce metadata size from block size. (If metadata size > 8 bytes, PI
+ * is passed (read) or replaced (write). So block size is not necessary to
+ * change.)
+ */
+ block_size = spdk_nvme_ns_get_sector_size(ns);
+ }
+
+ lba = io_u->offset / block_size;
+ lba_count = io_u->xfer_buflen / block_size;
+
+ /* TODO: considering situations that fio will randomize and verify io_u */
+ if (fio_qpair->nvme_pi_enabled) {
+ if (fio_qpair->extended_lba) {
+ rc = fio_extended_lba_setup_pi(fio_qpair, io_u);
+ } else {
+ rc = fio_separate_md_setup_pi(fio_qpair, io_u);
+ }
+ if (rc < 0) {
+ io_u->error = -rc;
+ return FIO_Q_COMPLETED;
+ }
+ }
+
+ switch (io_u->ddir) {
+ case DDIR_READ:
+ if (!g_spdk_enable_sgl) {
+ rc = spdk_nvme_ns_cmd_read_with_md(ns, fio_qpair->qpair, io_u->buf, md_buf, lba, lba_count,
+ spdk_fio_completion_cb, fio_req,
+ fio_qpair->io_flags, dif_ctx->apptag_mask, dif_ctx->app_tag);
+ } else {
+ rc = spdk_nvme_ns_cmd_readv_with_md(ns, fio_qpair->qpair, lba,
+ lba_count, spdk_fio_completion_cb, fio_req, fio_qpair->io_flags,
+ spdk_nvme_io_reset_sgl, spdk_nvme_io_next_sge, md_buf,
+ dif_ctx->apptag_mask, dif_ctx->app_tag);
+ }
+ break;
+ case DDIR_WRITE:
+ if (!g_spdk_enable_sgl) {
+ rc = spdk_nvme_ns_cmd_write_with_md(ns, fio_qpair->qpair, io_u->buf, md_buf, lba,
+ lba_count,
+ spdk_fio_completion_cb, fio_req,
+ fio_qpair->io_flags, dif_ctx->apptag_mask, dif_ctx->app_tag);
+ } else {
+ rc = spdk_nvme_ns_cmd_writev_with_md(ns, fio_qpair->qpair, lba,
+ lba_count, spdk_fio_completion_cb, fio_req, fio_qpair->io_flags,
+ spdk_nvme_io_reset_sgl, spdk_nvme_io_next_sge, md_buf,
+ dif_ctx->apptag_mask, dif_ctx->app_tag);
+ }
+ break;
+ default:
+ assert(false);
+ break;
+ }
+
+ /* NVMe read/write functions return -ENOMEM if there are no free requests. */
+ if (rc == -ENOMEM) {
+ return FIO_Q_BUSY;
+ }
+
+ if (rc != 0) {
+ io_u->error = abs(rc);
+ return FIO_Q_COMPLETED;
+ }
+
+ return FIO_Q_QUEUED;
+}
+
+static struct io_u *spdk_fio_event(struct thread_data *td, int event)
+{
+ struct spdk_fio_thread *fio_thread = td->io_ops_data;
+
+ assert(event >= 0);
+ assert((unsigned)event < fio_thread->iocq_count);
+ return fio_thread->iocq[event];
+}
+
+static int spdk_fio_getevents(struct thread_data *td, unsigned int min,
+ unsigned int max, const struct timespec *t)
+{
+ struct spdk_fio_thread *fio_thread = td->io_ops_data;
+ struct spdk_fio_qpair *fio_qpair = NULL;
+ struct timespec t0, t1;
+ uint64_t timeout = 0;
+
+ if (t) {
+ timeout = t->tv_sec * 1000000000L + t->tv_nsec;
+ clock_gettime(CLOCK_MONOTONIC_RAW, &t0);
+ }
+
+ fio_thread->iocq_count = 0;
+
+ /* fetch the next qpair */
+ if (fio_thread->fio_qpair_current) {
+ fio_qpair = fio_thread->fio_qpair_current->next;
+ }
+
+ for (;;) {
+ if (fio_qpair == NULL) {
+ fio_qpair = fio_thread->fio_qpair;
+ }
+
+ while (fio_qpair != NULL) {
+ spdk_nvme_qpair_process_completions(fio_qpair->qpair, max - fio_thread->iocq_count);
+
+ if (fio_thread->iocq_count >= min) {
+ /* reset the currrent handling qpair */
+ fio_thread->fio_qpair_current = fio_qpair;
+ return fio_thread->iocq_count;
+ }
+
+ fio_qpair = fio_qpair->next;
+ }
+
+ if (t) {
+ uint64_t elapse;
+
+ clock_gettime(CLOCK_MONOTONIC_RAW, &t1);
+ elapse = ((t1.tv_sec - t0.tv_sec) * 1000000000L)
+ + t1.tv_nsec - t0.tv_nsec;
+ if (elapse > timeout) {
+ break;
+ }
+ }
+ }
+
+ /* reset the currrent handling qpair */
+ fio_thread->fio_qpair_current = fio_qpair;
+ return fio_thread->iocq_count;
+}
+
+static int spdk_fio_invalidate(struct thread_data *td, struct fio_file *f)
+{
+ /* TODO: This should probably send a flush to the device, but for now just return successful. */
+ return 0;
+}
+
+static void spdk_fio_cleanup(struct thread_data *td)
+{
+ struct spdk_fio_thread *fio_thread = td->io_ops_data;
+ struct spdk_fio_qpair *fio_qpair, *fio_qpair_tmp;
+ struct spdk_fio_options *fio_options = td->eo;
+
+ fio_qpair = fio_thread->fio_qpair;
+ while (fio_qpair != NULL) {
+ spdk_nvme_ctrlr_free_io_qpair(fio_qpair->qpair);
+ fio_qpair_tmp = fio_qpair->next;
+ free(fio_qpair);
+ fio_qpair = fio_qpair_tmp;
+ }
+
+ free(fio_thread->iocq);
+ free(fio_thread);
+
+ pthread_mutex_lock(&g_mutex);
+ g_td_count--;
+ if (g_td_count == 0) {
+ struct spdk_fio_ctrlr *fio_ctrlr, *fio_ctrlr_tmp;
+
+ fio_ctrlr = g_ctrlr;
+ while (fio_ctrlr != NULL) {
+ spdk_nvme_detach(fio_ctrlr->ctrlr);
+ fio_ctrlr_tmp = fio_ctrlr->next;
+ free(fio_ctrlr);
+ fio_ctrlr = fio_ctrlr_tmp;
+ }
+ g_ctrlr = NULL;
+
+ if (fio_options->enable_vmd) {
+ spdk_vmd_fini();
+ }
+ }
+ pthread_mutex_unlock(&g_mutex);
+ if (!g_ctrlr) {
+ if (pthread_cancel(g_ctrlr_thread_id) == 0) {
+ pthread_join(g_ctrlr_thread_id, NULL);
+ }
+ }
+}
+
+/* This function enables addition of SPDK parameters to the fio config
+ * Adding new parameters by defining them here and defining a callback
+ * function to read the parameter value. */
+static struct fio_option options[] = {
+ {
+ .name = "enable_wrr",
+ .lname = "Enable weighted round robin (WRR) for IO submission queues",
+ .type = FIO_OPT_INT,
+ .off1 = offsetof(struct spdk_fio_options, enable_wrr),
+ .def = "0",
+ .help = "Enable weighted round robin (WRR) for IO submission queues",
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_INVALID,
+ },
+ {
+ .name = "arbitration_burst",
+ .lname = "Arbitration Burst",
+ .type = FIO_OPT_INT,
+ .off1 = offsetof(struct spdk_fio_options, arbitration_burst),
+ .def = "0",
+ .help = "Arbitration Burst used for WRR (valid range from 0 - 7)",
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_INVALID,
+ },
+ {
+ .name = "low_weight",
+ .lname = "low_weight for WRR",
+ .type = FIO_OPT_INT,
+ .off1 = offsetof(struct spdk_fio_options, low_weight),
+ .def = "0",
+ .help = "low_weight used for WRR (valid range from 0 - 255)",
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_INVALID,
+ },
+ {
+ .name = "medium_weight",
+ .lname = "medium_weight for WRR",
+ .type = FIO_OPT_INT,
+ .off1 = offsetof(struct spdk_fio_options, medium_weight),
+ .def = "0",
+ .help = "medium weight used for WRR (valid range from 0 - 255)",
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_INVALID,
+ },
+ {
+ .name = "high_weight",
+ .lname = "high_weight for WRR",
+ .type = FIO_OPT_INT,
+ .off1 = offsetof(struct spdk_fio_options, high_weight),
+ .def = "0",
+ .help = "high weight used for WRR (valid range from 0 - 255)",
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_INVALID,
+ },
+ {
+ .name = "wrr_priority",
+ .lname = "priority used for WRR",
+ .type = FIO_OPT_INT,
+ .off1 = offsetof(struct spdk_fio_options, wrr_priority),
+ .def = "0",
+ .help = "priority used for WRR (valid range from 0-3)",
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_INVALID,
+ },
+ {
+ .name = "mem_size_mb",
+ .lname = "Memory size in MB",
+ .type = FIO_OPT_INT,
+ .off1 = offsetof(struct spdk_fio_options, mem_size),
+ .def = "0",
+ .help = "Memory Size for SPDK (MB)",
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_INVALID,
+ },
+ {
+ .name = "shm_id",
+ .lname = "shared memory ID",
+ .type = FIO_OPT_INT,
+ .off1 = offsetof(struct spdk_fio_options, shm_id),
+ .def = "-1",
+ .help = "Shared Memory ID",
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_INVALID,
+ },
+ {
+ .name = "enable_sgl",
+ .lname = "SGL used for I/O commands",
+ .type = FIO_OPT_INT,
+ .off1 = offsetof(struct spdk_fio_options, enable_sgl),
+ .def = "0",
+ .help = "SGL Used for I/O Commands (enable_sgl=1 or enable_sgl=0)",
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_INVALID,
+ },
+ {
+ .name = "sge_size",
+ .lname = "SGL size used for I/O commands",
+ .type = FIO_OPT_INT,
+ .off1 = offsetof(struct spdk_fio_options, sge_size),
+ .def = "4096",
+ .help = "SGL size in bytes for I/O Commands (default 4096)",
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_INVALID,
+ },
+ {
+ .name = "bit_bucket_data_len",
+ .lname = "Amount of data used for Bit Bucket",
+ .type = FIO_OPT_INT,
+ .off1 = offsetof(struct spdk_fio_options, bit_bucket_data_len),
+ .def = "0",
+ .help = "Bit Bucket Data Length for READ commands (disabled by default)",
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_INVALID,
+ },
+ {
+ .name = "hostnqn",
+ .lname = "Host NQN to use when connecting to controllers.",
+ .type = FIO_OPT_STR_STORE,
+ .off1 = offsetof(struct spdk_fio_options, hostnqn),
+ .help = "Host NQN",
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_INVALID,
+ },
+ {
+ .name = "pi_act",
+ .lname = "Protection Information Action",
+ .type = FIO_OPT_INT,
+ .off1 = offsetof(struct spdk_fio_options, pi_act),
+ .def = "1",
+ .help = "Protection Information Action bit (pi_act=1 or pi_act=0)",
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_INVALID,
+ },
+ {
+ .name = "pi_chk",
+ .lname = "Protection Information Check(GUARD|REFTAG|APPTAG)",
+ .type = FIO_OPT_STR_STORE,
+ .off1 = offsetof(struct spdk_fio_options, pi_chk),
+ .def = NULL,
+ .help = "Control of Protection Information Checking (pi_chk=GUARD|REFTAG|APPTAG)",
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_INVALID,
+ },
+ {
+ .name = "md_per_io_size",
+ .lname = "Separate Metadata Buffer Size per I/O",
+ .type = FIO_OPT_INT,
+ .off1 = offsetof(struct spdk_fio_options, md_per_io_size),
+ .def = "4096",
+ .help = "Size of separate metadata buffer per I/O (Default: 4096)",
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_INVALID,
+ },
+ {
+ .name = "apptag",
+ .lname = "Application Tag used in Protection Information",
+ .type = FIO_OPT_INT,
+ .off1 = offsetof(struct spdk_fio_options, apptag),
+ .def = "0x1234",
+ .help = "Application Tag used in Protection Information field (Default: 0x1234)",
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_INVALID,
+ },
+ {
+ .name = "apptag_mask",
+ .lname = "Application Tag Mask",
+ .type = FIO_OPT_INT,
+ .off1 = offsetof(struct spdk_fio_options, apptag_mask),
+ .def = "0xffff",
+ .help = "Application Tag Mask used with Application Tag (Default: 0xffff)",
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_INVALID,
+ },
+ {
+ .name = "digest_enable",
+ .lname = "PDU digest choice for NVMe/TCP Transport(NONE|HEADER|DATA|BOTH)",
+ .type = FIO_OPT_STR_STORE,
+ .off1 = offsetof(struct spdk_fio_options, digest_enable),
+ .def = NULL,
+ .help = "Control the NVMe/TCP control(digest_enable=NONE|HEADER|DATA|BOTH)",
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_INVALID,
+ },
+ {
+ .name = "enable_vmd",
+ .lname = "Enable VMD enumeration",
+ .type = FIO_OPT_INT,
+ .off1 = offsetof(struct spdk_fio_options, enable_vmd),
+ .def = "0",
+ .help = "Enable VMD enumeration (enable_vmd=1 or enable_vmd=0)",
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_INVALID,
+ },
+ {
+ .name = NULL,
+ },
+};
+
+/* FIO imports this structure using dlsym */
+struct ioengine_ops ioengine = {
+ .name = "spdk",
+ .version = FIO_IOOPS_VERSION,
+ .queue = spdk_fio_queue,
+ .getevents = spdk_fio_getevents,
+ .event = spdk_fio_event,
+ .cleanup = spdk_fio_cleanup,
+ .open_file = spdk_fio_open,
+ .close_file = spdk_fio_close,
+ .invalidate = spdk_fio_invalidate,
+ .iomem_alloc = spdk_fio_iomem_alloc,
+ .iomem_free = spdk_fio_iomem_free,
+ .setup = spdk_fio_setup,
+ .io_u_init = spdk_fio_io_u_init,
+ .io_u_free = spdk_fio_io_u_free,
+ .flags = FIO_RAWIO | FIO_NOEXTEND | FIO_NODISKUTIL | FIO_MEMALIGN,
+ .options = options,
+ .option_struct_size = sizeof(struct spdk_fio_options),
+};
+
+static void fio_init fio_spdk_register(void)
+{
+ register_ioengine(&ioengine);
+}
+
+static void fio_exit fio_spdk_unregister(void)
+{
+ unregister_ioengine(&ioengine);
+}
diff --git a/src/spdk/examples/nvme/fio_plugin/full_bench.fio b/src/spdk/examples/nvme/fio_plugin/full_bench.fio
new file mode 100644
index 000000000..4dea21d13
--- /dev/null
+++ b/src/spdk/examples/nvme/fio_plugin/full_bench.fio
@@ -0,0 +1,40 @@
+[global]
+thread=1
+group_reporting=1
+direct=1
+verify=0
+norandommap=1
+cpumask=1
+disable_slat=1
+disable_bw=1
+lat_percentiles=1
+clat_percentiles=0
+percentile_list=50:99:99.999
+
+[precondition-sequential]
+stonewall
+description="Sequentially write to the device twice"
+rw=write
+iodepth=128
+bs=128k
+loops=2
+
+[4k_randwrite_qd1]
+stonewall
+description="4KiB Random Write QD=1"
+bs=4k
+rw=randwrite
+iodepth=1
+time_based=1
+ramp_time=60
+runtime=240
+
+[4k_randread_qd1]
+stonewall
+description="4KiB Random Read QD=1"
+bs=4k
+rw=randread
+iodepth=1
+time_based=1
+ramp_time=60
+runtime=240
diff --git a/src/spdk/examples/nvme/fio_plugin/mock_sgl_config.fio b/src/spdk/examples/nvme/fio_plugin/mock_sgl_config.fio
new file mode 100644
index 000000000..713fce0a2
--- /dev/null
+++ b/src/spdk/examples/nvme/fio_plugin/mock_sgl_config.fio
@@ -0,0 +1,17 @@
+[global]
+ioengine=spdk
+thread=1
+group_reporting=1
+direct=1
+enable_sgl=1
+time_based=1
+ramp_time=0
+runtime=2
+iodepth=128
+rw=randrw
+bs=16k
+verify=md5
+verify_backlog=32
+
+[test]
+numjobs=1
diff --git a/src/spdk/examples/nvme/hello_world/.gitignore b/src/spdk/examples/nvme/hello_world/.gitignore
new file mode 100644
index 000000000..242c034c1
--- /dev/null
+++ b/src/spdk/examples/nvme/hello_world/.gitignore
@@ -0,0 +1 @@
+hello_world
diff --git a/src/spdk/examples/nvme/hello_world/Makefile b/src/spdk/examples/nvme/hello_world/Makefile
new file mode 100644
index 000000000..bbb3527cb
--- /dev/null
+++ b/src/spdk/examples/nvme/hello_world/Makefile
@@ -0,0 +1,38 @@
+#
+# BSD LICENSE
+#
+# Copyright (c) Intel Corporation.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Intel Corporation nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
+
+APP = hello_world
+
+include $(SPDK_ROOT_DIR)/mk/nvme.libtest.mk
diff --git a/src/spdk/examples/nvme/hello_world/hello_world.c b/src/spdk/examples/nvme/hello_world/hello_world.c
new file mode 100644
index 000000000..6e1d9d62a
--- /dev/null
+++ b/src/spdk/examples/nvme/hello_world/hello_world.c
@@ -0,0 +1,435 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+
+#include "spdk/nvme.h"
+#include "spdk/vmd.h"
+#include "spdk/env.h"
+
+struct ctrlr_entry {
+ struct spdk_nvme_ctrlr *ctrlr;
+ struct ctrlr_entry *next;
+ char name[1024];
+};
+
+struct ns_entry {
+ struct spdk_nvme_ctrlr *ctrlr;
+ struct spdk_nvme_ns *ns;
+ struct ns_entry *next;
+ struct spdk_nvme_qpair *qpair;
+};
+
+static struct ctrlr_entry *g_controllers = NULL;
+static struct ns_entry *g_namespaces = NULL;
+
+static bool g_vmd = false;
+
+static void
+register_ns(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns *ns)
+{
+ struct ns_entry *entry;
+
+ if (!spdk_nvme_ns_is_active(ns)) {
+ return;
+ }
+
+ entry = malloc(sizeof(struct ns_entry));
+ if (entry == NULL) {
+ perror("ns_entry malloc");
+ exit(1);
+ }
+
+ entry->ctrlr = ctrlr;
+ entry->ns = ns;
+ entry->next = g_namespaces;
+ g_namespaces = entry;
+
+ printf(" Namespace ID: %d size: %juGB\n", spdk_nvme_ns_get_id(ns),
+ spdk_nvme_ns_get_size(ns) / 1000000000);
+}
+
+struct hello_world_sequence {
+ struct ns_entry *ns_entry;
+ char *buf;
+ unsigned using_cmb_io;
+ int is_completed;
+};
+
+static void
+read_complete(void *arg, const struct spdk_nvme_cpl *completion)
+{
+ struct hello_world_sequence *sequence = arg;
+
+ /* Assume the I/O was successful */
+ sequence->is_completed = 1;
+ /* See if an error occurred. If so, display information
+ * about it, and set completion value so that I/O
+ * caller is aware that an error occurred.
+ */
+ if (spdk_nvme_cpl_is_error(completion)) {
+ spdk_nvme_qpair_print_completion(sequence->ns_entry->qpair, (struct spdk_nvme_cpl *)completion);
+ fprintf(stderr, "I/O error status: %s\n", spdk_nvme_cpl_get_status_string(&completion->status));
+ fprintf(stderr, "Read I/O failed, aborting run\n");
+ sequence->is_completed = 2;
+ }
+
+ /*
+ * The read I/O has completed. Print the contents of the
+ * buffer, free the buffer, then mark the sequence as
+ * completed. This will trigger the hello_world() function
+ * to exit its polling loop.
+ */
+ printf("%s", sequence->buf);
+ spdk_free(sequence->buf);
+}
+
+static void
+write_complete(void *arg, const struct spdk_nvme_cpl *completion)
+{
+ struct hello_world_sequence *sequence = arg;
+ struct ns_entry *ns_entry = sequence->ns_entry;
+ int rc;
+
+ /* See if an error occurred. If so, display information
+ * about it, and set completion value so that I/O
+ * caller is aware that an error occurred.
+ */
+ if (spdk_nvme_cpl_is_error(completion)) {
+ spdk_nvme_qpair_print_completion(sequence->ns_entry->qpair, (struct spdk_nvme_cpl *)completion);
+ fprintf(stderr, "I/O error status: %s\n", spdk_nvme_cpl_get_status_string(&completion->status));
+ fprintf(stderr, "Write I/O failed, aborting run\n");
+ sequence->is_completed = 2;
+ exit(1);
+ }
+ /*
+ * The write I/O has completed. Free the buffer associated with
+ * the write I/O and allocate a new zeroed buffer for reading
+ * the data back from the NVMe namespace.
+ */
+ if (sequence->using_cmb_io) {
+ spdk_nvme_ctrlr_unmap_cmb(ns_entry->ctrlr);
+ } else {
+ spdk_free(sequence->buf);
+ }
+ sequence->buf = spdk_zmalloc(0x1000, 0x1000, NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA);
+
+ rc = spdk_nvme_ns_cmd_read(ns_entry->ns, ns_entry->qpair, sequence->buf,
+ 0, /* LBA start */
+ 1, /* number of LBAs */
+ read_complete, (void *)sequence, 0);
+ if (rc != 0) {
+ fprintf(stderr, "starting read I/O failed\n");
+ exit(1);
+ }
+}
+
+static void
+hello_world(void)
+{
+ struct ns_entry *ns_entry;
+ struct hello_world_sequence sequence;
+ int rc;
+ size_t sz;
+
+ ns_entry = g_namespaces;
+ while (ns_entry != NULL) {
+ /*
+ * Allocate an I/O qpair that we can use to submit read/write requests
+ * to namespaces on the controller. NVMe controllers typically support
+ * many qpairs per controller. Any I/O qpair allocated for a controller
+ * can submit I/O to any namespace on that controller.
+ *
+ * The SPDK NVMe driver provides no synchronization for qpair accesses -
+ * the application must ensure only a single thread submits I/O to a
+ * qpair, and that same thread must also check for completions on that
+ * qpair. This enables extremely efficient I/O processing by making all
+ * I/O operations completely lockless.
+ */
+ ns_entry->qpair = spdk_nvme_ctrlr_alloc_io_qpair(ns_entry->ctrlr, NULL, 0);
+ if (ns_entry->qpair == NULL) {
+ printf("ERROR: spdk_nvme_ctrlr_alloc_io_qpair() failed\n");
+ return;
+ }
+
+ /*
+ * Use spdk_dma_zmalloc to allocate a 4KB zeroed buffer. This memory
+ * will be pinned, which is required for data buffers used for SPDK NVMe
+ * I/O operations.
+ */
+ sequence.using_cmb_io = 1;
+ sequence.buf = spdk_nvme_ctrlr_map_cmb(ns_entry->ctrlr, &sz);
+ if (sequence.buf == NULL || sz < 0x1000) {
+ sequence.using_cmb_io = 0;
+ sequence.buf = spdk_zmalloc(0x1000, 0x1000, NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA);
+ }
+ if (sequence.buf == NULL) {
+ printf("ERROR: write buffer allocation failed\n");
+ return;
+ }
+ if (sequence.using_cmb_io) {
+ printf("INFO: using controller memory buffer for IO\n");
+ } else {
+ printf("INFO: using host memory buffer for IO\n");
+ }
+ sequence.is_completed = 0;
+ sequence.ns_entry = ns_entry;
+
+ /*
+ * Print "Hello world!" to sequence.buf. We will write this data to LBA
+ * 0 on the namespace, and then later read it back into a separate buffer
+ * to demonstrate the full I/O path.
+ */
+ snprintf(sequence.buf, 0x1000, "%s", "Hello world!\n");
+
+ /*
+ * Write the data buffer to LBA 0 of this namespace. "write_complete" and
+ * "&sequence" are specified as the completion callback function and
+ * argument respectively. write_complete() will be called with the
+ * value of &sequence as a parameter when the write I/O is completed.
+ * This allows users to potentially specify different completion
+ * callback routines for each I/O, as well as pass a unique handle
+ * as an argument so the application knows which I/O has completed.
+ *
+ * Note that the SPDK NVMe driver will only check for completions
+ * when the application calls spdk_nvme_qpair_process_completions().
+ * It is the responsibility of the application to trigger the polling
+ * process.
+ */
+ rc = spdk_nvme_ns_cmd_write(ns_entry->ns, ns_entry->qpair, sequence.buf,
+ 0, /* LBA start */
+ 1, /* number of LBAs */
+ write_complete, &sequence, 0);
+ if (rc != 0) {
+ fprintf(stderr, "starting write I/O failed\n");
+ exit(1);
+ }
+
+ /*
+ * Poll for completions. 0 here means process all available completions.
+ * In certain usage models, the caller may specify a positive integer
+ * instead of 0 to signify the maximum number of completions it should
+ * process. This function will never block - if there are no
+ * completions pending on the specified qpair, it will return immediately.
+ *
+ * When the write I/O completes, write_complete() will submit a new I/O
+ * to read LBA 0 into a separate buffer, specifying read_complete() as its
+ * completion routine. When the read I/O completes, read_complete() will
+ * print the buffer contents and set sequence.is_completed = 1. That will
+ * break this loop and then exit the program.
+ */
+ while (!sequence.is_completed) {
+ spdk_nvme_qpair_process_completions(ns_entry->qpair, 0);
+ }
+
+ /*
+ * Free the I/O qpair. This typically is done when an application exits.
+ * But SPDK does support freeing and then reallocating qpairs during
+ * operation. It is the responsibility of the caller to ensure all
+ * pending I/O are completed before trying to free the qpair.
+ */
+ spdk_nvme_ctrlr_free_io_qpair(ns_entry->qpair);
+ ns_entry = ns_entry->next;
+ }
+}
+
+static bool
+probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
+ struct spdk_nvme_ctrlr_opts *opts)
+{
+ printf("Attaching to %s\n", trid->traddr);
+
+ return true;
+}
+
+static void
+attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
+ struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts)
+{
+ int nsid, num_ns;
+ struct ctrlr_entry *entry;
+ struct spdk_nvme_ns *ns;
+ const struct spdk_nvme_ctrlr_data *cdata;
+
+ entry = malloc(sizeof(struct ctrlr_entry));
+ if (entry == NULL) {
+ perror("ctrlr_entry malloc");
+ exit(1);
+ }
+
+ printf("Attached to %s\n", trid->traddr);
+
+ /*
+ * spdk_nvme_ctrlr is the logical abstraction in SPDK for an NVMe
+ * controller. During initialization, the IDENTIFY data for the
+ * controller is read using an NVMe admin command, and that data
+ * can be retrieved using spdk_nvme_ctrlr_get_data() to get
+ * detailed information on the controller. Refer to the NVMe
+ * specification for more details on IDENTIFY for NVMe controllers.
+ */
+ cdata = spdk_nvme_ctrlr_get_data(ctrlr);
+
+ snprintf(entry->name, sizeof(entry->name), "%-20.20s (%-20.20s)", cdata->mn, cdata->sn);
+
+ entry->ctrlr = ctrlr;
+ entry->next = g_controllers;
+ g_controllers = entry;
+
+ /*
+ * Each controller has one or more namespaces. An NVMe namespace is basically
+ * equivalent to a SCSI LUN. The controller's IDENTIFY data tells us how
+ * many namespaces exist on the controller. For Intel(R) P3X00 controllers,
+ * it will just be one namespace.
+ *
+ * Note that in NVMe, namespace IDs start at 1, not 0.
+ */
+ num_ns = spdk_nvme_ctrlr_get_num_ns(ctrlr);
+ printf("Using controller %s with %d namespaces.\n", entry->name, num_ns);
+ for (nsid = 1; nsid <= num_ns; nsid++) {
+ ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid);
+ if (ns == NULL) {
+ continue;
+ }
+ register_ns(ctrlr, ns);
+ }
+}
+
+static void
+cleanup(void)
+{
+ struct ns_entry *ns_entry = g_namespaces;
+ struct ctrlr_entry *ctrlr_entry = g_controllers;
+
+ while (ns_entry) {
+ struct ns_entry *next = ns_entry->next;
+ free(ns_entry);
+ ns_entry = next;
+ }
+
+ while (ctrlr_entry) {
+ struct ctrlr_entry *next = ctrlr_entry->next;
+
+ spdk_nvme_detach(ctrlr_entry->ctrlr);
+ free(ctrlr_entry);
+ ctrlr_entry = next;
+ }
+}
+
+static void
+usage(const char *program_name)
+{
+ printf("%s [options]", program_name);
+ printf("\n");
+ printf("options:\n");
+ printf(" -V enumerate VMD\n");
+}
+
+static int
+parse_args(int argc, char **argv)
+{
+ int op;
+
+ while ((op = getopt(argc, argv, "V")) != -1) {
+ switch (op) {
+ case 'V':
+ g_vmd = true;
+ break;
+ default:
+ usage(argv[0]);
+ return 1;
+ }
+ }
+
+ return 0;
+}
+
+int main(int argc, char **argv)
+{
+ int rc;
+ struct spdk_env_opts opts;
+
+ rc = parse_args(argc, argv);
+ if (rc != 0) {
+ return rc;
+ }
+
+ /*
+ * SPDK relies on an abstraction around the local environment
+ * named env that handles memory allocation and PCI device operations.
+ * This library must be initialized first.
+ *
+ */
+ spdk_env_opts_init(&opts);
+ opts.name = "hello_world";
+ opts.shm_id = 0;
+ if (spdk_env_init(&opts) < 0) {
+ fprintf(stderr, "Unable to initialize SPDK env\n");
+ return 1;
+ }
+
+ printf("Initializing NVMe Controllers\n");
+
+ if (g_vmd && spdk_vmd_init()) {
+ fprintf(stderr, "Failed to initialize VMD."
+ " Some NVMe devices can be unavailable.\n");
+ }
+
+ /*
+ * Start the SPDK NVMe enumeration process. probe_cb will be called
+ * for each NVMe controller found, giving our application a choice on
+ * whether to attach to each controller. attach_cb will then be
+ * called for each controller after the SPDK NVMe driver has completed
+ * initializing the controller we chose to attach.
+ */
+ rc = spdk_nvme_probe(NULL, NULL, probe_cb, attach_cb, NULL);
+ if (rc != 0) {
+ fprintf(stderr, "spdk_nvme_probe() failed\n");
+ cleanup();
+ return 1;
+ }
+
+ if (g_controllers == NULL) {
+ fprintf(stderr, "no NVMe controllers found\n");
+ cleanup();
+ return 1;
+ }
+
+ printf("Initialization complete.\n");
+ hello_world();
+ cleanup();
+ if (g_vmd) {
+ spdk_vmd_fini();
+ }
+
+ return 0;
+}
diff --git a/src/spdk/examples/nvme/hotplug/.gitignore b/src/spdk/examples/nvme/hotplug/.gitignore
new file mode 100644
index 000000000..e6ff53805
--- /dev/null
+++ b/src/spdk/examples/nvme/hotplug/.gitignore
@@ -0,0 +1 @@
+hotplug
diff --git a/src/spdk/examples/nvme/hotplug/Makefile b/src/spdk/examples/nvme/hotplug/Makefile
new file mode 100644
index 000000000..c77c61227
--- /dev/null
+++ b/src/spdk/examples/nvme/hotplug/Makefile
@@ -0,0 +1,38 @@
+#
+# BSD LICENSE
+#
+# Copyright (c) Intel Corporation.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Intel Corporation nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
+
+APP = hotplug
+
+include $(SPDK_ROOT_DIR)/mk/nvme.libtest.mk
diff --git a/src/spdk/examples/nvme/hotplug/hotplug.c b/src/spdk/examples/nvme/hotplug/hotplug.c
new file mode 100644
index 000000000..ff821c7ed
--- /dev/null
+++ b/src/spdk/examples/nvme/hotplug/hotplug.c
@@ -0,0 +1,525 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+
+#include "spdk/nvme.h"
+#include "spdk/queue.h"
+#include "spdk/string.h"
+#include "spdk/util.h"
+
+struct dev_ctx {
+ TAILQ_ENTRY(dev_ctx) tailq;
+ bool is_new;
+ bool is_removed;
+ bool is_draining;
+ struct spdk_nvme_ctrlr *ctrlr;
+ struct spdk_nvme_ns *ns;
+ struct spdk_nvme_qpair *qpair;
+ uint32_t io_size_blocks;
+ uint64_t size_in_ios;
+ uint64_t io_completed;
+ uint64_t prev_io_completed;
+ uint64_t current_queue_depth;
+ uint64_t offset_in_ios;
+ char name[1024];
+};
+
+struct perf_task {
+ struct dev_ctx *dev;
+ void *buf;
+};
+
+static TAILQ_HEAD(, dev_ctx) g_devs = TAILQ_HEAD_INITIALIZER(g_devs);
+
+static uint64_t g_tsc_rate;
+
+static uint32_t g_io_size_bytes = 4096;
+static int g_queue_depth = 4;
+static int g_time_in_sec;
+static int g_expected_insert_times = -1;
+static int g_expected_removal_times = -1;
+static int g_insert_times;
+static int g_removal_times;
+static int g_shm_id = -1;
+static uint64_t g_timeout_in_us = SPDK_SEC_TO_USEC;
+
+static void
+task_complete(struct perf_task *task);
+
+static void
+timeout_cb(void *cb_arg, struct spdk_nvme_ctrlr *ctrlr,
+ struct spdk_nvme_qpair *qpair, uint16_t cid);
+
+static void
+register_dev(struct spdk_nvme_ctrlr *ctrlr)
+{
+ struct dev_ctx *dev;
+ const struct spdk_nvme_ctrlr_data *cdata = spdk_nvme_ctrlr_get_data(ctrlr);
+
+ dev = calloc(1, sizeof(*dev));
+ if (dev == NULL) {
+ perror("dev_ctx malloc");
+ exit(1);
+ }
+
+ snprintf(dev->name, sizeof(dev->name), "%-20.20s (%-20.20s)", cdata->mn, cdata->sn);
+
+ dev->ctrlr = ctrlr;
+ dev->is_new = true;
+ dev->is_removed = false;
+ dev->is_draining = false;
+
+ spdk_nvme_ctrlr_register_timeout_callback(ctrlr, g_timeout_in_us, timeout_cb, NULL);
+
+ dev->ns = spdk_nvme_ctrlr_get_ns(ctrlr, 1);
+
+ if (!dev->ns || !spdk_nvme_ns_is_active(dev->ns)) {
+ fprintf(stderr, "Controller %s: No active namespace; skipping\n", dev->name);
+ goto skip;
+ }
+
+ if (spdk_nvme_ns_get_size(dev->ns) < g_io_size_bytes ||
+ spdk_nvme_ns_get_sector_size(dev->ns) > g_io_size_bytes) {
+ fprintf(stderr, "Controller %s: Invalid "
+ "ns size %" PRIu64 " / block size %u for I/O size %u\n",
+ dev->name,
+ spdk_nvme_ns_get_size(dev->ns),
+ spdk_nvme_ns_get_sector_size(dev->ns),
+ g_io_size_bytes);
+ goto skip;
+ }
+
+ dev->size_in_ios = spdk_nvme_ns_get_size(dev->ns) / g_io_size_bytes;
+ dev->io_size_blocks = g_io_size_bytes / spdk_nvme_ns_get_sector_size(dev->ns);
+
+ dev->qpair = spdk_nvme_ctrlr_alloc_io_qpair(ctrlr, NULL, 0);
+ if (!dev->qpair) {
+ fprintf(stderr, "ERROR: spdk_nvme_ctrlr_alloc_io_qpair() failed\n");
+ goto skip;
+ }
+ g_insert_times++;
+ TAILQ_INSERT_TAIL(&g_devs, dev, tailq);
+ return;
+
+skip:
+ free(dev);
+}
+
+static void
+unregister_dev(struct dev_ctx *dev)
+{
+ fprintf(stderr, "unregister_dev: %s\n", dev->name);
+
+ spdk_nvme_ctrlr_free_io_qpair(dev->qpair);
+ spdk_nvme_detach(dev->ctrlr);
+
+ TAILQ_REMOVE(&g_devs, dev, tailq);
+ free(dev);
+}
+
+static struct perf_task *
+alloc_task(struct dev_ctx *dev)
+{
+ struct perf_task *task;
+
+ task = calloc(1, sizeof(*task));
+ if (task == NULL) {
+ return NULL;
+ }
+
+ task->buf = spdk_dma_zmalloc(g_io_size_bytes, 0x200, NULL);
+ if (task->buf == NULL) {
+ free(task);
+ return NULL;
+ }
+
+ task->dev = dev;
+
+ return task;
+}
+
+static void
+free_task(struct perf_task *task)
+{
+ spdk_dma_free(task->buf);
+ free(task);
+}
+
+static void io_complete(void *ctx, const struct spdk_nvme_cpl *completion);
+
+static void
+submit_single_io(struct perf_task *task)
+{
+ struct dev_ctx *dev = task->dev;
+ uint64_t offset_in_ios;
+ int rc;
+
+ offset_in_ios = dev->offset_in_ios++;
+ if (dev->offset_in_ios == dev->size_in_ios) {
+ dev->offset_in_ios = 0;
+ }
+
+ rc = spdk_nvme_ns_cmd_read(dev->ns, dev->qpair, task->buf,
+ offset_in_ios * dev->io_size_blocks,
+ dev->io_size_blocks, io_complete, task, 0);
+
+ if (rc != 0) {
+ fprintf(stderr, "starting I/O failed\n");
+ free_task(task);
+ } else {
+ dev->current_queue_depth++;
+ }
+}
+
+static void
+task_complete(struct perf_task *task)
+{
+ struct dev_ctx *dev;
+
+ dev = task->dev;
+ dev->current_queue_depth--;
+ dev->io_completed++;
+
+ /*
+ * is_draining indicates when time has expired for the test run
+ * and we are just waiting for the previously submitted I/O
+ * to complete. In this case, do not submit a new I/O to replace
+ * the one just completed.
+ */
+ if (!dev->is_draining && !dev->is_removed) {
+ submit_single_io(task);
+ } else {
+ free_task(task);
+ }
+}
+
+static void
+io_complete(void *ctx, const struct spdk_nvme_cpl *completion)
+{
+ task_complete((struct perf_task *)ctx);
+}
+
+static void
+check_io(struct dev_ctx *dev)
+{
+ spdk_nvme_qpair_process_completions(dev->qpair, 0);
+}
+
+static void
+submit_io(struct dev_ctx *dev, int queue_depth)
+{
+ struct perf_task *task;
+
+ while (queue_depth-- > 0) {
+ task = alloc_task(dev);
+ if (task == NULL) {
+ fprintf(stderr, "task allocation failed\n");
+ exit(1);
+ }
+
+ submit_single_io(task);
+ }
+}
+
+static void
+drain_io(struct dev_ctx *dev)
+{
+ dev->is_draining = true;
+ while (dev->current_queue_depth > 0) {
+ check_io(dev);
+ }
+}
+
+static void
+print_stats(void)
+{
+ struct dev_ctx *dev;
+
+ TAILQ_FOREACH(dev, &g_devs, tailq) {
+ fprintf(stderr, "%-43.43s: %10" PRIu64 " I/Os completed (+%" PRIu64 ")\n",
+ dev->name,
+ dev->io_completed,
+ dev->io_completed - dev->prev_io_completed);
+ dev->prev_io_completed = dev->io_completed;
+ }
+
+ fprintf(stderr, "\n");
+}
+
+static bool
+probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
+ struct spdk_nvme_ctrlr_opts *opts)
+{
+ fprintf(stderr, "Attaching to %s\n", trid->traddr);
+
+ return true;
+}
+
+static void
+attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
+ struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts)
+{
+ fprintf(stderr, "Attached to %s\n", trid->traddr);
+
+ register_dev(ctrlr);
+}
+
+static void
+remove_cb(void *cb_ctx, struct spdk_nvme_ctrlr *ctrlr)
+{
+ struct dev_ctx *dev;
+
+ TAILQ_FOREACH(dev, &g_devs, tailq) {
+ if (dev->ctrlr == ctrlr) {
+ /*
+ * Mark the device as removed, but don't detach yet.
+ *
+ * The I/O handling code will detach once it sees that
+ * is_removed is true and all outstanding I/O have been completed.
+ */
+ dev->is_removed = true;
+ fprintf(stderr, "Controller removed: %s\n", dev->name);
+ return;
+ }
+ }
+
+ /*
+ * If we get here, this remove_cb is for a controller that we are not tracking
+ * in g_devs (for example, because we skipped it during register_dev),
+ * so immediately detach it.
+ */
+ spdk_nvme_detach(ctrlr);
+}
+
+static void
+timeout_cb(void *cb_arg, struct spdk_nvme_ctrlr *ctrlr,
+ struct spdk_nvme_qpair *qpair, uint16_t cid)
+{
+ /* leave hotplug monitor loop, use the timeout_cb to monitor the hotplug */
+ if (spdk_nvme_probe(NULL, NULL, probe_cb, attach_cb, remove_cb) != 0) {
+ fprintf(stderr, "spdk_nvme_probe() failed\n");
+ }
+}
+
+static void
+io_loop(void)
+{
+ struct dev_ctx *dev, *dev_tmp;
+ uint64_t tsc_end;
+ uint64_t next_stats_tsc;
+
+ tsc_end = spdk_get_ticks() + g_time_in_sec * g_tsc_rate;
+ next_stats_tsc = spdk_get_ticks();
+
+ while (1) {
+ uint64_t now;
+
+ /*
+ * Check for completed I/O for each controller. A new
+ * I/O will be submitted in the io_complete callback
+ * to replace each I/O that is completed.
+ */
+ TAILQ_FOREACH(dev, &g_devs, tailq) {
+ if (dev->is_new) {
+ /* Submit initial I/O for this controller. */
+ submit_io(dev, g_queue_depth);
+ dev->is_new = false;
+ }
+
+ check_io(dev);
+ }
+
+ /*
+ * Check for hotplug events.
+ */
+ if (spdk_nvme_probe(NULL, NULL, probe_cb, attach_cb, remove_cb) != 0) {
+ fprintf(stderr, "spdk_nvme_probe() failed\n");
+ break;
+ }
+
+ /*
+ * Check for devices which were hot-removed and have finished
+ * processing outstanding I/Os.
+ *
+ * unregister_dev() may remove devs from the list, so use the
+ * removal-safe iterator.
+ */
+ TAILQ_FOREACH_SAFE(dev, &g_devs, tailq, dev_tmp) {
+ if (dev->is_removed && dev->current_queue_depth == 0) {
+ g_removal_times++;
+ unregister_dev(dev);
+ }
+ }
+
+ now = spdk_get_ticks();
+ if (now > tsc_end) {
+ break;
+ }
+ if (now > next_stats_tsc) {
+ print_stats();
+ next_stats_tsc += g_tsc_rate;
+ }
+
+ if (g_insert_times == g_expected_insert_times && g_removal_times == g_expected_removal_times) {
+ break;
+ }
+ }
+
+ TAILQ_FOREACH_SAFE(dev, &g_devs, tailq, dev_tmp) {
+ drain_io(dev);
+ unregister_dev(dev);
+ }
+}
+
+static void usage(char *program_name)
+{
+ printf("%s options", program_name);
+ printf("\n");
+ printf("\t[-c timeout for each command in second(default:1s)]\n");
+ printf("\t[-i shm id (optional)]\n");
+ printf("\t[-n expected hot insert times]\n");
+ printf("\t[-r expected hot removal times]\n");
+ printf("\t[-t time in seconds]\n");
+}
+
+static int
+parse_args(int argc, char **argv)
+{
+ int op;
+ long int val;
+
+ /* default value */
+ g_time_in_sec = 0;
+
+ while ((op = getopt(argc, argv, "c:i:n:r:t:")) != -1) {
+ if (op == '?') {
+ usage(argv[0]);
+ return 1;
+ }
+
+ val = spdk_strtol(optarg, 10);
+ if (val < 0) {
+ fprintf(stderr, "Converting a string to integer failed\n");
+ return val;
+ }
+ switch (op) {
+ case 'c':
+ g_timeout_in_us = val * SPDK_SEC_TO_USEC;
+ break;
+ case 'i':
+ g_shm_id = val;
+ break;
+ case 'n':
+ g_expected_insert_times = val;
+ break;
+ case 'r':
+ g_expected_removal_times = val;
+ break;
+ case 't':
+ g_time_in_sec = val;
+ break;
+ default:
+ usage(argv[0]);
+ return 1;
+ }
+ }
+
+ if (!g_time_in_sec) {
+ usage(argv[0]);
+ return 1;
+ }
+
+ return 0;
+}
+
+
+static int
+register_controllers(void)
+{
+ fprintf(stderr, "Initializing NVMe Controllers\n");
+
+ if (spdk_nvme_probe(NULL, NULL, probe_cb, attach_cb, remove_cb) != 0) {
+ fprintf(stderr, "spdk_nvme_probe() failed\n");
+ return 1;
+ }
+ /* Reset g_insert_times to 0 so that we do not count controllers attached at start as hotplug events. */
+ g_insert_times = 0;
+ return 0;
+}
+
+int main(int argc, char **argv)
+{
+ int rc;
+ struct spdk_env_opts opts;
+
+ rc = parse_args(argc, argv);
+ if (rc != 0) {
+ return rc;
+ }
+
+ spdk_env_opts_init(&opts);
+ opts.name = "hotplug";
+ opts.core_mask = "0x1";
+ if (g_shm_id > -1) {
+ opts.shm_id = g_shm_id;
+ }
+ if (spdk_env_init(&opts) < 0) {
+ fprintf(stderr, "Unable to initialize SPDK env\n");
+ return 1;
+ }
+
+ g_tsc_rate = spdk_get_ticks_hz();
+
+ /* Detect the controllers that are plugged in at startup. */
+ if (register_controllers() != 0) {
+ return 1;
+ }
+
+ fprintf(stderr, "Initialization complete. Starting I/O...\n");
+ io_loop();
+
+ if (g_expected_insert_times != -1 && g_insert_times != g_expected_insert_times) {
+ fprintf(stderr, "Expected inserts %d != actual inserts %d\n",
+ g_expected_insert_times, g_insert_times);
+ return 1;
+ }
+
+ if (g_expected_removal_times != -1 && g_removal_times != g_expected_removal_times) {
+ fprintf(stderr, "Expected removals %d != actual removals %d\n",
+ g_expected_removal_times, g_removal_times);
+ return 1;
+ }
+
+ return 0;
+}
diff --git a/src/spdk/examples/nvme/identify/.gitignore b/src/spdk/examples/nvme/identify/.gitignore
new file mode 100644
index 000000000..5c5444c1e
--- /dev/null
+++ b/src/spdk/examples/nvme/identify/.gitignore
@@ -0,0 +1 @@
+identify
diff --git a/src/spdk/examples/nvme/identify/Makefile b/src/spdk/examples/nvme/identify/Makefile
new file mode 100644
index 000000000..ed7aa60a8
--- /dev/null
+++ b/src/spdk/examples/nvme/identify/Makefile
@@ -0,0 +1,44 @@
+#
+# BSD LICENSE
+#
+# Copyright (c) Intel Corporation.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Intel Corporation nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
+
+APP = identify
+
+include $(SPDK_ROOT_DIR)/mk/nvme.libtest.mk
+
+install: $(APP)
+ $(INSTALL_EXAMPLE)
+
+uninstall:
+ $(UNINSTALL_EXAMPLE)
diff --git a/src/spdk/examples/nvme/identify/identify.c b/src/spdk/examples/nvme/identify/identify.c
new file mode 100644
index 000000000..722f8d3ee
--- /dev/null
+++ b/src/spdk/examples/nvme/identify/identify.c
@@ -0,0 +1,1827 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+
+#include "spdk/endian.h"
+#include "spdk/log.h"
+#include "spdk/nvme.h"
+#include "spdk/vmd.h"
+#include "spdk/nvme_ocssd.h"
+#include "spdk/env.h"
+#include "spdk/nvme_intel.h"
+#include "spdk/nvmf_spec.h"
+#include "spdk/pci_ids.h"
+#include "spdk/string.h"
+#include "spdk/util.h"
+#include "spdk/uuid.h"
+
+#define MAX_DISCOVERY_LOG_ENTRIES ((uint64_t)1000)
+
+#define NUM_CHUNK_INFO_ENTRIES 8
+
+static int outstanding_commands;
+
+struct feature {
+ uint32_t result;
+ bool valid;
+};
+
+static struct feature features[256] = {};
+
+static struct spdk_nvme_error_information_entry error_page[256];
+
+static struct spdk_nvme_health_information_page health_page;
+
+static struct spdk_nvme_firmware_page firmware_page;
+
+static struct spdk_nvme_cmds_and_effect_log_page cmd_effects_log_page;
+
+static struct spdk_nvme_intel_smart_information_page intel_smart_page;
+
+static struct spdk_nvme_intel_temperature_page intel_temperature_page;
+
+static struct spdk_nvme_intel_marketing_description_page intel_md_page;
+
+static struct spdk_nvmf_discovery_log_page *g_discovery_page;
+static size_t g_discovery_page_size;
+static uint64_t g_discovery_page_numrec;
+
+static struct spdk_ocssd_geometry_data geometry_data;
+
+static struct spdk_ocssd_chunk_information_entry g_ocssd_chunk_info_page[NUM_CHUNK_INFO_ENTRIES ];
+
+static bool g_hex_dump = false;
+
+static int g_shm_id = -1;
+
+static int g_dpdk_mem = 0;
+
+static int g_master_core = 0;
+
+static char g_core_mask[16] = "0x1";
+
+static struct spdk_nvme_transport_id g_trid;
+
+static int g_controllers_found = 0;
+
+static bool g_vmd = false;
+
+static void
+hex_dump(const void *data, size_t size)
+{
+ size_t offset = 0, i;
+ const uint8_t *bytes = data;
+
+ while (size) {
+ printf("%08zX:", offset);
+
+ for (i = 0; i < 16; i++) {
+ if (i == 8) {
+ printf("-");
+ } else {
+ printf(" ");
+ }
+
+ if (i < size) {
+ printf("%02X", bytes[offset + i]);
+ } else {
+ printf(" ");
+ }
+ }
+
+ printf(" ");
+
+ for (i = 0; i < 16; i++) {
+ if (i < size) {
+ if (bytes[offset + i] > 0x20 && bytes[offset + i] < 0x7F) {
+ printf("%c", bytes[offset + i]);
+ } else {
+ printf(".");
+ }
+ }
+ }
+
+ printf("\n");
+
+ offset += 16;
+ if (size > 16) {
+ size -= 16;
+ } else {
+ break;
+ }
+ }
+}
+
+static void
+get_feature_completion(void *cb_arg, const struct spdk_nvme_cpl *cpl)
+{
+ struct feature *feature = cb_arg;
+ int fid = feature - features;
+
+ if (spdk_nvme_cpl_is_error(cpl)) {
+ printf("get_feature(0x%02X) failed\n", fid);
+ } else {
+ feature->result = cpl->cdw0;
+ feature->valid = true;
+ }
+ outstanding_commands--;
+}
+
+static void
+get_log_page_completion(void *cb_arg, const struct spdk_nvme_cpl *cpl)
+{
+ if (spdk_nvme_cpl_is_error(cpl)) {
+ printf("get log page failed\n");
+ }
+ outstanding_commands--;
+}
+
+static void
+get_ocssd_geometry_completion(void *cb_arg, const struct spdk_nvme_cpl *cpl)
+{
+ if (spdk_nvme_cpl_is_error(cpl)) {
+ printf("get ocssd geometry failed\n");
+ }
+ outstanding_commands--;
+}
+
+static int
+get_feature(struct spdk_nvme_ctrlr *ctrlr, uint8_t fid)
+{
+ struct spdk_nvme_cmd cmd = {};
+ struct feature *feature = &features[fid];
+
+ feature->valid = false;
+
+ cmd.opc = SPDK_NVME_OPC_GET_FEATURES;
+ cmd.cdw10_bits.get_features.fid = fid;
+
+ return spdk_nvme_ctrlr_cmd_admin_raw(ctrlr, &cmd, NULL, 0, get_feature_completion, feature);
+}
+
+static void
+get_features(struct spdk_nvme_ctrlr *ctrlr)
+{
+ size_t i;
+
+ uint8_t features_to_get[] = {
+ SPDK_NVME_FEAT_ARBITRATION,
+ SPDK_NVME_FEAT_POWER_MANAGEMENT,
+ SPDK_NVME_FEAT_TEMPERATURE_THRESHOLD,
+ SPDK_NVME_FEAT_ERROR_RECOVERY,
+ SPDK_NVME_FEAT_NUMBER_OF_QUEUES,
+ SPDK_OCSSD_FEAT_MEDIA_FEEDBACK,
+ };
+
+ /* Submit several GET FEATURES commands and wait for them to complete */
+ outstanding_commands = 0;
+ for (i = 0; i < SPDK_COUNTOF(features_to_get); i++) {
+ if (!spdk_nvme_ctrlr_is_ocssd_supported(ctrlr) &&
+ features_to_get[i] == SPDK_OCSSD_FEAT_MEDIA_FEEDBACK) {
+ continue;
+ }
+ if (get_feature(ctrlr, features_to_get[i]) == 0) {
+ outstanding_commands++;
+ } else {
+ printf("get_feature(0x%02X) failed to submit command\n", features_to_get[i]);
+ }
+ }
+
+ while (outstanding_commands) {
+ spdk_nvme_ctrlr_process_admin_completions(ctrlr);
+ }
+}
+
+static int
+get_error_log_page(struct spdk_nvme_ctrlr *ctrlr)
+{
+ const struct spdk_nvme_ctrlr_data *cdata;
+
+ cdata = spdk_nvme_ctrlr_get_data(ctrlr);
+
+ if (spdk_nvme_ctrlr_cmd_get_log_page(ctrlr, SPDK_NVME_LOG_ERROR,
+ SPDK_NVME_GLOBAL_NS_TAG, error_page,
+ sizeof(*error_page) * (cdata->elpe + 1),
+ 0,
+ get_log_page_completion, NULL)) {
+ printf("spdk_nvme_ctrlr_cmd_get_log_page() failed\n");
+ exit(1);
+ }
+
+ return 0;
+}
+
+static int
+get_health_log_page(struct spdk_nvme_ctrlr *ctrlr)
+{
+ if (spdk_nvme_ctrlr_cmd_get_log_page(ctrlr, SPDK_NVME_LOG_HEALTH_INFORMATION,
+ SPDK_NVME_GLOBAL_NS_TAG, &health_page, sizeof(health_page), 0, get_log_page_completion, NULL)) {
+ printf("spdk_nvme_ctrlr_cmd_get_log_page() failed\n");
+ exit(1);
+ }
+
+ return 0;
+}
+
+static int
+get_firmware_log_page(struct spdk_nvme_ctrlr *ctrlr)
+{
+ if (spdk_nvme_ctrlr_cmd_get_log_page(ctrlr, SPDK_NVME_LOG_FIRMWARE_SLOT,
+ SPDK_NVME_GLOBAL_NS_TAG, &firmware_page, sizeof(firmware_page), 0, get_log_page_completion, NULL)) {
+ printf("spdk_nvme_ctrlr_cmd_get_log_page() failed\n");
+ exit(1);
+ }
+
+ return 0;
+}
+
+static int
+get_cmd_effects_log_page(struct spdk_nvme_ctrlr *ctrlr)
+{
+ if (spdk_nvme_ctrlr_cmd_get_log_page(ctrlr, SPDK_NVME_LOG_COMMAND_EFFECTS_LOG,
+ SPDK_NVME_GLOBAL_NS_TAG, &cmd_effects_log_page, sizeof(cmd_effects_log_page), 0,
+ get_log_page_completion, NULL)) {
+ printf("spdk_nvme_ctrlr_cmd_get_log_page() failed\n");
+ exit(1);
+ }
+
+ return 0;
+}
+
+static int
+get_intel_smart_log_page(struct spdk_nvme_ctrlr *ctrlr)
+{
+ if (spdk_nvme_ctrlr_cmd_get_log_page(ctrlr, SPDK_NVME_INTEL_LOG_SMART, SPDK_NVME_GLOBAL_NS_TAG,
+ &intel_smart_page, sizeof(intel_smart_page), 0, get_log_page_completion, NULL)) {
+ printf("spdk_nvme_ctrlr_cmd_get_log_page() failed\n");
+ exit(1);
+ }
+
+ return 0;
+}
+
+static int
+get_intel_temperature_log_page(struct spdk_nvme_ctrlr *ctrlr)
+{
+ if (spdk_nvme_ctrlr_cmd_get_log_page(ctrlr, SPDK_NVME_INTEL_LOG_TEMPERATURE,
+ SPDK_NVME_GLOBAL_NS_TAG, &intel_temperature_page, sizeof(intel_temperature_page), 0,
+ get_log_page_completion, NULL)) {
+ printf("spdk_nvme_ctrlr_cmd_get_log_page() failed\n");
+ exit(1);
+ }
+ return 0;
+}
+
+static int
+get_intel_md_log_page(struct spdk_nvme_ctrlr *ctrlr)
+{
+ if (spdk_nvme_ctrlr_cmd_get_log_page(ctrlr, SPDK_NVME_INTEL_MARKETING_DESCRIPTION,
+ SPDK_NVME_GLOBAL_NS_TAG, &intel_md_page, sizeof(intel_md_page), 0,
+ get_log_page_completion, NULL)) {
+ printf("spdk_nvme_ctrlr_cmd_get_log_page() failed\n");
+ exit(1);
+ }
+ return 0;
+}
+
+static void
+get_discovery_log_page_header_completion(void *cb_arg, const struct spdk_nvme_cpl *cpl)
+{
+ struct spdk_nvmf_discovery_log_page *new_discovery_page;
+ struct spdk_nvme_ctrlr *ctrlr = cb_arg;
+ uint16_t recfmt;
+ uint64_t remaining;
+ uint64_t offset;
+
+ outstanding_commands--;
+ if (spdk_nvme_cpl_is_error(cpl)) {
+ /* Return without printing anything - this may not be a discovery controller */
+ free(g_discovery_page);
+ g_discovery_page = NULL;
+ return;
+ }
+
+ /* Got the first 4K of the discovery log page */
+ recfmt = from_le16(&g_discovery_page->recfmt);
+ if (recfmt != 0) {
+ printf("Unrecognized discovery log record format %" PRIu16 "\n", recfmt);
+ return;
+ }
+
+ g_discovery_page_numrec = from_le64(&g_discovery_page->numrec);
+
+ /* Pick an arbitrary limit to avoid ridiculously large buffer size. */
+ if (g_discovery_page_numrec > MAX_DISCOVERY_LOG_ENTRIES) {
+ printf("Discovery log has %" PRIu64 " entries - limiting to %" PRIu64 ".\n",
+ g_discovery_page_numrec, MAX_DISCOVERY_LOG_ENTRIES);
+ g_discovery_page_numrec = MAX_DISCOVERY_LOG_ENTRIES;
+ }
+
+ /*
+ * Now that we now how many entries should be in the log page, we can allocate
+ * the full log page buffer.
+ */
+ g_discovery_page_size += g_discovery_page_numrec * sizeof(struct
+ spdk_nvmf_discovery_log_page_entry);
+ new_discovery_page = realloc(g_discovery_page, g_discovery_page_size);
+ if (new_discovery_page == NULL) {
+ free(g_discovery_page);
+ printf("Discovery page allocation failed!\n");
+ return;
+ }
+
+ g_discovery_page = new_discovery_page;
+
+ /* Retrieve the rest of the discovery log page */
+ offset = offsetof(struct spdk_nvmf_discovery_log_page, entries);
+ remaining = g_discovery_page_size - offset;
+ while (remaining) {
+ uint32_t size;
+
+ /* Retrieve up to 4 KB at a time */
+ size = spdk_min(remaining, 4096);
+
+ if (spdk_nvme_ctrlr_cmd_get_log_page(ctrlr, SPDK_NVME_LOG_DISCOVERY,
+ 0, (char *)g_discovery_page + offset, size, offset,
+ get_log_page_completion, NULL)) {
+ printf("spdk_nvme_ctrlr_cmd_get_log_page() failed\n");
+ exit(1);
+ }
+
+ offset += size;
+ remaining -= size;
+ outstanding_commands++;
+ }
+}
+
+static int
+get_discovery_log_page(struct spdk_nvme_ctrlr *ctrlr)
+{
+ /* Allocate the initial discovery log page buffer - this will be resized later. */
+ g_discovery_page_size = sizeof(*g_discovery_page);
+ g_discovery_page = calloc(1, g_discovery_page_size);
+ if (g_discovery_page == NULL) {
+ printf("Discovery log page allocation failed!\n");
+ exit(1);
+ }
+
+ if (spdk_nvme_ctrlr_cmd_get_log_page(ctrlr, SPDK_NVME_LOG_DISCOVERY,
+ 0, g_discovery_page, g_discovery_page_size, 0,
+ get_discovery_log_page_header_completion, ctrlr)) {
+ printf("spdk_nvme_ctrlr_cmd_get_log_page() failed\n");
+ exit(1);
+ }
+
+ return 0;
+}
+
+static void
+get_log_pages(struct spdk_nvme_ctrlr *ctrlr)
+{
+ const struct spdk_nvme_ctrlr_data *cdata;
+ outstanding_commands = 0;
+ bool is_discovery = spdk_nvme_ctrlr_is_discovery(ctrlr);
+
+ cdata = spdk_nvme_ctrlr_get_data(ctrlr);
+
+ if (!is_discovery) {
+ /*
+ * Only attempt to retrieve the following log pages
+ * when the NVM subsystem that's being targeted is
+ * NOT the Discovery Controller which only fields
+ * a Discovery Log Page.
+ */
+ if (get_error_log_page(ctrlr) == 0) {
+ outstanding_commands++;
+ } else {
+ printf("Get Error Log Page failed\n");
+ }
+
+ if (get_health_log_page(ctrlr) == 0) {
+ outstanding_commands++;
+ } else {
+ printf("Get Log Page (SMART/health) failed\n");
+ }
+
+ if (get_firmware_log_page(ctrlr) == 0) {
+ outstanding_commands++;
+ } else {
+ printf("Get Log Page (Firmware Slot Information) failed\n");
+ }
+ }
+
+ if (cdata->lpa.celp) {
+ if (get_cmd_effects_log_page(ctrlr) == 0) {
+ outstanding_commands++;
+ } else {
+ printf("Get Log Page (Commands Supported and Effects) failed\n");
+ }
+ }
+
+ if (cdata->vid == SPDK_PCI_VID_INTEL) {
+ if (spdk_nvme_ctrlr_is_log_page_supported(ctrlr, SPDK_NVME_INTEL_LOG_SMART)) {
+ if (get_intel_smart_log_page(ctrlr) == 0) {
+ outstanding_commands++;
+ } else {
+ printf("Get Log Page (Intel SMART/health) failed\n");
+ }
+ }
+ if (spdk_nvme_ctrlr_is_log_page_supported(ctrlr, SPDK_NVME_INTEL_LOG_TEMPERATURE)) {
+ if (get_intel_temperature_log_page(ctrlr) == 0) {
+ outstanding_commands++;
+ } else {
+ printf("Get Log Page (Intel temperature) failed\n");
+ }
+ }
+ if (spdk_nvme_ctrlr_is_log_page_supported(ctrlr, SPDK_NVME_INTEL_MARKETING_DESCRIPTION)) {
+ if (get_intel_md_log_page(ctrlr) == 0) {
+ outstanding_commands++;
+ } else {
+ printf("Get Log Page (Intel Marketing Description) failed\n");
+ }
+ }
+
+ }
+
+ if (is_discovery && (get_discovery_log_page(ctrlr) == 0)) {
+ outstanding_commands++;
+ }
+
+ while (outstanding_commands) {
+ spdk_nvme_ctrlr_process_admin_completions(ctrlr);
+ }
+}
+
+static int
+get_ocssd_chunk_info_log_page(struct spdk_nvme_ns *ns)
+{
+ struct spdk_nvme_ctrlr *ctrlr = spdk_nvme_ns_get_ctrlr(ns);
+ int nsid = spdk_nvme_ns_get_id(ns);
+ outstanding_commands = 0;
+
+ if (spdk_nvme_ctrlr_cmd_get_log_page(ctrlr, SPDK_OCSSD_LOG_CHUNK_INFO,
+ nsid, &g_ocssd_chunk_info_page, sizeof(g_ocssd_chunk_info_page), 0,
+ get_log_page_completion, NULL) == 0) {
+ outstanding_commands++;
+ } else {
+ printf("get_ocssd_chunk_info_log_page() failed\n");
+ return -1;
+ }
+
+ while (outstanding_commands) {
+ spdk_nvme_ctrlr_process_admin_completions(ctrlr);
+ }
+
+ return 0;
+}
+
+static void
+get_ocssd_geometry(struct spdk_nvme_ns *ns, struct spdk_ocssd_geometry_data *geometry_data)
+{
+ struct spdk_nvme_ctrlr *ctrlr = spdk_nvme_ns_get_ctrlr(ns);
+ int nsid = spdk_nvme_ns_get_id(ns);
+ outstanding_commands = 0;
+
+ if (spdk_nvme_ocssd_ctrlr_cmd_geometry(ctrlr, nsid, geometry_data,
+ sizeof(*geometry_data), get_ocssd_geometry_completion, NULL)) {
+ printf("Get OpenChannel SSD geometry failed\n");
+ exit(1);
+ } else {
+ outstanding_commands++;
+ }
+
+ while (outstanding_commands) {
+ spdk_nvme_ctrlr_process_admin_completions(ctrlr);
+ }
+}
+
+static void
+print_hex_be(const void *v, size_t size)
+{
+ const uint8_t *buf = v;
+
+ while (size--) {
+ printf("%02X", *buf++);
+ }
+}
+
+static void
+print_uint128_hex(uint64_t *v)
+{
+ unsigned long long lo = v[0], hi = v[1];
+ if (hi) {
+ printf("0x%llX%016llX", hi, lo);
+ } else {
+ printf("0x%llX", lo);
+ }
+}
+
+static void
+print_uint128_dec(uint64_t *v)
+{
+ unsigned long long lo = v[0], hi = v[1];
+ if (hi) {
+ /* can't handle large (>64-bit) decimal values for now, so fall back to hex */
+ print_uint128_hex(v);
+ } else {
+ printf("%llu", (unsigned long long)lo);
+ }
+}
+
+/* The len should be <= 8. */
+static void
+print_uint_var_dec(uint8_t *array, unsigned int len)
+{
+ uint64_t result = 0;
+ int i = len;
+
+ while (i > 0) {
+ result += (uint64_t)array[i - 1] << (8 * (i - 1));
+ i--;
+ }
+ printf("%lu", result);
+}
+
+/* Print ASCII string as defined by the NVMe spec */
+static void
+print_ascii_string(const void *buf, size_t size)
+{
+ const uint8_t *str = buf;
+
+ /* Trim trailing spaces */
+ while (size > 0 && str[size - 1] == ' ') {
+ size--;
+ }
+
+ while (size--) {
+ if (*str >= 0x20 && *str <= 0x7E) {
+ printf("%c", *str);
+ } else {
+ printf(".");
+ }
+ str++;
+ }
+}
+
+static void
+print_ocssd_chunk_info(struct spdk_ocssd_chunk_information_entry *chk_info, int chk_num)
+{
+ int i;
+ char *cs_str, *ct_str;
+
+ printf("OCSSD Chunk Info Glance\n");
+ printf("======================\n");
+
+ for (i = 0; i < chk_num; i++) {
+ cs_str = chk_info[i].cs.free ? "Free" :
+ chk_info[i].cs.closed ? "Closed" :
+ chk_info[i].cs.open ? "Open" :
+ chk_info[i].cs.offline ? "Offline" : "Unknown";
+ ct_str = chk_info[i].ct.seq_write ? "Sequential Write" :
+ chk_info[i].ct.rnd_write ? "Random Write" : "Unknown";
+
+ printf("------------\n");
+ printf("Chunk index: %d\n", i);
+ printf("Chunk state: %s(0x%x)\n", cs_str, *(uint8_t *) & (chk_info[i].cs));
+ printf("Chunk type (write mode): %s\n", ct_str);
+ printf("Chunk type (size_deviate): %s\n", chk_info[i].ct.size_deviate ? "Yes" : "No");
+ printf("Wear-level Index: %d\n", chk_info[i].wli);
+ printf("Starting LBA: %ld\n", chk_info[i].slba);
+ printf("Number of blocks in chunk: %ld\n", chk_info[i].cnlb);
+ printf("Write Pointer: %ld\n", chk_info[i].wp);
+ }
+}
+
+static void
+print_ocssd_geometry(struct spdk_ocssd_geometry_data *geometry_data)
+{
+ printf("Namespace OCSSD Geometry\n");
+ printf("=======================\n");
+
+ if (geometry_data->mjr < 2) {
+ printf("Open-Channel Spec version is less than 2.0\n");
+ printf("OC version: maj:%d\n", geometry_data->mjr);
+ return;
+ }
+
+ printf("OC version: maj:%d min:%d\n", geometry_data->mjr, geometry_data->mnr);
+ printf("LBA format:\n");
+ printf(" Group bits: %d\n", geometry_data->lbaf.grp_len);
+ printf(" PU bits: %d\n", geometry_data->lbaf.pu_len);
+ printf(" Chunk bits: %d\n", geometry_data->lbaf.chk_len);
+ printf(" Logical block bits: %d\n", geometry_data->lbaf.lbk_len);
+
+ printf("Media and Controller Capabilities:\n");
+ printf(" Namespace supports Vector Chunk Copy: %s\n",
+ geometry_data->mccap.vec_chk_cpy ? "Supported" : "Not Supported");
+ printf(" Namespace supports multiple resets a free chunk: %s\n",
+ geometry_data->mccap.multi_reset ? "Supported" : "Not Supported");
+
+ printf("Wear-level Index Delta Threshold: %d\n", geometry_data->wit);
+ printf("Groups (channels): %d\n", geometry_data->num_grp);
+ printf("PUs (LUNs) per group: %d\n", geometry_data->num_pu);
+ printf("Chunks per LUN: %d\n", geometry_data->num_chk);
+ printf("Logical blks per chunk: %d\n", geometry_data->clba);
+ printf("MIN write size: %d\n", geometry_data->ws_min);
+ printf("OPT write size: %d\n", geometry_data->ws_opt);
+ printf("Cache min write size: %d\n", geometry_data->mw_cunits);
+ printf("Max open chunks: %d\n", geometry_data->maxoc);
+ printf("Max open chunks per PU: %d\n", geometry_data->maxocpu);
+ printf("\n");
+}
+
+static void
+print_namespace(struct spdk_nvme_ns *ns)
+{
+ const struct spdk_nvme_ns_data *nsdata;
+ const struct spdk_uuid *uuid;
+ uint32_t i;
+ uint32_t flags;
+ char uuid_str[SPDK_UUID_STRING_LEN];
+ uint32_t blocksize;
+
+ nsdata = spdk_nvme_ns_get_data(ns);
+ flags = spdk_nvme_ns_get_flags(ns);
+
+ printf("Namespace ID:%d\n", spdk_nvme_ns_get_id(ns));
+
+ if (g_hex_dump) {
+ hex_dump(nsdata, sizeof(*nsdata));
+ printf("\n");
+ }
+
+ /* This function is only called for active namespaces. */
+ assert(spdk_nvme_ns_is_active(ns));
+
+ printf("Deallocate: %s\n",
+ (flags & SPDK_NVME_NS_DEALLOCATE_SUPPORTED) ? "Supported" : "Not Supported");
+ printf("Deallocated/Unwritten Error: %s\n",
+ nsdata->nsfeat.dealloc_or_unwritten_error ? "Supported" : "Not Supported");
+ printf("Deallocated Read Value: %s\n",
+ nsdata->dlfeat.bits.read_value == SPDK_NVME_DEALLOC_READ_00 ? "All 0x00" :
+ nsdata->dlfeat.bits.read_value == SPDK_NVME_DEALLOC_READ_FF ? "All 0xFF" :
+ "Unknown");
+ printf("Deallocate in Write Zeroes: %s\n",
+ nsdata->dlfeat.bits.write_zero_deallocate ? "Supported" : "Not Supported");
+ printf("Deallocated Guard Field: %s\n",
+ nsdata->dlfeat.bits.guard_value ? "CRC for Read Value" : "0xFFFF");
+ printf("Flush: %s\n",
+ (flags & SPDK_NVME_NS_FLUSH_SUPPORTED) ? "Supported" : "Not Supported");
+ printf("Reservation: %s\n",
+ (flags & SPDK_NVME_NS_RESERVATION_SUPPORTED) ? "Supported" : "Not Supported");
+ if (flags & SPDK_NVME_NS_DPS_PI_SUPPORTED) {
+ printf("End-to-End Data Protection: Supported\n");
+ printf("Protection Type: Type%d\n", nsdata->dps.pit);
+ printf("Protection Information Transferred as: %s\n",
+ nsdata->dps.md_start ? "First 8 Bytes" : "Last 8 Bytes");
+ }
+ if (nsdata->lbaf[nsdata->flbas.format].ms > 0) {
+ printf("Metadata Transferred as: %s\n",
+ nsdata->flbas.extended ? "Extended Data LBA" : "Separate Metadata Buffer");
+ }
+ printf("Namespace Sharing Capabilities: %s\n",
+ nsdata->nmic.can_share ? "Multiple Controllers" : "Private");
+ blocksize = 1 << nsdata->lbaf[nsdata->flbas.format].lbads;
+ printf("Size (in LBAs): %lld (%lldGiB)\n",
+ (long long)nsdata->nsze,
+ (long long)nsdata->nsze * blocksize / 1024 / 1024 / 1024);
+ printf("Capacity (in LBAs): %lld (%lldGiB)\n",
+ (long long)nsdata->ncap,
+ (long long)nsdata->ncap * blocksize / 1024 / 1024 / 1024);
+ printf("Utilization (in LBAs): %lld (%lldGiB)\n",
+ (long long)nsdata->nuse,
+ (long long)nsdata->nuse * blocksize / 1024 / 1024 / 1024);
+ if (nsdata->noiob) {
+ printf("Optimal I/O Boundary: %u blocks\n", nsdata->noiob);
+ }
+ if (!spdk_mem_all_zero(nsdata->nguid, sizeof(nsdata->nguid))) {
+ printf("NGUID: ");
+ print_hex_be(nsdata->nguid, sizeof(nsdata->nguid));
+ printf("\n");
+ }
+ if (!spdk_mem_all_zero(&nsdata->eui64, sizeof(nsdata->eui64))) {
+ printf("EUI64: ");
+ print_hex_be(&nsdata->eui64, sizeof(nsdata->eui64));
+ printf("\n");
+ }
+ uuid = spdk_nvme_ns_get_uuid(ns);
+ if (uuid) {
+ spdk_uuid_fmt_lower(uuid_str, sizeof(uuid_str), uuid);
+ printf("UUID: %s\n", uuid_str);
+ }
+ printf("Thin Provisioning: %s\n",
+ nsdata->nsfeat.thin_prov ? "Supported" : "Not Supported");
+ printf("Per-NS Atomic Units: %s\n",
+ nsdata->nsfeat.ns_atomic_write_unit ? "Yes" : "No");
+ if (nsdata->nsfeat.ns_atomic_write_unit) {
+ if (nsdata->nawun) {
+ printf(" Atomic Write Unit (Normal): %d\n", nsdata->nawun + 1);
+ }
+
+ if (nsdata->nawupf) {
+ printf(" Atomic Write Unit (PFail): %d\n", nsdata->nawupf + 1);
+ }
+
+ if (nsdata->nacwu) {
+ printf(" Atomic Compare & Write Unit: %d\n", nsdata->nacwu + 1);
+ }
+
+ printf(" Atomic Boundary Size (Normal): %d\n", nsdata->nabsn);
+ printf(" Atomic Boundary Size (PFail): %d\n", nsdata->nabspf);
+ printf(" Atomic Boundary Offset: %d\n", nsdata->nabo);
+ }
+
+ printf("NGUID/EUI64 Never Reused: %s\n",
+ nsdata->nsfeat.guid_never_reused ? "Yes" : "No");
+ printf("Number of LBA Formats: %d\n", nsdata->nlbaf + 1);
+ printf("Current LBA Format: LBA Format #%02d\n",
+ nsdata->flbas.format);
+ for (i = 0; i <= nsdata->nlbaf; i++)
+ printf("LBA Format #%02d: Data Size: %5d Metadata Size: %5d\n",
+ i, 1 << nsdata->lbaf[i].lbads, nsdata->lbaf[i].ms);
+ printf("\n");
+
+ if (spdk_nvme_ctrlr_is_ocssd_supported(spdk_nvme_ns_get_ctrlr(ns))) {
+ get_ocssd_geometry(ns, &geometry_data);
+ print_ocssd_geometry(&geometry_data);
+ get_ocssd_chunk_info_log_page(ns);
+ print_ocssd_chunk_info(g_ocssd_chunk_info_page, NUM_CHUNK_INFO_ENTRIES);
+ }
+}
+
+static const char *
+admin_opc_name(uint8_t opc)
+{
+ switch (opc) {
+ case SPDK_NVME_OPC_DELETE_IO_SQ:
+ return "Delete I/O Submission Queue";
+ case SPDK_NVME_OPC_CREATE_IO_SQ:
+ return "Create I/O Submission Queue";
+ case SPDK_NVME_OPC_GET_LOG_PAGE:
+ return "Get Log Page";
+ case SPDK_NVME_OPC_DELETE_IO_CQ:
+ return "Delete I/O Completion Queue";
+ case SPDK_NVME_OPC_CREATE_IO_CQ:
+ return "Create I/O Completion Queue";
+ case SPDK_NVME_OPC_IDENTIFY:
+ return "Identify";
+ case SPDK_NVME_OPC_ABORT:
+ return "Abort";
+ case SPDK_NVME_OPC_SET_FEATURES:
+ return "Set Features";
+ case SPDK_NVME_OPC_GET_FEATURES:
+ return "Get Features";
+ case SPDK_NVME_OPC_ASYNC_EVENT_REQUEST:
+ return "Asynchronous Event Request";
+ case SPDK_NVME_OPC_NS_MANAGEMENT:
+ return "Namespace Management";
+ case SPDK_NVME_OPC_FIRMWARE_COMMIT:
+ return "Firmware Commit";
+ case SPDK_NVME_OPC_FIRMWARE_IMAGE_DOWNLOAD:
+ return "Firmware Image Download";
+ case SPDK_NVME_OPC_DEVICE_SELF_TEST:
+ return "Device Self-test";
+ case SPDK_NVME_OPC_NS_ATTACHMENT:
+ return "Namespace Attachment";
+ case SPDK_NVME_OPC_KEEP_ALIVE:
+ return "Keep Alive";
+ case SPDK_NVME_OPC_DIRECTIVE_SEND:
+ return "Directive Send";
+ case SPDK_NVME_OPC_DIRECTIVE_RECEIVE:
+ return "Directive Receive";
+ case SPDK_NVME_OPC_VIRTUALIZATION_MANAGEMENT:
+ return "Virtualization Management";
+ case SPDK_NVME_OPC_NVME_MI_SEND:
+ return "NVMe-MI Send";
+ case SPDK_NVME_OPC_NVME_MI_RECEIVE:
+ return "NVMe-MI Receive";
+ case SPDK_NVME_OPC_DOORBELL_BUFFER_CONFIG:
+ return "Doorbell Buffer Config";
+ case SPDK_NVME_OPC_FORMAT_NVM:
+ return "Format NVM";
+ case SPDK_NVME_OPC_SECURITY_SEND:
+ return "Security Send";
+ case SPDK_NVME_OPC_SECURITY_RECEIVE:
+ return "Security Receive";
+ case SPDK_NVME_OPC_SANITIZE:
+ return "Sanitize";
+ default:
+ if (opc >= 0xC0) {
+ return "Vendor specific";
+ }
+ return "Unknown";
+ }
+}
+
+static const char *
+io_opc_name(uint8_t opc)
+{
+ switch (opc) {
+ case SPDK_NVME_OPC_FLUSH:
+ return "Flush";
+ case SPDK_NVME_OPC_WRITE:
+ return "Write";
+ case SPDK_NVME_OPC_READ:
+ return "Read";
+ case SPDK_NVME_OPC_WRITE_UNCORRECTABLE:
+ return "Write Uncorrectable";
+ case SPDK_NVME_OPC_COMPARE:
+ return "Compare";
+ case SPDK_NVME_OPC_WRITE_ZEROES:
+ return "Write Zeroes";
+ case SPDK_NVME_OPC_DATASET_MANAGEMENT:
+ return "Dataset Management";
+ case SPDK_NVME_OPC_RESERVATION_REGISTER:
+ return "Reservation Register";
+ case SPDK_NVME_OPC_RESERVATION_REPORT:
+ return "Reservation Report";
+ case SPDK_NVME_OPC_RESERVATION_ACQUIRE:
+ return "Reservation Acquire";
+ case SPDK_NVME_OPC_RESERVATION_RELEASE:
+ return "Reservation Release";
+ default:
+ if (opc >= 0x80) {
+ return "Vendor specific";
+ }
+ return "Unknown";
+ }
+}
+
+static void
+print_controller(struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_transport_id *trid)
+{
+ const struct spdk_nvme_ctrlr_data *cdata;
+ union spdk_nvme_cap_register cap;
+ union spdk_nvme_vs_register vs;
+ union spdk_nvme_cmbsz_register cmbsz;
+ uint8_t str[512];
+ uint32_t i;
+ struct spdk_nvme_error_information_entry *error_entry;
+ struct spdk_pci_addr pci_addr;
+ struct spdk_pci_device *pci_dev;
+ struct spdk_pci_id pci_id;
+ uint32_t nsid;
+
+ cap = spdk_nvme_ctrlr_get_regs_cap(ctrlr);
+ vs = spdk_nvme_ctrlr_get_regs_vs(ctrlr);
+ cmbsz = spdk_nvme_ctrlr_get_regs_cmbsz(ctrlr);
+
+ if (!spdk_nvme_ctrlr_is_discovery(ctrlr)) {
+ /*
+ * Discovery Controller only supports the
+ * IDENTIFY and GET_LOG_PAGE cmd set, so only
+ * attempt GET_FEATURES when NOT targeting a
+ * Discovery Controller.
+ */
+ get_features(ctrlr);
+ }
+ get_log_pages(ctrlr);
+
+ cdata = spdk_nvme_ctrlr_get_data(ctrlr);
+
+ printf("=====================================================\n");
+ if (trid->trtype != SPDK_NVME_TRANSPORT_PCIE) {
+ printf("NVMe over Fabrics controller at %s:%s: %s\n",
+ trid->traddr, trid->trsvcid, trid->subnqn);
+ } else {
+ if (spdk_pci_addr_parse(&pci_addr, trid->traddr) != 0) {
+ return;
+ }
+
+ pci_dev = spdk_nvme_ctrlr_get_pci_device(ctrlr);
+ if (!pci_dev) {
+ return;
+ }
+
+ pci_id = spdk_pci_device_get_id(pci_dev);
+
+ printf("NVMe Controller at %04x:%02x:%02x.%x [%04x:%04x]\n",
+ pci_addr.domain, pci_addr.bus,
+ pci_addr.dev, pci_addr.func,
+ pci_id.vendor_id, pci_id.device_id);
+ }
+ printf("=====================================================\n");
+
+ if (g_hex_dump) {
+ hex_dump(cdata, sizeof(*cdata));
+ printf("\n");
+ }
+
+ printf("Controller Capabilities/Features\n");
+ printf("================================\n");
+ printf("Vendor ID: %04x\n", cdata->vid);
+ printf("Subsystem Vendor ID: %04x\n", cdata->ssvid);
+ printf("Serial Number: ");
+ print_ascii_string(cdata->sn, sizeof(cdata->sn));
+ printf("\n");
+ printf("Model Number: ");
+ print_ascii_string(cdata->mn, sizeof(cdata->mn));
+ printf("\n");
+ printf("Firmware Version: ");
+ print_ascii_string(cdata->fr, sizeof(cdata->fr));
+ printf("\n");
+ printf("Recommended Arb Burst: %d\n", cdata->rab);
+ printf("IEEE OUI Identifier: %02x %02x %02x\n",
+ cdata->ieee[0], cdata->ieee[1], cdata->ieee[2]);
+ printf("Multi-path I/O\n");
+ printf(" May have multiple subsystem ports: %s\n", cdata->cmic.multi_port ? "Yes" : "No");
+ printf(" May be connected to multiple hosts: %s\n", cdata->cmic.multi_host ? "Yes" : "No");
+ printf(" Associated with SR-IOV VF: %s\n", cdata->cmic.sr_iov ? "Yes" : "No");
+ printf("Max Data Transfer Size: ");
+ if (cdata->mdts == 0) {
+ printf("Unlimited\n");
+ } else {
+ printf("%" PRIu64 "\n", (uint64_t)1 << (12 + cap.bits.mpsmin + cdata->mdts));
+ }
+ printf("Max Number of Namespaces: %d\n", cdata->nn);
+ if (features[SPDK_NVME_FEAT_ERROR_RECOVERY].valid) {
+ unsigned tler = features[SPDK_NVME_FEAT_ERROR_RECOVERY].result & 0xFFFF;
+ printf("Error Recovery Timeout: ");
+ if (tler == 0) {
+ printf("Unlimited\n");
+ } else {
+ printf("%u milliseconds\n", tler * 100);
+ }
+ }
+ printf("NVMe Specification Version (VS): %u.%u", vs.bits.mjr, vs.bits.mnr);
+ if (vs.bits.ter) {
+ printf(".%u", vs.bits.ter);
+ }
+ printf("\n");
+ if (cdata->ver.raw != 0) {
+ printf("NVMe Specification Version (Identify): %u.%u", cdata->ver.bits.mjr, cdata->ver.bits.mnr);
+ if (cdata->ver.bits.ter) {
+ printf(".%u", cdata->ver.bits.ter);
+ }
+ printf("\n");
+ }
+
+ printf("Maximum Queue Entries: %u\n", cap.bits.mqes + 1);
+ printf("Contiguous Queues Required: %s\n", cap.bits.cqr ? "Yes" : "No");
+ printf("Arbitration Mechanisms Supported\n");
+ printf(" Weighted Round Robin: %s\n",
+ cap.bits.ams & SPDK_NVME_CAP_AMS_WRR ? "Supported" : "Not Supported");
+ printf(" Vendor Specific: %s\n",
+ cap.bits.ams & SPDK_NVME_CAP_AMS_VS ? "Supported" : "Not Supported");
+ printf("Reset Timeout: %" PRIu64 " ms\n", (uint64_t)500 * cap.bits.to);
+ printf("Doorbell Stride: %" PRIu64 " bytes\n",
+ (uint64_t)1 << (2 + cap.bits.dstrd));
+ printf("NVM Subsystem Reset: %s\n",
+ cap.bits.nssrs ? "Supported" : "Not Supported");
+ printf("Command Sets Supported\n");
+ printf(" NVM Command Set: %s\n",
+ cap.bits.css & SPDK_NVME_CAP_CSS_NVM ? "Supported" : "Not Supported");
+ printf("Boot Partition: %s\n",
+ cap.bits.bps ? "Supported" : "Not Supported");
+ printf("Memory Page Size Minimum: %" PRIu64 " bytes\n",
+ (uint64_t)1 << (12 + cap.bits.mpsmin));
+ printf("Memory Page Size Maximum: %" PRIu64 " bytes\n",
+ (uint64_t)1 << (12 + cap.bits.mpsmax));
+ printf("Optional Asynchronous Events Supported\n");
+ printf(" Namespace Attribute Notices: %s\n",
+ cdata->oaes.ns_attribute_notices ? "Supported" : "Not Supported");
+ printf(" Firmware Activation Notices: %s\n",
+ cdata->oaes.fw_activation_notices ? "Supported" : "Not Supported");
+
+ printf("128-bit Host Identifier: %s\n",
+ cdata->ctratt.host_id_exhid_supported ? "Supported" : "Not Supported");
+ printf("\n");
+
+ printf("Controller Memory Buffer Support\n");
+ printf("================================\n");
+ if (cmbsz.raw != 0) {
+ uint64_t size = cmbsz.bits.sz;
+
+ /* Convert the size to bytes by multiplying by the granularity.
+ By spec, szu is at most 6 and sz is 20 bits, so size requires
+ at most 56 bits. */
+ size *= (0x1000 << (cmbsz.bits.szu * 4));
+
+ printf("Supported: Yes\n");
+ printf("Total Size: %lu bytes\n", size);
+ printf("Submission Queues in CMB: %s\n",
+ cmbsz.bits.sqs ? "Supported" : "Not Supported");
+ printf("Completion Queues in CMB: %s\n",
+ cmbsz.bits.cqs ? "Supported" : "Not Supported");
+ printf("Read data and metadata in CMB %s\n",
+ cmbsz.bits.rds ? "Supported" : "Not Supported");
+ printf("Write data and metadata in CMB: %s\n",
+ cmbsz.bits.wds ? "Supported" : "Not Supported");
+ } else {
+ printf("Supported: No\n");
+ }
+ printf("\n");
+
+ printf("Admin Command Set Attributes\n");
+ printf("============================\n");
+ printf("Security Send/Receive: %s\n",
+ cdata->oacs.security ? "Supported" : "Not Supported");
+ printf("Format NVM: %s\n",
+ cdata->oacs.format ? "Supported" : "Not Supported");
+ printf("Firmware Activate/Download: %s\n",
+ cdata->oacs.firmware ? "Supported" : "Not Supported");
+ printf("Namespace Management: %s\n",
+ cdata->oacs.ns_manage ? "Supported" : "Not Supported");
+ printf("Device Self-Test: %s\n",
+ cdata->oacs.device_self_test ? "Supported" : "Not Supported");
+ printf("Directives: %s\n",
+ cdata->oacs.directives ? "Supported" : "Not Supported");
+ printf("NVMe-MI: %s\n",
+ cdata->oacs.nvme_mi ? "Supported" : "Not Supported");
+ printf("Virtualization Management: %s\n",
+ cdata->oacs.virtualization_management ? "Supported" : "Not Supported");
+ printf("Doorbell Buffer Config: %s\n",
+ cdata->oacs.doorbell_buffer_config ? "Supported" : "Not Supported");
+ printf("Abort Command Limit: %d\n", cdata->acl + 1);
+ printf("Async Event Request Limit: %d\n", cdata->aerl + 1);
+ printf("Number of Firmware Slots: ");
+ if (cdata->oacs.firmware != 0) {
+ printf("%d\n", cdata->frmw.num_slots);
+ } else {
+ printf("N/A\n");
+ }
+ printf("Firmware Slot 1 Read-Only: ");
+ if (cdata->oacs.firmware != 0) {
+ printf("%s\n", cdata->frmw.slot1_ro ? "Yes" : "No");
+ } else {
+ printf("N/A\n");
+ }
+ if (cdata->fwug == 0x00) {
+ printf("Firmware Update Granularity: No Information Provided\n");
+ } else if (cdata->fwug == 0xFF) {
+ printf("Firmware Update Granularity: No Restriction\n");
+ } else {
+ printf("Firmware Update Granularity: %u KiB\n",
+ cdata->fwug * 4);
+ }
+ printf("Per-Namespace SMART Log: %s\n",
+ cdata->lpa.ns_smart ? "Yes" : "No");
+ printf("Command Effects Log Page: %s\n",
+ cdata->lpa.celp ? "Supported" : "Not Supported");
+ printf("Get Log Page Extended Data: %s\n",
+ cdata->lpa.edlp ? "Supported" : "Not Supported");
+ printf("Telemetry Log Pages: %s\n",
+ cdata->lpa.telemetry ? "Supported" : "Not Supported");
+ printf("Error Log Page Entries Supported: %d\n", cdata->elpe + 1);
+ if (cdata->kas == 0) {
+ printf("Keep Alive: Not Supported\n");
+ } else {
+ printf("Keep Alive: Supported\n");
+ printf("Keep Alive Granularity: %u ms\n",
+ cdata->kas * 100);
+ }
+ printf("\n");
+
+ printf("NVM Command Set Attributes\n");
+ printf("==========================\n");
+ printf("Submission Queue Entry Size\n");
+ printf(" Max: %d\n", 1 << cdata->sqes.max);
+ printf(" Min: %d\n", 1 << cdata->sqes.min);
+ printf("Completion Queue Entry Size\n");
+ printf(" Max: %d\n", 1 << cdata->cqes.max);
+ printf(" Min: %d\n", 1 << cdata->cqes.min);
+ printf("Number of Namespaces: %d\n", cdata->nn);
+ printf("Compare Command: %s\n",
+ cdata->oncs.compare ? "Supported" : "Not Supported");
+ printf("Write Uncorrectable Command: %s\n",
+ cdata->oncs.write_unc ? "Supported" : "Not Supported");
+ printf("Dataset Management Command: %s\n",
+ cdata->oncs.dsm ? "Supported" : "Not Supported");
+ printf("Write Zeroes Command: %s\n",
+ cdata->oncs.write_zeroes ? "Supported" : "Not Supported");
+ printf("Set Features Save Field: %s\n",
+ cdata->oncs.set_features_save ? "Supported" : "Not Supported");
+ printf("Reservations: %s\n",
+ cdata->oncs.reservations ? "Supported" : "Not Supported");
+ printf("Timestamp: %s\n",
+ cdata->oncs.timestamp ? "Supported" : "Not Supported");
+ printf("Volatile Write Cache: %s\n",
+ cdata->vwc.present ? "Present" : "Not Present");
+ printf("Atomic Write Unit (Normal): %d\n", cdata->awun + 1);
+ printf("Atomic Write Unit (PFail): %d\n", cdata->awupf + 1);
+ printf("Atomic Compare & Write Unit: %d\n", cdata->acwu + 1);
+ printf("Fused Compare & Write: %s\n",
+ cdata->fuses.compare_and_write ? "Supported" : "Not Supported");
+ printf("Scatter-Gather List\n");
+ printf(" SGL Command Set: %s\n",
+ cdata->sgls.supported == SPDK_NVME_SGLS_SUPPORTED ? "Supported" :
+ cdata->sgls.supported == SPDK_NVME_SGLS_SUPPORTED_DWORD_ALIGNED ? "Supported (Dword aligned)" :
+ "Not Supported");
+ printf(" SGL Keyed: %s\n",
+ cdata->sgls.keyed_sgl ? "Supported" : "Not Supported");
+ printf(" SGL Bit Bucket Descriptor: %s\n",
+ cdata->sgls.bit_bucket_descriptor ? "Supported" : "Not Supported");
+ printf(" SGL Metadata Pointer: %s\n",
+ cdata->sgls.metadata_pointer ? "Supported" : "Not Supported");
+ printf(" Oversized SGL: %s\n",
+ cdata->sgls.oversized_sgl ? "Supported" : "Not Supported");
+ printf(" SGL Metadata Address: %s\n",
+ cdata->sgls.metadata_address ? "Supported" : "Not Supported");
+ printf(" SGL Offset: %s\n",
+ cdata->sgls.sgl_offset ? "Supported" : "Not Supported");
+ printf(" Transport SGL Data Block: %s\n",
+ cdata->sgls.transport_sgl ? "Supported" : "Not Supported");
+ printf("Replay Protected Memory Block:");
+ if (cdata->rpmbs.num_rpmb_units > 0) {
+ printf(" Supported\n");
+ printf(" Number of RPMB Units: %d\n", cdata->rpmbs.num_rpmb_units);
+ printf(" Authentication Method: %s\n", cdata->rpmbs.auth_method == 0 ? "HMAC SHA-256" : "Unknown");
+ printf(" Total Size (in 128KB units) = %d\n", cdata->rpmbs.total_size + 1);
+ printf(" Access Size (in 512B units) = %d\n", cdata->rpmbs.access_size + 1);
+ } else {
+ printf(" Not Supported\n");
+ }
+ printf("\n");
+
+ printf("Firmware Slot Information\n");
+ printf("=========================\n");
+ if (g_hex_dump) {
+ hex_dump(&firmware_page, sizeof(firmware_page));
+ printf("\n");
+ }
+ printf("Active slot: %u\n", firmware_page.afi.active_slot);
+ if (firmware_page.afi.next_reset_slot) {
+ printf("Next controller reset slot: %u\n", firmware_page.afi.next_reset_slot);
+ }
+ for (i = 0; i < 7; i++) {
+ if (!spdk_mem_all_zero(firmware_page.revision[i], sizeof(firmware_page.revision[i]))) {
+ printf("Slot %u Firmware Revision: ", i + 1);
+ print_ascii_string(firmware_page.revision[i], sizeof(firmware_page.revision[i]));
+ printf("\n");
+ }
+ }
+ printf("\n");
+
+ if (cdata->lpa.celp) {
+ printf("Commands Supported and Effects\n");
+ printf("==============================\n");
+
+ if (g_hex_dump) {
+ hex_dump(&cmd_effects_log_page, sizeof(cmd_effects_log_page));
+ printf("\n");
+ }
+
+ printf("Admin Commands\n");
+ printf("--------------\n");
+ for (i = 0; i < SPDK_COUNTOF(cmd_effects_log_page.admin_cmds_supported); i++) {
+ struct spdk_nvme_cmds_and_effect_entry *cmd = &cmd_effects_log_page.admin_cmds_supported[i];
+ if (cmd->csupp) {
+ printf("%30s (%02Xh): Supported %s%s%s%s%s\n",
+ admin_opc_name(i), i,
+ cmd->lbcc ? "LBA-Change " : "",
+ cmd->ncc ? "NS-Cap-Change " : "",
+ cmd->nic ? "NS-Inventory-Change " : "",
+ cmd->ccc ? "Ctrlr-Cap-Change " : "",
+ cmd->cse == 0 ? "" : cmd->cse == 1 ? "Per-NS-Exclusive" : cmd->cse == 2 ? "All-NS-Exclusive" : "");
+ }
+ }
+
+ printf("I/O Commands\n");
+ printf("------------\n");
+ for (i = 0; i < SPDK_COUNTOF(cmd_effects_log_page.io_cmds_supported); i++) {
+ struct spdk_nvme_cmds_and_effect_entry *cmd = &cmd_effects_log_page.io_cmds_supported[i];
+ if (cmd->csupp) {
+ printf("%30s (%02Xh): Supported %s%s%s%s%s\n",
+ io_opc_name(i), i,
+ cmd->lbcc ? "LBA-Change " : "",
+ cmd->ncc ? "NS-Cap-Change " : "",
+ cmd->nic ? "NS-Inventory-Change " : "",
+ cmd->ccc ? "Ctrlr-Cap-Change " : "",
+ cmd->cse == 0 ? "" : cmd->cse == 1 ? "Per-NS-Exclusive" : cmd->cse == 2 ? "All-NS-Exclusive" : "");
+ }
+ }
+ printf("\n");
+ }
+
+ printf("Error Log\n");
+ printf("=========\n");
+ for (i = 0; i <= cdata->elpe; i++) {
+ error_entry = &error_page[i];
+ if (error_entry->error_count == 0) {
+ continue;
+ }
+ if (i != 0) {
+ printf("-----------\n");
+ }
+
+ printf("Entry: %u\n", i);
+ printf("Error Count: 0x%"PRIx64"\n", error_entry->error_count);
+ printf("Submission Queue Id: 0x%x\n", error_entry->sqid);
+ printf("Command Id: 0x%x\n", error_entry->cid);
+ printf("Phase Bit: %x\n", error_entry->status.p);
+ printf("Status Code: 0x%x\n", error_entry->status.sc);
+ printf("Status Code Type: 0x%x\n", error_entry->status.sct);
+ printf("Do Not Retry: %x\n", error_entry->status.dnr);
+ printf("Error Location: 0x%x\n", error_entry->error_location);
+ printf("LBA: 0x%"PRIx64"\n", error_entry->lba);
+ printf("Namespace: 0x%x\n", error_entry->nsid);
+ printf("Vendor Log Page: 0x%x\n", error_entry->vendor_specific);
+
+ }
+ printf("\n");
+
+ if (features[SPDK_NVME_FEAT_ARBITRATION].valid) {
+ uint32_t arb = features[SPDK_NVME_FEAT_ARBITRATION].result;
+ unsigned ab, lpw, mpw, hpw;
+
+ ab = arb & 0x7;
+ lpw = ((arb >> 8) & 0xFF) + 1;
+ mpw = ((arb >> 16) & 0xFF) + 1;
+ hpw = ((arb >> 24) & 0xFF) + 1;
+
+ printf("Arbitration\n");
+ printf("===========\n");
+ printf("Arbitration Burst: ");
+ if (ab == 0x7) {
+ printf("no limit\n");
+ } else {
+ printf("%u\n", 1u << ab);
+ }
+
+ if (cap.bits.ams & SPDK_NVME_CAP_AMS_WRR) {
+ printf("Low Priority Weight: %u\n", lpw);
+ printf("Medium Priority Weight: %u\n", mpw);
+ printf("High Priority Weight: %u\n", hpw);
+ }
+ printf("\n");
+ }
+
+ if (features[SPDK_NVME_FEAT_POWER_MANAGEMENT].valid) {
+ unsigned ps = features[SPDK_NVME_FEAT_POWER_MANAGEMENT].result & 0x1F;
+ printf("Power Management\n");
+ printf("================\n");
+ printf("Number of Power States: %u\n", cdata->npss + 1);
+ printf("Current Power State: Power State #%u\n", ps);
+ for (i = 0; i <= cdata->npss; i++) {
+ const struct spdk_nvme_power_state *psd = &cdata->psd[i];
+ printf("Power State #%u: ", i);
+ if (psd->mps) {
+ /* MP scale is 0.0001 W */
+ printf("Max Power: %u.%04u W\n",
+ psd->mp / 10000,
+ psd->mp % 10000);
+ } else {
+ /* MP scale is 0.01 W */
+ printf("Max Power: %3u.%02u W\n",
+ psd->mp / 100,
+ psd->mp % 100);
+ }
+ /* TODO: print other power state descriptor fields */
+ }
+ printf("Non-Operational Permissive Mode: %s\n",
+ cdata->ctratt.non_operational_power_state_permissive_mode ? "Supported" : "Not Supported");
+ printf("\n");
+ }
+
+ if (features[SPDK_NVME_FEAT_TEMPERATURE_THRESHOLD].valid) {
+ printf("Health Information\n");
+ printf("==================\n");
+
+ if (g_hex_dump) {
+ hex_dump(&health_page, sizeof(health_page));
+ printf("\n");
+ }
+
+ printf("Critical Warnings:\n");
+ printf(" Available Spare Space: %s\n",
+ health_page.critical_warning.bits.available_spare ? "WARNING" : "OK");
+ printf(" Temperature: %s\n",
+ health_page.critical_warning.bits.temperature ? "WARNING" : "OK");
+ printf(" Device Reliability: %s\n",
+ health_page.critical_warning.bits.device_reliability ? "WARNING" : "OK");
+ printf(" Read Only: %s\n",
+ health_page.critical_warning.bits.read_only ? "Yes" : "No");
+ printf(" Volatile Memory Backup: %s\n",
+ health_page.critical_warning.bits.volatile_memory_backup ? "WARNING" : "OK");
+ printf("Current Temperature: %u Kelvin (%d Celsius)\n",
+ health_page.temperature,
+ (int)health_page.temperature - 273);
+ printf("Temperature Threshold: %u Kelvin (%d Celsius)\n",
+ features[SPDK_NVME_FEAT_TEMPERATURE_THRESHOLD].result,
+ (int)features[SPDK_NVME_FEAT_TEMPERATURE_THRESHOLD].result - 273);
+ printf("Available Spare: %u%%\n", health_page.available_spare);
+ printf("Available Spare Threshold: %u%%\n", health_page.available_spare_threshold);
+ printf("Life Percentage Used: %u%%\n", health_page.percentage_used);
+ printf("Data Units Read: ");
+ print_uint128_dec(health_page.data_units_read);
+ printf("\n");
+ printf("Data Units Written: ");
+ print_uint128_dec(health_page.data_units_written);
+ printf("\n");
+ printf("Host Read Commands: ");
+ print_uint128_dec(health_page.host_read_commands);
+ printf("\n");
+ printf("Host Write Commands: ");
+ print_uint128_dec(health_page.host_write_commands);
+ printf("\n");
+ printf("Controller Busy Time: ");
+ print_uint128_dec(health_page.controller_busy_time);
+ printf(" minutes\n");
+ printf("Power Cycles: ");
+ print_uint128_dec(health_page.power_cycles);
+ printf("\n");
+ printf("Power On Hours: ");
+ print_uint128_dec(health_page.power_on_hours);
+ printf(" hours\n");
+ printf("Unsafe Shutdowns: ");
+ print_uint128_dec(health_page.unsafe_shutdowns);
+ printf("\n");
+ printf("Unrecoverable Media Errors: ");
+ print_uint128_dec(health_page.media_errors);
+ printf("\n");
+ printf("Lifetime Error Log Entries: ");
+ print_uint128_dec(health_page.num_error_info_log_entries);
+ printf("\n");
+ printf("Warning Temperature Time: %u minutes\n", health_page.warning_temp_time);
+ printf("Critical Temperature Time: %u minutes\n", health_page.critical_temp_time);
+ for (i = 0; i < 8; i++) {
+ if (health_page.temp_sensor[i] != 0) {
+ printf("Temperature Sensor %d: %u Kelvin (%d Celsius)\n",
+ i + 1, health_page.temp_sensor[i],
+ (int)health_page.temp_sensor[i] - 273);
+ }
+ }
+ printf("\n");
+ }
+
+ if (features[SPDK_NVME_FEAT_NUMBER_OF_QUEUES].valid) {
+ uint32_t result = features[SPDK_NVME_FEAT_NUMBER_OF_QUEUES].result;
+
+ printf("Number of Queues\n");
+ printf("================\n");
+ printf("Number of I/O Submission Queues: %u\n", (result & 0xFFFF) + 1);
+ printf("Number of I/O Completion Queues: %u\n", (result & 0xFFFF0000 >> 16) + 1);
+ printf("\n");
+ }
+
+ if (features[SPDK_OCSSD_FEAT_MEDIA_FEEDBACK].valid) {
+ uint32_t result = features[SPDK_OCSSD_FEAT_MEDIA_FEEDBACK].result;
+
+ printf("OCSSD Media Feedback\n");
+ printf("=======================\n");
+ printf("High ECC status: %u\n", (result & 0x1));
+ printf("Vector High ECC status: %u\n", (result & 0x2 >> 1));
+ printf("\n");
+ }
+
+ if (cdata->hctma.bits.supported) {
+ printf("Host Controlled Thermal Management\n");
+ printf("==================================\n");
+ printf("Minimum Thermal Management Temperature: ");
+ if (cdata->mntmt) {
+ printf("%u Kelvin (%d Celsius)\n", cdata->mntmt, (int)cdata->mntmt - 273);
+ } else {
+ printf("Not Reported\n");
+ }
+ printf("Maximum Thermal Managment Temperature: ");
+ if (cdata->mxtmt) {
+ printf("%u Kelvin (%d Celsius)\n", cdata->mxtmt, (int)cdata->mxtmt - 273);
+ } else {
+ printf("Not Reported\n");
+ }
+ printf("\n");
+ }
+
+ if (spdk_nvme_ctrlr_is_log_page_supported(ctrlr, SPDK_NVME_INTEL_LOG_SMART)) {
+ size_t i = 0;
+
+ printf("Intel Health Information\n");
+ printf("==================\n");
+ for (i = 0;
+ i < SPDK_COUNTOF(intel_smart_page.attributes); i++) {
+ if (intel_smart_page.attributes[i].code == SPDK_NVME_INTEL_SMART_PROGRAM_FAIL_COUNT) {
+ printf("Program Fail Count:\n");
+ printf(" Normalized Value : %d\n",
+ intel_smart_page.attributes[i].normalized_value);
+ printf(" Current Raw Value: ");
+ print_uint_var_dec(intel_smart_page.attributes[i].raw_value, 6);
+ printf("\n");
+ }
+ if (intel_smart_page.attributes[i].code == SPDK_NVME_INTEL_SMART_ERASE_FAIL_COUNT) {
+ printf("Erase Fail Count:\n");
+ printf(" Normalized Value : %d\n",
+ intel_smart_page.attributes[i].normalized_value);
+ printf(" Current Raw Value: ");
+ print_uint_var_dec(intel_smart_page.attributes[i].raw_value, 6);
+ printf("\n");
+ }
+ if (intel_smart_page.attributes[i].code == SPDK_NVME_INTEL_SMART_WEAR_LEVELING_COUNT) {
+ printf("Wear Leveling Count:\n");
+ printf(" Normalized Value : %d\n",
+ intel_smart_page.attributes[i].normalized_value);
+ printf(" Current Raw Value:\n");
+ printf(" Min: ");
+ print_uint_var_dec(&intel_smart_page.attributes[i].raw_value[0], 2);
+ printf("\n");
+ printf(" Max: ");
+ print_uint_var_dec(&intel_smart_page.attributes[i].raw_value[2], 2);
+ printf("\n");
+ printf(" Avg: ");
+ print_uint_var_dec(&intel_smart_page.attributes[i].raw_value[4], 2);
+ printf("\n");
+ }
+ if (intel_smart_page.attributes[i].code == SPDK_NVME_INTEL_SMART_E2E_ERROR_COUNT) {
+ printf("End to End Error Detection Count:\n");
+ printf(" Normalized Value : %d\n",
+ intel_smart_page.attributes[i].normalized_value);
+ printf(" Current Raw Value: ");
+ print_uint_var_dec(intel_smart_page.attributes[i].raw_value, 6);
+ printf("\n");
+ }
+ if (intel_smart_page.attributes[i].code == SPDK_NVME_INTEL_SMART_CRC_ERROR_COUNT) {
+ printf("CRC Error Count:\n");
+ printf(" Normalized Value : %d\n",
+ intel_smart_page.attributes[i].normalized_value);
+ printf(" Current Raw Value: ");
+ print_uint_var_dec(intel_smart_page.attributes[i].raw_value, 6);
+ printf("\n");
+ }
+ if (intel_smart_page.attributes[i].code == SPDK_NVME_INTEL_SMART_MEDIA_WEAR) {
+ printf("Timed Workload, Media Wear:\n");
+ printf(" Normalized Value : %d\n",
+ intel_smart_page.attributes[i].normalized_value);
+ printf(" Current Raw Value: ");
+ print_uint_var_dec(intel_smart_page.attributes[i].raw_value, 6);
+ printf("\n");
+ }
+ if (intel_smart_page.attributes[i].code == SPDK_NVME_INTEL_SMART_HOST_READ_PERCENTAGE) {
+ printf("Timed Workload, Host Read/Write Ratio:\n");
+ printf(" Normalized Value : %d\n",
+ intel_smart_page.attributes[i].normalized_value);
+ printf(" Current Raw Value: ");
+ print_uint_var_dec(intel_smart_page.attributes[i].raw_value, 6);
+ printf("%%");
+ printf("\n");
+ }
+ if (intel_smart_page.attributes[i].code == SPDK_NVME_INTEL_SMART_TIMER) {
+ printf("Timed Workload, Timer:\n");
+ printf(" Normalized Value : %d\n",
+ intel_smart_page.attributes[i].normalized_value);
+ printf(" Current Raw Value: ");
+ print_uint_var_dec(intel_smart_page.attributes[i].raw_value, 6);
+ printf("\n");
+ }
+ if (intel_smart_page.attributes[i].code == SPDK_NVME_INTEL_SMART_THERMAL_THROTTLE_STATUS) {
+ printf("Thermal Throttle Status:\n");
+ printf(" Normalized Value : %d\n",
+ intel_smart_page.attributes[i].normalized_value);
+ printf(" Current Raw Value:\n");
+ printf(" Percentage: %d%%\n", intel_smart_page.attributes[i].raw_value[0]);
+ printf(" Throttling Event Count: ");
+ print_uint_var_dec(&intel_smart_page.attributes[i].raw_value[1], 4);
+ printf("\n");
+ }
+ if (intel_smart_page.attributes[i].code == SPDK_NVME_INTEL_SMART_RETRY_BUFFER_OVERFLOW_COUNTER) {
+ printf("Retry Buffer Overflow Counter:\n");
+ printf(" Normalized Value : %d\n",
+ intel_smart_page.attributes[i].normalized_value);
+ printf(" Current Raw Value: ");
+ print_uint_var_dec(intel_smart_page.attributes[i].raw_value, 6);
+ printf("\n");
+ }
+ if (intel_smart_page.attributes[i].code == SPDK_NVME_INTEL_SMART_PLL_LOCK_LOSS_COUNT) {
+ printf("PLL Lock Loss Count:\n");
+ printf(" Normalized Value : %d\n",
+ intel_smart_page.attributes[i].normalized_value);
+ printf(" Current Raw Value: ");
+ print_uint_var_dec(intel_smart_page.attributes[i].raw_value, 6);
+ printf("\n");
+ }
+ if (intel_smart_page.attributes[i].code == SPDK_NVME_INTEL_SMART_NAND_BYTES_WRITTEN) {
+ printf("NAND Bytes Written:\n");
+ printf(" Normalized Value : %d\n",
+ intel_smart_page.attributes[i].normalized_value);
+ printf(" Current Raw Value: ");
+ print_uint_var_dec(intel_smart_page.attributes[i].raw_value, 6);
+ printf("\n");
+ }
+ if (intel_smart_page.attributes[i].code == SPDK_NVME_INTEL_SMART_HOST_BYTES_WRITTEN) {
+ printf("Host Bytes Written:\n");
+ printf(" Normalized Value : %d\n",
+ intel_smart_page.attributes[i].normalized_value);
+ printf(" Current Raw Value: ");
+ print_uint_var_dec(intel_smart_page.attributes[i].raw_value, 6);
+ printf("\n");
+ }
+ }
+ printf("\n");
+ }
+
+ if (spdk_nvme_ctrlr_is_log_page_supported(ctrlr, SPDK_NVME_INTEL_LOG_TEMPERATURE)) {
+ printf("Intel Temperature Information\n");
+ printf("==================\n");
+ printf("Current Temperature: %lu\n", intel_temperature_page.current_temperature);
+ printf("Overtemp shutdown Flag for last critical component temperature: %lu\n",
+ intel_temperature_page.shutdown_flag_last);
+ printf("Overtemp shutdown Flag for life critical component temperature: %lu\n",
+ intel_temperature_page.shutdown_flag_life);
+ printf("Highest temperature: %lu\n", intel_temperature_page.highest_temperature);
+ printf("Lowest temperature: %lu\n", intel_temperature_page.lowest_temperature);
+ printf("Specified Maximum Operating Temperature: %lu\n",
+ intel_temperature_page.specified_max_op_temperature);
+ printf("Specified Minimum Operating Temperature: %lu\n",
+ intel_temperature_page.specified_min_op_temperature);
+ printf("Estimated offset: %ld\n", intel_temperature_page.estimated_offset);
+ printf("\n");
+ printf("\n");
+
+ }
+
+ if (spdk_nvme_ctrlr_is_log_page_supported(ctrlr, SPDK_NVME_INTEL_MARKETING_DESCRIPTION)) {
+ printf("Intel Marketing Information\n");
+ printf("==================\n");
+ snprintf(str, sizeof(intel_md_page.marketing_product), "%s", intel_md_page.marketing_product);
+ printf("Marketing Product Information: %s\n", str);
+ printf("\n");
+ printf("\n");
+ }
+
+ printf("Active Namespaces\n");
+ printf("=================\n");
+ for (nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr);
+ nsid != 0; nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid)) {
+ print_namespace(spdk_nvme_ctrlr_get_ns(ctrlr, nsid));
+ }
+
+ if (g_discovery_page) {
+ printf("Discovery Log Page\n");
+ printf("==================\n");
+
+ if (g_hex_dump) {
+ hex_dump(g_discovery_page, g_discovery_page_size);
+ printf("\n");
+ }
+
+ printf("Generation Counter: %" PRIu64 "\n",
+ from_le64(&g_discovery_page->genctr));
+ printf("Number of Records: %" PRIu64 "\n",
+ from_le64(&g_discovery_page->numrec));
+ printf("Record Format: %" PRIu16 "\n",
+ from_le16(&g_discovery_page->recfmt));
+ printf("\n");
+
+ for (i = 0; i < g_discovery_page_numrec; i++) {
+ struct spdk_nvmf_discovery_log_page_entry *entry = &g_discovery_page->entries[i];
+
+ printf("Discovery Log Entry %u\n", i);
+ printf("----------------------\n");
+ printf("Transport Type: %u (%s)\n",
+ entry->trtype, spdk_nvme_transport_id_trtype_str(entry->trtype));
+ printf("Address Family: %u (%s)\n",
+ entry->adrfam, spdk_nvme_transport_id_adrfam_str(entry->adrfam));
+ printf("Subsystem Type: %u (%s)\n",
+ entry->subtype,
+ entry->subtype == SPDK_NVMF_SUBTYPE_DISCOVERY ? "Discovery Service" :
+ entry->subtype == SPDK_NVMF_SUBTYPE_NVME ? "NVM Subsystem" :
+ "Unknown");
+ printf("Transport Requirements:\n");
+ printf(" Secure Channel: %s\n",
+ entry->treq.secure_channel == SPDK_NVMF_TREQ_SECURE_CHANNEL_NOT_SPECIFIED ? "Not Specified" :
+ entry->treq.secure_channel == SPDK_NVMF_TREQ_SECURE_CHANNEL_REQUIRED ? "Required" :
+ entry->treq.secure_channel == SPDK_NVMF_TREQ_SECURE_CHANNEL_NOT_REQUIRED ? "Not Required" :
+ "Reserved");
+ printf("Port ID: %" PRIu16 " (0x%04" PRIx16 ")\n",
+ from_le16(&entry->portid), from_le16(&entry->portid));
+ printf("Controller ID: %" PRIu16 " (0x%04" PRIx16 ")\n",
+ from_le16(&entry->cntlid), from_le16(&entry->cntlid));
+ printf("Admin Max SQ Size: %" PRIu16 "\n",
+ from_le16(&entry->asqsz));
+ snprintf(str, sizeof(entry->trsvcid) + 1, "%s", entry->trsvcid);
+ printf("Transport Service Identifier: %s\n", str);
+ snprintf(str, sizeof(entry->subnqn) + 1, "%s", entry->subnqn);
+ printf("NVM Subsystem Qualified Name: %s\n", str);
+ snprintf(str, sizeof(entry->traddr) + 1, "%s", entry->traddr);
+ printf("Transport Address: %s\n", str);
+
+ if (entry->trtype == SPDK_NVMF_TRTYPE_RDMA) {
+ printf("Transport Specific Address Subtype - RDMA\n");
+ printf(" RDMA QP Service Type: %u (%s)\n",
+ entry->tsas.rdma.rdma_qptype,
+ entry->tsas.rdma.rdma_qptype == SPDK_NVMF_RDMA_QPTYPE_RELIABLE_CONNECTED ? "Reliable Connected" :
+ entry->tsas.rdma.rdma_qptype == SPDK_NVMF_RDMA_QPTYPE_RELIABLE_DATAGRAM ? "Reliable Datagram" :
+ "Unknown");
+ printf(" RDMA Provider Type: %u (%s)\n",
+ entry->tsas.rdma.rdma_prtype,
+ entry->tsas.rdma.rdma_prtype == SPDK_NVMF_RDMA_PRTYPE_NONE ? "No provider specified" :
+ entry->tsas.rdma.rdma_prtype == SPDK_NVMF_RDMA_PRTYPE_IB ? "InfiniBand" :
+ entry->tsas.rdma.rdma_prtype == SPDK_NVMF_RDMA_PRTYPE_ROCE ? "InfiniBand RoCE" :
+ entry->tsas.rdma.rdma_prtype == SPDK_NVMF_RDMA_PRTYPE_ROCE2 ? "InfiniBand RoCE v2" :
+ entry->tsas.rdma.rdma_prtype == SPDK_NVMF_RDMA_PRTYPE_IWARP ? "iWARP" :
+ "Unknown");
+ printf(" RDMA CM Service: %u (%s)\n",
+ entry->tsas.rdma.rdma_cms,
+ entry->tsas.rdma.rdma_cms == SPDK_NVMF_RDMA_CMS_RDMA_CM ? "RDMA_CM" :
+ "Unknown");
+ if (entry->adrfam == SPDK_NVMF_ADRFAM_IB) {
+ printf(" RDMA Partition Key: %" PRIu32 "\n",
+ from_le32(&entry->tsas.rdma.rdma_pkey));
+ }
+ }
+ }
+ free(g_discovery_page);
+ g_discovery_page = NULL;
+ }
+}
+
+static void
+usage(const char *program_name)
+{
+ printf("%s [options]", program_name);
+ printf("\n");
+ printf("options:\n");
+ printf(" -r trid remote NVMe over Fabrics target address\n");
+ printf(" Format: 'key:value [key:value] ...'\n");
+ printf(" Keys:\n");
+ printf(" trtype Transport type (e.g. RDMA)\n");
+ printf(" adrfam Address family (e.g. IPv4, IPv6)\n");
+ printf(" traddr Transport address (e.g. 192.168.100.8)\n");
+ printf(" trsvcid Transport service identifier (e.g. 4420)\n");
+ printf(" subnqn Subsystem NQN (default: %s)\n", SPDK_NVMF_DISCOVERY_NQN);
+ printf(" Example: -r 'trtype:RDMA adrfam:IPv4 traddr:192.168.100.8 trsvcid:4420'\n");
+
+ spdk_log_usage(stdout, "-L");
+
+ printf(" -i shared memory group ID\n");
+ printf(" -p core number in decimal to run this application which started from 0\n");
+ printf(" -d DPDK huge memory size in MB\n");
+ printf(" -x print hex dump of raw data\n");
+ printf(" -v verbose (enable warnings)\n");
+ printf(" -V enumerate VMD\n");
+ printf(" -H show this usage\n");
+}
+
+static int
+parse_args(int argc, char **argv)
+{
+ int op, rc;
+
+ spdk_nvme_trid_populate_transport(&g_trid, SPDK_NVME_TRANSPORT_PCIE);
+ snprintf(g_trid.subnqn, sizeof(g_trid.subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN);
+
+ while ((op = getopt(argc, argv, "d:i:p:r:xHL:V")) != -1) {
+ switch (op) {
+ case 'd':
+ g_dpdk_mem = spdk_strtol(optarg, 10);
+ if (g_dpdk_mem < 0) {
+ fprintf(stderr, "Invalid DPDK memory size\n");
+ return g_dpdk_mem;
+ }
+ break;
+ case 'i':
+ g_shm_id = spdk_strtol(optarg, 10);
+ if (g_shm_id < 0) {
+ fprintf(stderr, "Invalid shared memory ID\n");
+ return g_shm_id;
+ }
+ break;
+ case 'p':
+ g_master_core = spdk_strtol(optarg, 10);
+ if (g_master_core < 0) {
+ fprintf(stderr, "Invalid core number\n");
+ return g_master_core;
+ }
+ snprintf(g_core_mask, sizeof(g_core_mask), "0x%llx", 1ULL << g_master_core);
+ break;
+ case 'r':
+ if (spdk_nvme_transport_id_parse(&g_trid, optarg) != 0) {
+ fprintf(stderr, "Error parsing transport address\n");
+ return 1;
+ }
+ break;
+ case 'x':
+ g_hex_dump = true;
+ break;
+ case 'L':
+ rc = spdk_log_set_flag(optarg);
+ if (rc < 0) {
+ fprintf(stderr, "unknown flag\n");
+ usage(argv[0]);
+ exit(EXIT_FAILURE);
+ }
+ spdk_log_set_print_level(SPDK_LOG_DEBUG);
+#ifndef DEBUG
+ fprintf(stderr, "%s must be rebuilt with CONFIG_DEBUG=y for -L flag.\n",
+ argv[0]);
+ usage(argv[0]);
+ return 0;
+#endif
+ break;
+ case 'H':
+ usage(argv[0]);
+ break;
+ case 'V':
+ g_vmd = true;
+ break;
+ default:
+ usage(argv[0]);
+ return 1;
+ }
+ }
+
+ return 0;
+}
+
+static bool
+probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
+ struct spdk_nvme_ctrlr_opts *opts)
+{
+ return true;
+}
+
+static void
+attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
+ struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts)
+{
+ g_controllers_found++;
+ print_controller(ctrlr, trid);
+ spdk_nvme_detach(ctrlr);
+}
+
+int main(int argc, char **argv)
+{
+ int rc;
+ struct spdk_env_opts opts;
+ struct spdk_nvme_ctrlr *ctrlr;
+
+ rc = parse_args(argc, argv);
+ if (rc != 0) {
+ return rc;
+ }
+
+ spdk_env_opts_init(&opts);
+ opts.name = "identify";
+ opts.shm_id = g_shm_id;
+ opts.mem_size = g_dpdk_mem;
+ opts.mem_channel = 1;
+ opts.master_core = g_master_core;
+ opts.core_mask = g_core_mask;
+ if (g_trid.trtype != SPDK_NVME_TRANSPORT_PCIE) {
+ opts.no_pci = true;
+ }
+ if (spdk_env_init(&opts) < 0) {
+ fprintf(stderr, "Unable to initialize SPDK env\n");
+ return 1;
+ }
+
+ if (g_vmd && spdk_vmd_init()) {
+ fprintf(stderr, "Failed to initialize VMD."
+ " Some NVMe devices can be unavailable.\n");
+ }
+
+ /* A specific trid is required. */
+ if (strlen(g_trid.traddr) != 0) {
+ ctrlr = spdk_nvme_connect(&g_trid, NULL, 0);
+ if (!ctrlr) {
+ fprintf(stderr, "spdk_nvme_connect() failed\n");
+ return 1;
+ }
+
+ g_controllers_found++;
+ print_controller(ctrlr, &g_trid);
+ spdk_nvme_detach(ctrlr);
+ } else if (spdk_nvme_probe(&g_trid, NULL, probe_cb, attach_cb, NULL) != 0) {
+ fprintf(stderr, "spdk_nvme_probe() failed\n");
+ return 1;
+ }
+
+ if (g_controllers_found == 0) {
+ fprintf(stderr, "No NVMe controllers found.\n");
+ }
+
+ if (g_vmd) {
+ spdk_vmd_fini();
+ }
+
+ return 0;
+}
diff --git a/src/spdk/examples/nvme/nvme_manage/.gitignore b/src/spdk/examples/nvme/nvme_manage/.gitignore
new file mode 100644
index 000000000..cdc78a1a1
--- /dev/null
+++ b/src/spdk/examples/nvme/nvme_manage/.gitignore
@@ -0,0 +1 @@
+nvme_manage
diff --git a/src/spdk/examples/nvme/nvme_manage/Makefile b/src/spdk/examples/nvme/nvme_manage/Makefile
new file mode 100644
index 000000000..ed467b884
--- /dev/null
+++ b/src/spdk/examples/nvme/nvme_manage/Makefile
@@ -0,0 +1,38 @@
+#
+# BSD LICENSE
+#
+# Copyright (c) Intel Corporation.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Intel Corporation nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
+
+APP = nvme_manage
+
+include $(SPDK_ROOT_DIR)/mk/nvme.libtest.mk
diff --git a/src/spdk/examples/nvme/nvme_manage/nvme_manage.c b/src/spdk/examples/nvme/nvme_manage/nvme_manage.c
new file mode 100644
index 000000000..c202dab42
--- /dev/null
+++ b/src/spdk/examples/nvme/nvme_manage/nvme_manage.c
@@ -0,0 +1,1703 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+
+#include "spdk/nvme.h"
+#include "spdk/env.h"
+#include "spdk/string.h"
+#include "spdk/util.h"
+#include "spdk/opal.h"
+
+#define MAX_DEVS 64
+
+struct dev {
+ struct spdk_pci_addr pci_addr;
+ struct spdk_nvme_ctrlr *ctrlr;
+ const struct spdk_nvme_ctrlr_data *cdata;
+ struct spdk_nvme_ns_data *common_ns_data;
+ int outstanding_admin_cmds;
+ struct spdk_opal_dev *opal_dev;
+};
+
+static struct dev devs[MAX_DEVS];
+static int num_devs = 0;
+static int g_shm_id = -1;
+
+#define foreach_dev(iter) \
+ for (iter = devs; iter - devs < num_devs; iter++)
+
+enum controller_display_model {
+ CONTROLLER_DISPLAY_ALL = 0x0,
+ CONTROLLER_DISPLAY_SIMPLISTIC = 0x1,
+};
+
+static int
+cmp_devs(const void *ap, const void *bp)
+{
+ const struct dev *a = ap, *b = bp;
+
+ return spdk_pci_addr_compare(&a->pci_addr, &b->pci_addr);
+}
+
+static bool
+probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
+ struct spdk_nvme_ctrlr_opts *opts)
+{
+ return true;
+}
+
+static void
+identify_common_ns_cb(void *cb_arg, const struct spdk_nvme_cpl *cpl)
+{
+ struct dev *dev = cb_arg;
+
+ if (cpl->status.sc != SPDK_NVME_SC_SUCCESS) {
+ /* Identify Namespace for NSID = FFFFFFFFh is optional, so failure is not fatal. */
+ spdk_dma_free(dev->common_ns_data);
+ dev->common_ns_data = NULL;
+ }
+
+ dev->outstanding_admin_cmds--;
+}
+
+static void
+attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
+ struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts)
+{
+ struct dev *dev;
+ struct spdk_nvme_cmd cmd;
+
+ /* add to dev list */
+ dev = &devs[num_devs++];
+ spdk_pci_addr_parse(&dev->pci_addr, trid->traddr);
+ dev->ctrlr = ctrlr;
+
+ /* Retrieve controller data */
+ dev->cdata = spdk_nvme_ctrlr_get_data(dev->ctrlr);
+
+ dev->common_ns_data = spdk_dma_zmalloc(sizeof(struct spdk_nvme_ns_data), 4096, NULL);
+ if (dev->common_ns_data == NULL) {
+ fprintf(stderr, "common_ns_data allocation failure\n");
+ return;
+ }
+
+ /* Identify Namespace with NSID set to FFFFFFFFh to get common namespace capabilities. */
+ memset(&cmd, 0, sizeof(cmd));
+ cmd.opc = SPDK_NVME_OPC_IDENTIFY;
+ cmd.cdw10_bits.identify.cns = 0; /* CNS = 0 (Identify Namespace) */
+ cmd.nsid = SPDK_NVME_GLOBAL_NS_TAG;
+
+ dev->outstanding_admin_cmds++;
+ if (spdk_nvme_ctrlr_cmd_admin_raw(ctrlr, &cmd, dev->common_ns_data,
+ sizeof(struct spdk_nvme_ns_data), identify_common_ns_cb, dev) != 0) {
+ dev->outstanding_admin_cmds--;
+ spdk_dma_free(dev->common_ns_data);
+ dev->common_ns_data = NULL;
+ }
+
+ while (dev->outstanding_admin_cmds) {
+ spdk_nvme_ctrlr_process_admin_completions(ctrlr);
+ }
+}
+
+static void usage(void)
+{
+ printf("NVMe Management Options");
+ printf("\n");
+ printf("\t[1: list controllers]\n");
+ printf("\t[2: create namespace]\n");
+ printf("\t[3: delete namespace]\n");
+ printf("\t[4: attach namespace to controller]\n");
+ printf("\t[5: detach namespace from controller]\n");
+ printf("\t[6: format namespace or controller]\n");
+ printf("\t[7: firmware update]\n");
+ printf("\t[8: opal]\n");
+ printf("\t[9: quit]\n");
+}
+
+static void
+display_namespace_dpc(const struct spdk_nvme_ns_data *nsdata)
+{
+ if (nsdata->dpc.pit1 || nsdata->dpc.pit2 || nsdata->dpc.pit3) {
+ if (nsdata->dpc.pit1) {
+ printf("PIT1 ");
+ }
+
+ if (nsdata->dpc.pit2) {
+ printf("PIT2 ");
+ }
+
+ if (nsdata->dpc.pit3) {
+ printf("PIT3 ");
+ }
+ } else {
+ printf("Not Supported\n");
+ return;
+ }
+
+ if (nsdata->dpc.md_start && nsdata->dpc.md_end) {
+ printf("Location: Head or Tail\n");
+ } else if (nsdata->dpc.md_start) {
+ printf("Location: Head\n");
+ } else if (nsdata->dpc.md_end) {
+ printf("Location: Tail\n");
+ } else {
+ printf("Not Supported\n");
+ }
+}
+
+static void
+display_namespace(struct spdk_nvme_ns *ns)
+{
+ const struct spdk_nvme_ns_data *nsdata;
+ uint32_t i;
+
+ nsdata = spdk_nvme_ns_get_data(ns);
+
+ printf("Namespace ID:%d\n", spdk_nvme_ns_get_id(ns));
+
+ printf("Size (in LBAs): %lld (%lldM)\n",
+ (long long)nsdata->nsze,
+ (long long)nsdata->nsze / 1024 / 1024);
+ printf("Capacity (in LBAs): %lld (%lldM)\n",
+ (long long)nsdata->ncap,
+ (long long)nsdata->ncap / 1024 / 1024);
+ printf("Utilization (in LBAs): %lld (%lldM)\n",
+ (long long)nsdata->nuse,
+ (long long)nsdata->nuse / 1024 / 1024);
+ printf("Format Progress Indicator: %s\n",
+ nsdata->fpi.fpi_supported ? "Supported" : "Not Supported");
+ if (nsdata->fpi.fpi_supported && nsdata->fpi.percentage_remaining) {
+ printf("Formatted Percentage: %d%%\n", 100 - nsdata->fpi.percentage_remaining);
+ }
+ printf("Number of LBA Formats: %d\n", nsdata->nlbaf + 1);
+ printf("Current LBA Format: LBA Format #%02d\n",
+ nsdata->flbas.format);
+ for (i = 0; i <= nsdata->nlbaf; i++)
+ printf("LBA Format #%02d: Data Size: %5d Metadata Size: %5d\n",
+ i, 1 << nsdata->lbaf[i].lbads, nsdata->lbaf[i].ms);
+ printf("Data Protection Capabilities:");
+ display_namespace_dpc(nsdata);
+ if (SPDK_NVME_FMT_NVM_PROTECTION_DISABLE == nsdata->dps.pit) {
+ printf("Data Protection Setting: N/A\n");
+ } else {
+ printf("Data Protection Setting: PIT%d Location: %s\n",
+ nsdata->dps.pit, nsdata->dps.md_start ? "Head" : "Tail");
+ }
+ printf("Multipath IO and Sharing: %s\n",
+ nsdata->nmic.can_share ? "Supported" : "Not Supported");
+ printf("\n");
+}
+
+static void
+display_controller(struct dev *dev, int model)
+{
+ struct spdk_nvme_ns *ns;
+ const struct spdk_nvme_ctrlr_data *cdata;
+ uint8_t str[128];
+ uint32_t nsid;
+
+ cdata = spdk_nvme_ctrlr_get_data(dev->ctrlr);
+
+ if (model == CONTROLLER_DISPLAY_SIMPLISTIC) {
+ printf("%04x:%02x:%02x.%02x ",
+ dev->pci_addr.domain, dev->pci_addr.bus, dev->pci_addr.dev, dev->pci_addr.func);
+ printf("%-40.40s %-20.20s ",
+ cdata->mn, cdata->sn);
+ printf("%5d ", cdata->cntlid);
+ printf("\n");
+ return;
+ }
+
+ printf("=====================================================\n");
+ printf("NVMe Controller: %04x:%02x:%02x.%02x\n",
+ dev->pci_addr.domain, dev->pci_addr.bus, dev->pci_addr.dev, dev->pci_addr.func);
+ printf("============================\n");
+ printf("Controller Capabilities/Features\n");
+ printf("Controller ID: %d\n", cdata->cntlid);
+ snprintf(str, sizeof(cdata->sn) + 1, "%s", cdata->sn);
+ printf("Serial Number: %s\n", str);
+ printf("\n");
+
+ printf("Admin Command Set Attributes\n");
+ printf("============================\n");
+ printf("Namespace Manage And Attach: %s\n",
+ cdata->oacs.ns_manage ? "Supported" : "Not Supported");
+ printf("Namespace Format: %s\n",
+ cdata->oacs.format ? "Supported" : "Not Supported");
+ printf("\n");
+ printf("NVM Command Set Attributes\n");
+ printf("============================\n");
+ if (cdata->fna.format_all_ns) {
+ printf("Namespace format operation applies to all namespaces\n");
+ } else {
+ printf("Namespace format operation applies to per namespace\n");
+ }
+ printf("\n");
+ printf("Namespace Attributes\n");
+ printf("============================\n");
+ for (nsid = spdk_nvme_ctrlr_get_first_active_ns(dev->ctrlr);
+ nsid != 0; nsid = spdk_nvme_ctrlr_get_next_active_ns(dev->ctrlr, nsid)) {
+ ns = spdk_nvme_ctrlr_get_ns(dev->ctrlr, nsid);
+ assert(ns != NULL);
+ display_namespace(ns);
+ }
+}
+
+static void
+display_controller_list(void)
+{
+ struct dev *iter;
+
+ foreach_dev(iter) {
+ display_controller(iter, CONTROLLER_DISPLAY_ALL);
+ }
+}
+
+static char *
+get_line(char *buf, int buf_size, FILE *f, bool secret)
+{
+ char *ch;
+ size_t len;
+ struct termios default_attr = {}, new_attr = {};
+ int ret;
+
+ if (secret) {
+ ret = tcgetattr(STDIN_FILENO, &default_attr);
+ if (ret) {
+ return NULL;
+ }
+
+ new_attr = default_attr;
+ new_attr.c_lflag &= ~ECHO; /* disable echo */
+ ret = tcsetattr(STDIN_FILENO, TCSAFLUSH, &new_attr);
+ if (ret) {
+ return NULL;
+ }
+ }
+
+ ch = fgets(buf, buf_size, f);
+ if (ch == NULL) {
+ return NULL;
+ }
+
+ if (secret) {
+ ret = tcsetattr(STDIN_FILENO, TCSAFLUSH, &default_attr); /* restore default confing */
+ if (ret) {
+ return NULL;
+ }
+ }
+
+ len = strlen(buf);
+ if (len > 0 && buf[len - 1] == '\n') {
+ buf[len - 1] = '\0';
+ }
+ return buf;
+}
+
+static struct dev *
+get_controller(void)
+{
+ struct spdk_pci_addr pci_addr;
+ char address[64];
+ char *p;
+ int ch;
+ struct dev *iter;
+
+ memset(address, 0, sizeof(address));
+
+ foreach_dev(iter) {
+ display_controller(iter, CONTROLLER_DISPLAY_SIMPLISTIC);
+ }
+
+ printf("Please Input PCI Address(domain:bus:dev.func):\n");
+
+ while ((ch = getchar()) != '\n' && ch != EOF);
+ p = get_line(address, 64, stdin, false);
+ if (p == NULL) {
+ return NULL;
+ }
+
+ while (isspace(*p)) {
+ p++;
+ }
+
+ if (spdk_pci_addr_parse(&pci_addr, p) < 0) {
+ return NULL;
+ }
+
+ foreach_dev(iter) {
+ if (spdk_pci_addr_compare(&pci_addr, &iter->pci_addr) == 0) {
+ return iter;
+ }
+ }
+ return NULL;
+}
+
+static int
+get_lba_format(const struct spdk_nvme_ns_data *ns_data)
+{
+ int lbaf, i;
+
+ printf("\nSupported LBA formats:\n");
+ for (i = 0; i <= ns_data->nlbaf; i++) {
+ printf("%2d: %d data bytes", i, 1 << ns_data->lbaf[i].lbads);
+ if (ns_data->lbaf[i].ms) {
+ printf(" + %d metadata bytes", ns_data->lbaf[i].ms);
+ }
+ printf("\n");
+ }
+
+ printf("Please input LBA format index (0 - %d):\n", ns_data->nlbaf);
+ if (scanf("%d", &lbaf) != 1 || lbaf > ns_data->nlbaf) {
+ return -1;
+ }
+
+ return lbaf;
+}
+
+static void
+identify_allocated_ns_cb(void *cb_arg, const struct spdk_nvme_cpl *cpl)
+{
+ struct dev *dev = cb_arg;
+
+ dev->outstanding_admin_cmds--;
+}
+
+static uint32_t
+get_allocated_nsid(struct dev *dev)
+{
+ uint32_t nsid;
+ size_t i;
+ struct spdk_nvme_ns_list *ns_list;
+ struct spdk_nvme_cmd cmd = {0};
+
+ ns_list = spdk_dma_zmalloc(sizeof(*ns_list), 4096, NULL);
+ if (ns_list == NULL) {
+ printf("Allocation error\n");
+ return 0;
+ }
+
+ cmd.opc = SPDK_NVME_OPC_IDENTIFY;
+ cmd.cdw10_bits.identify.cns = SPDK_NVME_IDENTIFY_ALLOCATED_NS_LIST;
+ cmd.nsid = 0;
+
+ dev->outstanding_admin_cmds++;
+ if (spdk_nvme_ctrlr_cmd_admin_raw(dev->ctrlr, &cmd, ns_list, sizeof(*ns_list),
+ identify_allocated_ns_cb, dev)) {
+ printf("Identify command failed\n");
+ spdk_dma_free(ns_list);
+ return 0;
+ }
+
+ while (dev->outstanding_admin_cmds) {
+ spdk_nvme_ctrlr_process_admin_completions(dev->ctrlr);
+ }
+
+ printf("Allocated Namespace IDs:\n");
+ for (i = 0; i < SPDK_COUNTOF(ns_list->ns_list); i++) {
+ if (ns_list->ns_list[i] == 0) {
+ break;
+ }
+ printf("%u\n", ns_list->ns_list[i]);
+ }
+
+ spdk_dma_free(ns_list);
+
+ printf("Please Input Namespace ID:\n");
+ if (!scanf("%u", &nsid)) {
+ printf("Invalid Namespace ID\n");
+ nsid = 0;
+ }
+
+ return nsid;
+}
+
+static void
+ns_attach(struct dev *device, int attachment_op, int ctrlr_id, int ns_id)
+{
+ int ret = 0;
+ struct spdk_nvme_ctrlr_list *ctrlr_list;
+
+ ctrlr_list = spdk_dma_zmalloc(sizeof(struct spdk_nvme_ctrlr_list),
+ 4096, NULL);
+ if (ctrlr_list == NULL) {
+ printf("Allocation error (controller list)\n");
+ exit(1);
+ }
+
+ ctrlr_list->ctrlr_count = 1;
+ ctrlr_list->ctrlr_list[0] = ctrlr_id;
+
+ if (attachment_op == SPDK_NVME_NS_CTRLR_ATTACH) {
+ ret = spdk_nvme_ctrlr_attach_ns(device->ctrlr, ns_id, ctrlr_list);
+ } else if (attachment_op == SPDK_NVME_NS_CTRLR_DETACH) {
+ ret = spdk_nvme_ctrlr_detach_ns(device->ctrlr, ns_id, ctrlr_list);
+ }
+
+ if (ret) {
+ fprintf(stdout, "ns attach: Failed\n");
+ }
+
+ spdk_dma_free(ctrlr_list);
+}
+
+static void
+ns_manage_add(struct dev *device, uint64_t ns_size, uint64_t ns_capacity, int ns_lbasize,
+ uint8_t ns_dps_type, uint8_t ns_dps_location, uint8_t ns_nmic)
+{
+ uint32_t nsid;
+ struct spdk_nvme_ns_data *ndata;
+
+ ndata = spdk_dma_zmalloc(sizeof(struct spdk_nvme_ns_data), 4096, NULL);
+ if (ndata == NULL) {
+ printf("Allocation error (namespace data)\n");
+ exit(1);
+ }
+
+ ndata->nsze = ns_size;
+ ndata->ncap = ns_capacity;
+ ndata->flbas.format = ns_lbasize;
+ if (SPDK_NVME_FMT_NVM_PROTECTION_DISABLE != ns_dps_type) {
+ ndata->dps.pit = ns_dps_type;
+ ndata->dps.md_start = ns_dps_location;
+ }
+ ndata->nmic.can_share = ns_nmic;
+ nsid = spdk_nvme_ctrlr_create_ns(device->ctrlr, ndata);
+ if (nsid == 0) {
+ fprintf(stdout, "ns manage: Failed\n");
+ } else {
+ printf("Created namespace ID %u\n", nsid);
+ }
+
+ spdk_dma_free(ndata);
+}
+
+static void
+ns_manage_delete(struct dev *device, int ns_id)
+{
+ int ret = 0;
+
+ ret = spdk_nvme_ctrlr_delete_ns(device->ctrlr, ns_id);
+ if (ret) {
+ fprintf(stdout, "ns manage: Failed\n");
+ return;
+ }
+}
+
+static void
+nvme_manage_format(struct dev *device, int ns_id, int ses, int pi, int pil, int ms, int lbaf)
+{
+ int ret = 0;
+ struct spdk_nvme_format format = {};
+
+ format.lbaf = lbaf;
+ format.ms = ms;
+ format.pi = pi;
+ format.pil = pil;
+ format.ses = ses;
+ ret = spdk_nvme_ctrlr_format(device->ctrlr, ns_id, &format);
+ if (ret) {
+ fprintf(stdout, "nvme format: Failed\n");
+ return;
+ }
+}
+
+static void
+attach_and_detach_ns(int attachment_op)
+{
+ uint32_t nsid;
+ struct dev *ctrlr;
+
+ ctrlr = get_controller();
+ if (ctrlr == NULL) {
+ printf("Invalid controller PCI Address.\n");
+ return;
+ }
+
+ if (!ctrlr->cdata->oacs.ns_manage) {
+ printf("Controller does not support ns management\n");
+ return;
+ }
+
+ nsid = get_allocated_nsid(ctrlr);
+ if (nsid == 0) {
+ printf("Invalid Namespace ID\n");
+ return;
+ }
+
+ ns_attach(ctrlr, attachment_op, ctrlr->cdata->cntlid, nsid);
+}
+
+static void
+add_ns(void)
+{
+ uint64_t ns_size = 0;
+ uint64_t ns_capacity = 0;
+ int ns_lbasize;
+ int ns_dps_type = 0;
+ int ns_dps_location = 0;
+ int ns_nmic = 0;
+ struct dev *ctrlr = NULL;
+
+ ctrlr = get_controller();
+ if (ctrlr == NULL) {
+ printf("Invalid controller PCI Address.\n");
+ return;
+ }
+
+ if (!ctrlr->cdata->oacs.ns_manage) {
+ printf("Controller does not support ns management\n");
+ return;
+ }
+
+ if (!ctrlr->common_ns_data) {
+ printf("Controller did not return common namespace capabilities\n");
+ return;
+ }
+
+ ns_lbasize = get_lba_format(ctrlr->common_ns_data);
+ if (ns_lbasize < 0) {
+ printf("Invalid LBA format number\n");
+ return;
+ }
+
+ printf("Please Input Namespace Size (in LBAs):\n");
+ if (!scanf("%" SCNu64, &ns_size)) {
+ printf("Invalid Namespace Size\n");
+ while (getchar() != '\n');
+ return;
+ }
+
+ printf("Please Input Namespace Capacity (in LBAs):\n");
+ if (!scanf("%" SCNu64, &ns_capacity)) {
+ printf("Invalid Namespace Capacity\n");
+ while (getchar() != '\n');
+ return;
+ }
+
+ printf("Please Input Data Protection Type (0 - 3):\n");
+ if (!scanf("%d", &ns_dps_type)) {
+ printf("Invalid Data Protection Type\n");
+ while (getchar() != '\n');
+ return;
+ }
+
+ if (SPDK_NVME_FMT_NVM_PROTECTION_DISABLE != ns_dps_type) {
+ printf("Please Input Data Protection Location (1: Head; 0: Tail):\n");
+ if (!scanf("%d", &ns_dps_location)) {
+ printf("Invalid Data Protection Location\n");
+ while (getchar() != '\n');
+ return;
+ }
+ }
+
+ printf("Please Input Multi-path IO and Sharing Capabilities (1: Share; 0: Private):\n");
+ if (!scanf("%d", &ns_nmic)) {
+ printf("Invalid Multi-path IO and Sharing Capabilities\n");
+ while (getchar() != '\n');
+ return;
+ }
+
+ ns_manage_add(ctrlr, ns_size, ns_capacity, ns_lbasize,
+ ns_dps_type, ns_dps_location, ns_nmic);
+}
+
+static void
+delete_ns(void)
+{
+ int ns_id;
+ struct dev *ctrlr;
+
+ ctrlr = get_controller();
+ if (ctrlr == NULL) {
+ printf("Invalid controller PCI Address.\n");
+ return;
+ }
+
+ if (!ctrlr->cdata->oacs.ns_manage) {
+ printf("Controller does not support ns management\n");
+ return;
+ }
+
+ printf("Please Input Namespace ID:\n");
+ if (!scanf("%d", &ns_id)) {
+ printf("Invalid Namespace ID\n");
+ while (getchar() != '\n');
+ return;
+ }
+
+ ns_manage_delete(ctrlr, ns_id);
+}
+
+static void
+format_nvm(void)
+{
+ int ns_id;
+ int ses;
+ int pil;
+ int pi;
+ int ms;
+ int lbaf;
+ char option;
+ struct dev *ctrlr;
+ const struct spdk_nvme_ctrlr_data *cdata;
+ struct spdk_nvme_ns *ns;
+ const struct spdk_nvme_ns_data *nsdata;
+
+ ctrlr = get_controller();
+ if (ctrlr == NULL) {
+ printf("Invalid controller PCI BDF.\n");
+ return;
+ }
+
+ cdata = ctrlr->cdata;
+
+ if (!cdata->oacs.format) {
+ printf("Controller does not support Format NVM command\n");
+ return;
+ }
+
+ if (cdata->fna.format_all_ns) {
+ ns_id = SPDK_NVME_GLOBAL_NS_TAG;
+ ns = spdk_nvme_ctrlr_get_ns(ctrlr->ctrlr, 1);
+ } else {
+ printf("Please Input Namespace ID (1 - %d):\n", cdata->nn);
+ if (!scanf("%d", &ns_id)) {
+ printf("Invalid Namespace ID\n");
+ while (getchar() != '\n');
+ return;
+ }
+ ns = spdk_nvme_ctrlr_get_ns(ctrlr->ctrlr, ns_id);
+ }
+
+ if (ns == NULL) {
+ printf("Namespace ID %d not found\n", ns_id);
+ while (getchar() != '\n');
+ return;
+ }
+
+ nsdata = spdk_nvme_ns_get_data(ns);
+
+ printf("Please Input Secure Erase Setting:\n");
+ printf(" 0: No secure erase operation requested\n");
+ printf(" 1: User data erase\n");
+ if (cdata->fna.crypto_erase_supported) {
+ printf(" 2: Cryptographic erase\n");
+ }
+ if (!scanf("%d", &ses)) {
+ printf("Invalid Secure Erase Setting\n");
+ while (getchar() != '\n');
+ return;
+ }
+
+ lbaf = get_lba_format(nsdata);
+ if (lbaf < 0) {
+ printf("Invalid LBA format number\n");
+ return;
+ }
+
+ if (nsdata->lbaf[lbaf].ms) {
+ printf("Please Input Protection Information:\n");
+ printf(" 0: Protection information is not enabled\n");
+ printf(" 1: Protection information is enabled, Type 1\n");
+ printf(" 2: Protection information is enabled, Type 2\n");
+ printf(" 3: Protection information is enabled, Type 3\n");
+ if (!scanf("%d", &pi)) {
+ printf("Invalid protection information\n");
+ while (getchar() != '\n');
+ return;
+ }
+
+ if (pi) {
+ printf("Please Input Protection Information Location:\n");
+ printf(" 0: Protection information transferred as the last eight bytes of metadata\n");
+ printf(" 1: Protection information transferred as the first eight bytes of metadata\n");
+ if (!scanf("%d", &pil)) {
+ printf("Invalid protection information location\n");
+ while (getchar() != '\n');
+ return;
+ }
+ } else {
+ pil = 0;
+ }
+
+ printf("Please Input Metadata Setting:\n");
+ printf(" 0: Metadata is transferred as part of a separate buffer\n");
+ printf(" 1: Metadata is transferred as part of an extended data LBA\n");
+ if (!scanf("%d", &ms)) {
+ printf("Invalid metadata setting\n");
+ while (getchar() != '\n');
+ return;
+ }
+ } else {
+ ms = 0;
+ pi = 0;
+ pil = 0;
+ }
+
+ printf("Warning: use this utility at your own risk.\n"
+ "This command will format your namespace and all data will be lost.\n"
+ "This command may take several minutes to complete,\n"
+ "so do not interrupt the utility until it completes.\n"
+ "Press 'Y' to continue with the format operation.\n");
+
+ while (getchar() != '\n');
+ if (!scanf("%c", &option)) {
+ printf("Invalid option\n");
+ while (getchar() != '\n');
+ return;
+ }
+
+ if (option == 'y' || option == 'Y') {
+ nvme_manage_format(ctrlr, ns_id, ses, pi, pil, ms, lbaf);
+ } else {
+ printf("NVMe format abort\n");
+ }
+}
+
+static void
+update_firmware_image(void)
+{
+ int rc;
+ int fd = -1;
+ int slot;
+ unsigned int size;
+ struct stat fw_stat;
+ char path[256];
+ void *fw_image;
+ struct dev *ctrlr;
+ const struct spdk_nvme_ctrlr_data *cdata;
+ enum spdk_nvme_fw_commit_action commit_action;
+ struct spdk_nvme_status status;
+
+ ctrlr = get_controller();
+ if (ctrlr == NULL) {
+ printf("Invalid controller PCI BDF.\n");
+ return;
+ }
+
+ cdata = ctrlr->cdata;
+
+ if (!cdata->oacs.firmware) {
+ printf("Controller does not support firmware download and commit command\n");
+ return;
+ }
+
+ printf("Please Input The Path Of Firmware Image\n");
+
+ if (get_line(path, sizeof(path), stdin, false) == NULL) {
+ printf("Invalid path setting\n");
+ while (getchar() != '\n');
+ return;
+ }
+
+ fd = open(path, O_RDONLY);
+ if (fd < 0) {
+ perror("Open file failed");
+ return;
+ }
+ rc = fstat(fd, &fw_stat);
+ if (rc < 0) {
+ printf("Fstat failed\n");
+ close(fd);
+ return;
+ }
+
+ if (fw_stat.st_size % 4) {
+ printf("Firmware image size is not multiple of 4\n");
+ close(fd);
+ return;
+ }
+
+ size = fw_stat.st_size;
+
+ fw_image = spdk_dma_zmalloc(size, 4096, NULL);
+ if (fw_image == NULL) {
+ printf("Allocation error\n");
+ close(fd);
+ return;
+ }
+
+ if (read(fd, fw_image, size) != ((ssize_t)(size))) {
+ printf("Read firmware image failed\n");
+ close(fd);
+ spdk_dma_free(fw_image);
+ return;
+ }
+ close(fd);
+
+ printf("Please Input Slot(0 - 7):\n");
+ if (!scanf("%d", &slot)) {
+ printf("Invalid Slot\n");
+ spdk_dma_free(fw_image);
+ while (getchar() != '\n');
+ return;
+ }
+
+ commit_action = SPDK_NVME_FW_COMMIT_REPLACE_AND_ENABLE_IMG;
+ rc = spdk_nvme_ctrlr_update_firmware(ctrlr->ctrlr, fw_image, size, slot, commit_action, &status);
+ if (rc == -ENXIO && status.sct == SPDK_NVME_SCT_COMMAND_SPECIFIC &&
+ status.sc == SPDK_NVME_SC_FIRMWARE_REQ_CONVENTIONAL_RESET) {
+ printf("conventional reset is needed to enable firmware !\n");
+ } else if (rc) {
+ printf("spdk_nvme_ctrlr_update_firmware failed\n");
+ } else {
+ printf("spdk_nvme_ctrlr_update_firmware success\n");
+ }
+ spdk_dma_free(fw_image);
+}
+
+static void
+opal_dump_info(struct spdk_opal_d0_features_info *feat)
+{
+ if (feat->tper.hdr.code) {
+ printf("\nOpal TPer feature:\n");
+ printf("ACKNACK = %s", (feat->tper.acknack ? "Y, " : "N, "));
+ printf("ASYNC = %s", (feat->tper.async ? "Y, " : "N, "));
+ printf("BufferManagement = %s\n", (feat->tper.buffer_management ? "Y, " : "N, "));
+ printf("ComIDManagement = %s", (feat->tper.comid_management ? "Y, " : "N, "));
+ printf("Streaming = %s", (feat->tper.streaming ? "Y, " : "N, "));
+ printf("Sync = %s\n", (feat->tper.sync ? "Y" : "N"));
+ printf("\n");
+ }
+
+ if (feat->locking.hdr.code) {
+ printf("Opal Locking feature:\n");
+ printf("Locked = %s", (feat->locking.locked ? "Y, " : "N, "));
+ printf("Locking Enabled = %s", (feat->locking.locking_enabled ? "Y, " : "N, "));
+ printf("Locking supported = %s\n", (feat->locking.locking_supported ? "Y" : "N"));
+
+ printf("MBR done = %s", (feat->locking.mbr_done ? "Y, " : "N, "));
+ printf("MBR enabled = %s", (feat->locking.mbr_enabled ? "Y, " : "N, "));
+ printf("Media encrypt = %s\n", (feat->locking.media_encryption ? "Y" : "N"));
+ printf("\n");
+ }
+
+ if (feat->geo.hdr.code) {
+ printf("Opal Geometry feature:\n");
+ printf("Align = %s", (feat->geo.alignment_granularity ? "Y, " : "N, "));
+ printf("Logical block size = %d, ", from_be32(&feat->geo.logical_block_size));
+ printf("Lowest aligned LBA = %ld\n", from_be64(&feat->geo.lowest_aligned_lba));
+ printf("\n");
+ }
+
+ if (feat->single_user.hdr.code) {
+ printf("Opal Single User Mode feature:\n");
+ printf("Any in SUM = %s", (feat->single_user.any ? "Y, " : "N, "));
+ printf("All in SUM = %s", (feat->single_user.all ? "Y, " : "N, "));
+ printf("Policy: %s Authority,\n", (feat->single_user.policy ? "Admin" : "Users"));
+ printf("Number of locking objects = %d\n ", from_be32(&feat->single_user.num_locking_objects));
+ printf("\n");
+ }
+
+ if (feat->datastore.hdr.code) {
+ printf("Opal DataStore feature:\n");
+ printf("Table alignment = %d, ", from_be32(&feat->datastore.alignment));
+ printf("Max number of tables = %d, ", from_be16(&feat->datastore.max_tables));
+ printf("Max size of tables = %d\n", from_be32(&feat->datastore.max_table_size));
+ printf("\n");
+ }
+
+ if (feat->v100.hdr.code) {
+ printf("Opal V100 feature:\n");
+ printf("Base comID = %d, ", from_be16(&feat->v100.base_comid));
+ printf("Number of comIDs = %d, ", from_be16(&feat->v100.number_comids));
+ printf("Range crossing = %s\n", (feat->v100.range_crossing ? "N" : "Y"));
+ printf("\n");
+ }
+
+ if (feat->v200.hdr.code) {
+ printf("Opal V200 feature:\n");
+ printf("Base comID = %d, ", from_be16(&feat->v200.base_comid));
+ printf("Number of comIDs = %d, ", from_be16(&feat->v200.num_comids));
+ printf("Initial PIN = %d,\n", feat->v200.initial_pin);
+ printf("Reverted PIN = %d, ", feat->v200.reverted_pin);
+ printf("Number of admins = %d, ", from_be16(&feat->v200.num_locking_admin_auth));
+ printf("Number of users = %d\n", from_be16(&feat->v200.num_locking_user_auth));
+ printf("\n");
+ }
+}
+
+static void
+opal_usage(void)
+{
+ printf("Opal General Usage:\n");
+ printf("\n");
+ printf("\t[1: scan device]\n");
+ printf("\t[2: init - take ownership and activate locking]\n");
+ printf("\t[3: revert tper]\n");
+ printf("\t[4: setup locking range]\n");
+ printf("\t[5: list locking ranges]\n");
+ printf("\t[6: enable user]\n");
+ printf("\t[7: set new password]\n");
+ printf("\t[8: add user to locking range]\n");
+ printf("\t[9: lock/unlock range]\n");
+ printf("\t[10: erase locking range]\n");
+ printf("\t[0: quit]\n");
+}
+
+static void
+opal_scan(struct dev *iter)
+{
+ while (getchar() != '\n');
+ if (spdk_nvme_ctrlr_get_flags(iter->ctrlr) & SPDK_NVME_CTRLR_SECURITY_SEND_RECV_SUPPORTED) {
+ iter->opal_dev = spdk_opal_dev_construct(iter->ctrlr);
+ if (iter->opal_dev == NULL) {
+ return;
+ }
+
+ printf("\n\nOpal Supported:\n");
+ display_controller(iter, CONTROLLER_DISPLAY_SIMPLISTIC);
+ opal_dump_info(spdk_opal_get_d0_features_info(iter->opal_dev));
+ spdk_opal_dev_destruct(iter->opal_dev);
+ } else {
+ printf("%04x:%02x:%02x.%02x: NVMe Security Support/Receive Not supported.\n",
+ iter->pci_addr.domain, iter->pci_addr.bus, iter->pci_addr.dev, iter->pci_addr.func);
+ printf("%04x:%02x:%02x.%02x: Opal Not Supported\n\n\n",
+ iter->pci_addr.domain, iter->pci_addr.bus, iter->pci_addr.dev, iter->pci_addr.func);
+ }
+}
+
+static void
+opal_init(struct dev *iter)
+{
+ char new_passwd[SPDK_OPAL_MAX_PASSWORD_SIZE] = {0};
+ char *passwd_p;
+ int ret;
+ int ch;
+
+ if (spdk_nvme_ctrlr_get_flags(iter->ctrlr) & SPDK_NVME_CTRLR_SECURITY_SEND_RECV_SUPPORTED) {
+ iter->opal_dev = spdk_opal_dev_construct(iter->ctrlr);
+ if (iter->opal_dev == NULL) {
+ return;
+ }
+ printf("Please input the new password for ownership:");
+ while ((ch = getchar()) != '\n' && ch != EOF);
+ passwd_p = get_line(new_passwd, SPDK_OPAL_MAX_PASSWORD_SIZE, stdin, true);
+ printf("\n...\n");
+ if (passwd_p) {
+ ret = spdk_opal_cmd_take_ownership(iter->opal_dev, passwd_p);
+ if (ret) {
+ printf("Take ownership failure: %d\n", ret);
+ spdk_opal_dev_destruct(iter->opal_dev);
+ return;
+ }
+
+ ret = spdk_opal_cmd_activate_locking_sp(iter->opal_dev, passwd_p);
+ if (ret) {
+ printf("Locking SP activate failure: %d\n", ret);
+ spdk_opal_dev_destruct(iter->opal_dev);
+ return;
+ }
+ printf("...\nOpal Init Success\n");
+ } else {
+ printf("Input password invalid. Opal Init failure\n");
+ }
+ spdk_opal_dev_destruct(iter->opal_dev);
+ } else {
+ printf("%04x:%02x:%02x.%02x: NVMe Security Support/Receive Not supported.\nOpal Not Supported\n\n\n",
+ iter->pci_addr.domain, iter->pci_addr.bus, iter->pci_addr.dev, iter->pci_addr.func);
+ }
+}
+
+static void
+opal_locking_usage(void)
+{
+ printf("Choose Opal locking state:\n");
+ printf("\n");
+ printf("\t[1: read write lock]\n");
+ printf("\t[2: read only]\n");
+ printf("\t[3: read write unlock]\n");
+}
+
+static void
+opal_setup_lockingrange(struct dev *iter)
+{
+ char passwd[SPDK_OPAL_MAX_PASSWORD_SIZE] = {0};
+ char *passwd_p;
+ int ret;
+ int ch;
+ uint64_t range_start;
+ uint64_t range_length;
+ int locking_range_id;
+ struct spdk_opal_locking_range_info *info;
+
+ if (spdk_nvme_ctrlr_get_flags(iter->ctrlr) & SPDK_NVME_CTRLR_SECURITY_SEND_RECV_SUPPORTED) {
+ iter->opal_dev = spdk_opal_dev_construct(iter->ctrlr);
+ if (iter->opal_dev == NULL) {
+ return;
+ }
+ printf("Please input the password for setting up locking range:");
+ while ((ch = getchar()) != '\n' && ch != EOF);
+ passwd_p = get_line(passwd, SPDK_OPAL_MAX_PASSWORD_SIZE, stdin, true);
+ printf("\n");
+ if (passwd_p) {
+ printf("Specify locking range id:\n");
+ if (!scanf("%d", &locking_range_id)) {
+ printf("Invalid locking range id\n");
+ spdk_opal_dev_destruct(iter->opal_dev);
+ return;
+ }
+
+ printf("range length:\n");
+ if (!scanf("%" SCNu64, &range_length)) {
+ printf("Invalid range length\n");
+ spdk_opal_dev_destruct(iter->opal_dev);
+ return;
+ }
+
+ printf("range start:\n");
+ if (!scanf("%" SCNu64, &range_start)) {
+ printf("Invalid range start address\n");
+ spdk_opal_dev_destruct(iter->opal_dev);
+ return;
+ }
+ while (getchar() != '\n');
+
+ ret = spdk_opal_cmd_setup_locking_range(iter->opal_dev,
+ OPAL_ADMIN1, locking_range_id, range_start, range_length, passwd_p);
+ if (ret) {
+ printf("Setup locking range failure: %d\n", ret);
+ spdk_opal_dev_destruct(iter->opal_dev);
+ return;
+ }
+
+ ret = spdk_opal_cmd_get_locking_range_info(iter->opal_dev,
+ passwd_p, OPAL_ADMIN1, locking_range_id);
+ if (ret) {
+ printf("Get locking range info failure: %d\n", ret);
+ spdk_opal_dev_destruct(iter->opal_dev);
+ return;
+ }
+ info = spdk_opal_get_locking_range_info(iter->opal_dev, locking_range_id);
+
+ printf("\nlocking range ID: %d\n", info->locking_range_id);
+ printf("range start: %ld\n", info->range_start);
+ printf("range length: %ld\n", info->range_length);
+ printf("read lock enabled: %d\n", info->read_lock_enabled);
+ printf("write lock enabled: %d\n", info->write_lock_enabled);
+ printf("read locked: %d\n", info->read_locked);
+ printf("write locked: %d\n", info->write_locked);
+
+ printf("...\n...\nOpal setup locking range success\n");
+ } else {
+ printf("Input password invalid. Opal setup locking range failure\n");
+ }
+ spdk_opal_dev_destruct(iter->opal_dev);
+ } else {
+ printf("%04x:%02x:%02x.%02x: NVMe Security Support/Receive Not supported.\nOpal Not Supported\n\n\n",
+ iter->pci_addr.domain, iter->pci_addr.bus, iter->pci_addr.dev, iter->pci_addr.func);
+ }
+}
+
+static void
+opal_list_locking_ranges(struct dev *iter)
+{
+ char passwd[SPDK_OPAL_MAX_PASSWORD_SIZE] = {0};
+ char *passwd_p;
+ int ret;
+ int ch;
+ int max_ranges;
+ int i;
+ struct spdk_opal_locking_range_info *info;
+
+ if (spdk_nvme_ctrlr_get_flags(iter->ctrlr) & SPDK_NVME_CTRLR_SECURITY_SEND_RECV_SUPPORTED) {
+ iter->opal_dev = spdk_opal_dev_construct(iter->ctrlr);
+ if (iter->opal_dev == NULL) {
+ return;
+ }
+ printf("Please input password:");
+ while ((ch = getchar()) != '\n' && ch != EOF);
+ passwd_p = get_line(passwd, SPDK_OPAL_MAX_PASSWORD_SIZE, stdin, true);
+ printf("\n");
+ if (passwd_p) {
+ ret = spdk_opal_cmd_get_max_ranges(iter->opal_dev, passwd_p);
+ if (ret <= 0) {
+ printf("get max ranges failure: %d\n", ret);
+ spdk_opal_dev_destruct(iter->opal_dev);
+ return;
+ }
+
+ max_ranges = ret;
+ for (i = 0; i < max_ranges; i++) {
+ ret = spdk_opal_cmd_get_locking_range_info(iter->opal_dev,
+ passwd_p, OPAL_ADMIN1, i);
+ if (ret) {
+ printf("Get locking range info failure: %d\n", ret);
+ spdk_opal_dev_destruct(iter->opal_dev);
+ return;
+ }
+ info = spdk_opal_get_locking_range_info(iter->opal_dev, i);
+ if (info == NULL) {
+ continue;
+ }
+
+ printf("===============================================\n");
+ printf("locking range ID: %d\t", info->locking_range_id);
+ if (i == 0) { printf("(Global Range)"); }
+ printf("\n===============================================\n");
+ printf("range start: %ld\t", info->range_start);
+ printf("range length: %ld\n", info->range_length);
+ printf("read lock enabled: %d\t", info->read_lock_enabled);
+ printf("write lock enabled: %d\t", info->write_lock_enabled);
+ printf("read locked: %d\t", info->read_locked);
+ printf("write locked: %d\n", info->write_locked);
+ printf("\n");
+ }
+ } else {
+ printf("Input password invalid. List locking ranges failure\n");
+ }
+ spdk_opal_dev_destruct(iter->opal_dev);
+ } else {
+ printf("%04x:%02x:%02x.%02x: NVMe Security Support/Receive Not supported.\nOpal Not Supported\n\n\n",
+ iter->pci_addr.domain, iter->pci_addr.bus, iter->pci_addr.dev, iter->pci_addr.func);
+ }
+}
+
+static void
+opal_new_user_enable(struct dev *iter)
+{
+ int user_id;
+ char passwd[SPDK_OPAL_MAX_PASSWORD_SIZE] = {0};
+ char *passwd_p;
+ char user_pw[SPDK_OPAL_MAX_PASSWORD_SIZE] = {0};
+ char *user_pw_p;
+ int ret;
+ int ch;
+
+ if (spdk_nvme_ctrlr_get_flags(iter->ctrlr) & SPDK_NVME_CTRLR_SECURITY_SEND_RECV_SUPPORTED) {
+ iter->opal_dev = spdk_opal_dev_construct(iter->ctrlr);
+ if (iter->opal_dev == NULL) {
+ return;
+ }
+ printf("Please input admin password:");
+ while ((ch = getchar()) != '\n' && ch != EOF);
+ passwd_p = get_line(passwd, SPDK_OPAL_MAX_PASSWORD_SIZE, stdin, true);
+ printf("\n");
+ if (passwd_p) {
+ printf("which user to enable: ");
+ if (!scanf("%d", &user_id)) {
+ printf("Invalid user id\n");
+ spdk_opal_dev_destruct(iter->opal_dev);
+ return;
+ }
+
+ ret = spdk_opal_cmd_enable_user(iter->opal_dev, user_id, passwd_p);
+ if (ret) {
+ printf("Enable user failure error code: %d\n", ret);
+ spdk_opal_dev_destruct(iter->opal_dev);
+ return;
+ }
+ printf("Please set a new password for this user:");
+ while ((ch = getchar()) != '\n' && ch != EOF);
+ user_pw_p = get_line(user_pw, SPDK_OPAL_MAX_PASSWORD_SIZE, stdin, true);
+ if (user_pw_p == NULL) {
+ printf("Input password invalid. Enable user failure\n");
+ spdk_opal_dev_destruct(iter->opal_dev);
+ return;
+ }
+
+ ret = spdk_opal_cmd_set_new_passwd(iter->opal_dev, user_id, user_pw_p, passwd_p, true);
+ if (ret) {
+ printf("Set new password failure error code: %d\n", ret);
+ spdk_opal_dev_destruct(iter->opal_dev);
+ return;
+ }
+
+ printf("\n...\n...\nEnable User Success\n");
+ } else {
+ printf("Input password invalid. Enable user failure\n");
+ }
+ spdk_opal_dev_destruct(iter->opal_dev);
+ } else {
+ printf("%04x:%02x:%02x.%02x: NVMe Security Support/Receive Not supported.\nOpal Not Supported\n\n\n",
+ iter->pci_addr.domain, iter->pci_addr.bus, iter->pci_addr.dev, iter->pci_addr.func);
+ }
+}
+
+static void
+opal_change_password(struct dev *iter)
+{
+ int user_id;
+ char old_passwd[SPDK_OPAL_MAX_PASSWORD_SIZE] = {0};
+ char *old_passwd_p;
+ char new_passwd[SPDK_OPAL_MAX_PASSWORD_SIZE] = {0};
+ char *new_passwd_p;
+ int ret;
+ int ch;
+
+ if (spdk_nvme_ctrlr_get_flags(iter->ctrlr) & SPDK_NVME_CTRLR_SECURITY_SEND_RECV_SUPPORTED) {
+ iter->opal_dev = spdk_opal_dev_construct(iter->ctrlr);
+ if (iter->opal_dev == NULL) {
+ return;
+ }
+ printf("user id: ");
+ if (!scanf("%d", &user_id)) {
+ printf("Invalid user id\n");
+ spdk_opal_dev_destruct(iter->opal_dev);
+ return;
+ }
+ printf("Password:");
+ while ((ch = getchar()) != '\n' && ch != EOF);
+ old_passwd_p = get_line(old_passwd, SPDK_OPAL_MAX_PASSWORD_SIZE, stdin, true);
+ printf("\n");
+ if (old_passwd_p) {
+ printf("Please input new password:\n");
+ new_passwd_p = get_line(new_passwd, SPDK_OPAL_MAX_PASSWORD_SIZE, stdin, true);
+ printf("\n");
+ if (new_passwd_p == NULL) {
+ printf("Input password invalid. Change password failure\n");
+ spdk_opal_dev_destruct(iter->opal_dev);
+ return;
+ }
+
+ ret = spdk_opal_cmd_set_new_passwd(iter->opal_dev, user_id, new_passwd_p, old_passwd_p, false);
+ if (ret) {
+ printf("Set new password failure error code: %d\n", ret);
+ spdk_opal_dev_destruct(iter->opal_dev);
+ return;
+ }
+
+ printf("...\n...\nChange password Success\n");
+ } else {
+ printf("Input password invalid. Change password failure\n");
+ }
+ spdk_opal_dev_destruct(iter->opal_dev);
+ } else {
+ printf("%04x:%02x:%02x.%02x: NVMe Security Support/Receive Not supported.\nOpal Not Supported\n\n\n",
+ iter->pci_addr.domain, iter->pci_addr.bus, iter->pci_addr.dev, iter->pci_addr.func);
+ }
+}
+
+static void
+opal_add_user_to_locking_range(struct dev *iter)
+{
+ int locking_range_id, user_id;
+ char passwd[SPDK_OPAL_MAX_PASSWORD_SIZE] = {0};
+ char *passwd_p;
+ int ret;
+ int ch;
+
+ if (spdk_nvme_ctrlr_get_flags(iter->ctrlr) & SPDK_NVME_CTRLR_SECURITY_SEND_RECV_SUPPORTED) {
+ iter->opal_dev = spdk_opal_dev_construct(iter->ctrlr);
+ if (iter->opal_dev == NULL) {
+ return;
+ }
+ printf("Please input admin password:");
+ while ((ch = getchar()) != '\n' && ch != EOF);
+ passwd_p = get_line(passwd, SPDK_OPAL_MAX_PASSWORD_SIZE, stdin, true);
+ printf("\n");
+ if (passwd_p) {
+ printf("Specify locking range id:\n");
+ if (!scanf("%d", &locking_range_id)) {
+ printf("Invalid locking range id\n");
+ spdk_opal_dev_destruct(iter->opal_dev);
+ return;
+ }
+
+ printf("which user to enable:\n");
+ if (!scanf("%d", &user_id)) {
+ printf("Invalid user id\n");
+ spdk_opal_dev_destruct(iter->opal_dev);
+ return;
+ }
+ while (getchar() != '\n');
+
+ ret = spdk_opal_cmd_add_user_to_locking_range(iter->opal_dev, user_id, locking_range_id,
+ OPAL_READONLY, passwd_p);
+ ret += spdk_opal_cmd_add_user_to_locking_range(iter->opal_dev, user_id, locking_range_id,
+ OPAL_READWRITE, passwd_p);
+ if (ret) {
+ printf("Add user to locking range error: %d\n", ret);
+ spdk_opal_dev_destruct(iter->opal_dev);
+ return;
+ }
+
+ printf("...\n...\nAdd user to locking range Success\n");
+ } else {
+ printf("Input password invalid. Add user to locking range failure\n");
+ }
+ spdk_opal_dev_destruct(iter->opal_dev);
+ } else {
+ printf("%04x:%02x:%02x.%02x: NVMe Security Support/Receive Not supported.\nOpal Not Supported\n\n\n",
+ iter->pci_addr.domain, iter->pci_addr.bus, iter->pci_addr.dev, iter->pci_addr.func);
+ }
+}
+
+static void
+opal_user_lock_unlock_range(struct dev *iter)
+{
+ char passwd[SPDK_OPAL_MAX_PASSWORD_SIZE] = {0};
+ char *passwd_p;
+ int ch;
+ int ret;
+ int user_id;
+ int locking_range_id;
+ int state;
+ enum spdk_opal_lock_state state_flag;
+
+ if (spdk_nvme_ctrlr_get_flags(iter->ctrlr) & SPDK_NVME_CTRLR_SECURITY_SEND_RECV_SUPPORTED) {
+ iter->opal_dev = spdk_opal_dev_construct(iter->ctrlr);
+ if (iter->opal_dev == NULL) {
+ return;
+ }
+ printf("User id: ");
+ if (!scanf("%d", &user_id)) {
+ printf("Invalid user id\n");
+ spdk_opal_dev_destruct(iter->opal_dev);
+ return;
+ }
+
+ printf("Please input password:");
+ while ((ch = getchar()) != '\n' && ch != EOF);
+ passwd_p = get_line(passwd, SPDK_OPAL_MAX_PASSWORD_SIZE, stdin, true);
+ printf("\n");
+ if (passwd_p) {
+ printf("Specify locking range id:\n");
+ if (!scanf("%d", &locking_range_id)) {
+ printf("Invalid locking range id\n");
+ spdk_opal_dev_destruct(iter->opal_dev);
+ return;
+ }
+
+ opal_locking_usage();
+ if (!scanf("%d", &state)) {
+ printf("Invalid option\n");
+ }
+ switch (state) {
+ case 1:
+ state_flag = OPAL_RWLOCK;
+ break;
+ case 2:
+ state_flag = OPAL_READONLY;
+ break;
+ case 3:
+ state_flag = OPAL_READWRITE;
+ break;
+ default:
+ printf("Invalid options\n");
+ spdk_opal_dev_destruct(iter->opal_dev);
+ return;
+ }
+ while (getchar() != '\n');
+
+ ret = spdk_opal_cmd_lock_unlock(iter->opal_dev, user_id, state_flag,
+ locking_range_id, passwd_p);
+ if (ret) {
+ printf("lock/unlock range failure: %d\n", ret);
+ spdk_opal_dev_destruct(iter->opal_dev);
+ return;
+ }
+ printf("...\n...\nLock/unlock range Success\n");
+ } else {
+ printf("Input password invalid. lock/unlock range failure\n");
+ }
+ spdk_opal_dev_destruct(iter->opal_dev);
+ } else {
+ printf("%04x:%02x:%02x.%02x: NVMe Security Support/Receive Not supported.\nOpal Not Supported\n\n\n",
+ iter->pci_addr.domain, iter->pci_addr.bus, iter->pci_addr.dev, iter->pci_addr.func);
+ }
+}
+
+static void
+opal_revert_tper(struct dev *iter)
+{
+ char passwd[SPDK_OPAL_MAX_PASSWORD_SIZE] = {0};
+ char *passwd_p;
+ int ret;
+ int ch;
+
+ if (spdk_nvme_ctrlr_get_flags(iter->ctrlr) & SPDK_NVME_CTRLR_SECURITY_SEND_RECV_SUPPORTED) {
+ iter->opal_dev = spdk_opal_dev_construct(iter->ctrlr);
+ if (iter->opal_dev == NULL) {
+ return;
+ }
+ printf("Please be noted this operation will erase ALL DATA on this drive\n");
+ printf("Please don't ternminate this excecution. Otherwise undefined error may occur\n");
+ printf("Please input password for revert TPer:");
+ while ((ch = getchar()) != '\n' && ch != EOF);
+ passwd_p = get_line(passwd, SPDK_OPAL_MAX_PASSWORD_SIZE, stdin, true);
+ printf("\n...\n");
+ if (passwd_p) {
+ ret = spdk_opal_cmd_revert_tper(iter->opal_dev, passwd_p);
+ if (ret) {
+ printf("Revert TPer failure: %d\n", ret);
+ spdk_opal_dev_destruct(iter->opal_dev);
+ return;
+ }
+ printf("...\nRevert TPer Success\n");
+ } else {
+ printf("Input password invalid. Revert TPer failure\n");
+ }
+ spdk_opal_dev_destruct(iter->opal_dev);
+ } else {
+ printf("%04x:%02x:%02x.%02x: NVMe Security Support/Receive Not supported.\nOpal Not Supported\n\n\n",
+ iter->pci_addr.domain, iter->pci_addr.bus, iter->pci_addr.dev, iter->pci_addr.func);
+ }
+}
+
+static void
+opal_erase_locking_range(struct dev *iter)
+{
+ char passwd[SPDK_OPAL_MAX_PASSWORD_SIZE] = {0};
+ char *passwd_p;
+ int ret;
+ int ch;
+ int locking_range_id;
+
+ if (spdk_nvme_ctrlr_get_flags(iter->ctrlr) & SPDK_NVME_CTRLR_SECURITY_SEND_RECV_SUPPORTED) {
+ iter->opal_dev = spdk_opal_dev_construct(iter->ctrlr);
+ if (iter->opal_dev == NULL) {
+ return;
+ }
+ printf("Please be noted this operation will erase ALL DATA on this range\n");
+ printf("Please input password for erase locking range:");
+ while ((ch = getchar()) != '\n' && ch != EOF);
+ passwd_p = get_line(passwd, SPDK_OPAL_MAX_PASSWORD_SIZE, stdin, true);
+ if (passwd_p) {
+ printf("\nSpecify locking range id:\n");
+ if (!scanf("%d", &locking_range_id)) {
+ printf("Invalid locking range id\n");
+ spdk_opal_dev_destruct(iter->opal_dev);
+ return;
+ }
+ printf("\n...\n");
+ ret = spdk_opal_cmd_secure_erase_locking_range(iter->opal_dev, OPAL_ADMIN1, locking_range_id,
+ passwd_p);
+ if (ret) {
+ printf("Erase locking range failure: %d\n", ret);
+ spdk_opal_dev_destruct(iter->opal_dev);
+ return;
+ }
+ printf("...\nErase locking range Success\n");
+ } else {
+ printf("Input password invalid. Erase locking range failure\n");
+ }
+ spdk_opal_dev_destruct(iter->opal_dev);
+ } else {
+ printf("%04x:%02x:%02x.%02x: NVMe Security Support/Receive Not supported.\nOpal Not Supported\n\n\n",
+ iter->pci_addr.domain, iter->pci_addr.bus, iter->pci_addr.dev, iter->pci_addr.func);
+ }
+}
+
+static void
+test_opal(void)
+{
+ int exit_flag = false;
+ struct dev *ctrlr;
+
+ ctrlr = get_controller();
+ if (ctrlr == NULL) {
+ printf("Invalid controller PCI Address.\n");
+ return;
+ }
+
+ opal_usage();
+ while (!exit_flag) {
+ int cmd;
+ if (!scanf("%d", &cmd)) {
+ printf("Invalid Command: command must be number 0-9\n");
+ while (getchar() != '\n');
+ opal_usage();
+ continue;
+ }
+
+ switch (cmd) {
+ case 0:
+ exit_flag = true;
+ continue;
+ case 1:
+ opal_scan(ctrlr);
+ break;
+ case 2:
+ opal_init(ctrlr); /* Take ownership, Activate Locking SP */
+ break;
+ case 3:
+ opal_revert_tper(ctrlr);
+ break;
+ case 4:
+ opal_setup_lockingrange(ctrlr);
+ break;
+ case 5:
+ opal_list_locking_ranges(ctrlr);
+ break;
+ case 6:
+ opal_new_user_enable(ctrlr);
+ break;
+ case 7:
+ opal_change_password(ctrlr);
+ break;
+ case 8:
+ opal_add_user_to_locking_range(ctrlr);
+ break;
+ case 9:
+ opal_user_lock_unlock_range(ctrlr);
+ break;
+ case 10:
+ opal_erase_locking_range(ctrlr);
+ break;
+
+ default:
+ printf("Invalid option\n");
+ }
+
+ printf("\npress Enter to display Opal cmd menu ...\n");
+ while (getchar() != '\n');
+ opal_usage();
+ }
+}
+
+static void
+args_usage(const char *program_name)
+{
+ printf("%s [options]", program_name);
+ printf("\n");
+ printf("options:\n");
+ printf(" -i shared memory group ID\n");
+}
+
+static int
+parse_args(int argc, char **argv)
+{
+ int op;
+
+ while ((op = getopt(argc, argv, "i:")) != -1) {
+ switch (op) {
+ case 'i':
+ g_shm_id = spdk_strtol(optarg, 10);
+ if (g_shm_id < 0) {
+ fprintf(stderr, "Invalid shared memory ID\n");
+ return g_shm_id;
+ }
+ break;
+ default:
+ args_usage(argv[0]);
+ return 1;
+ }
+ }
+
+ return 0;
+}
+
+int main(int argc, char **argv)
+{
+ int i, rc;
+ struct spdk_env_opts opts;
+
+ rc = parse_args(argc, argv);
+ if (rc != 0) {
+ return rc;
+ }
+
+ spdk_env_opts_init(&opts);
+ opts.name = "nvme_manage";
+ opts.core_mask = "0x1";
+ opts.shm_id = g_shm_id;
+ if (spdk_env_init(&opts) < 0) {
+ fprintf(stderr, "Unable to initialize SPDK env\n");
+ return 1;
+ }
+
+ if (spdk_nvme_probe(NULL, NULL, probe_cb, attach_cb, NULL) != 0) {
+ fprintf(stderr, "spdk_nvme_probe() failed\n");
+ return 1;
+ }
+
+ qsort(devs, num_devs, sizeof(devs[0]), cmp_devs);
+
+ usage();
+
+ while (1) {
+ int cmd;
+ bool exit_flag = false;
+
+ if (!scanf("%d", &cmd)) {
+ printf("Invalid Command: command must be number 1-8\n");
+ while (getchar() != '\n');
+ usage();
+ continue;
+ }
+ switch (cmd) {
+ case 1:
+ display_controller_list();
+ break;
+ case 2:
+ add_ns();
+ break;
+ case 3:
+ delete_ns();
+ break;
+ case 4:
+ attach_and_detach_ns(SPDK_NVME_NS_CTRLR_ATTACH);
+ break;
+ case 5:
+ attach_and_detach_ns(SPDK_NVME_NS_CTRLR_DETACH);
+ break;
+ case 6:
+ format_nvm();
+ break;
+ case 7:
+ update_firmware_image();
+ break;
+ case 8:
+ test_opal();
+ break;
+ case 9:
+ exit_flag = true;
+ break;
+ default:
+ printf("Invalid Command\n");
+ break;
+ }
+
+ if (exit_flag) {
+ break;
+ }
+
+ while (getchar() != '\n');
+ printf("press Enter to display cmd menu ...\n");
+ while (getchar() != '\n');
+ usage();
+ }
+
+ printf("Cleaning up...\n");
+
+ for (i = 0; i < num_devs; i++) {
+ struct dev *dev = &devs[i];
+ spdk_nvme_detach(dev->ctrlr);
+ }
+
+ return 0;
+}
diff --git a/src/spdk/examples/nvme/perf/.gitignore b/src/spdk/examples/nvme/perf/.gitignore
new file mode 100644
index 000000000..bd14107d8
--- /dev/null
+++ b/src/spdk/examples/nvme/perf/.gitignore
@@ -0,0 +1 @@
+perf
diff --git a/src/spdk/examples/nvme/perf/Makefile b/src/spdk/examples/nvme/perf/Makefile
new file mode 100644
index 000000000..0742f1842
--- /dev/null
+++ b/src/spdk/examples/nvme/perf/Makefile
@@ -0,0 +1,49 @@
+#
+# BSD LICENSE
+#
+# Copyright (c) Intel Corporation.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Intel Corporation nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
+
+APP = perf
+
+include $(SPDK_ROOT_DIR)/mk/nvme.libtest.mk
+
+ifeq ($(OS),Linux)
+SYS_LIBS += -laio
+CFLAGS += -DHAVE_LIBAIO
+endif
+
+install: $(APP)
+ $(INSTALL_EXAMPLE)
+
+uninstall:
+ $(UNINSTALL_EXAMPLE)
diff --git a/src/spdk/examples/nvme/perf/README.md b/src/spdk/examples/nvme/perf/README.md
new file mode 100644
index 000000000..e5ec38d12
--- /dev/null
+++ b/src/spdk/examples/nvme/perf/README.md
@@ -0,0 +1,5 @@
+# Compiling perf on FreeBSD
+
+To use perf test on FreeBSD over NVMe-oF, explicitly link userspace library of HBA. For example, on a setup with Mellanox HBA,
+
+ LIBS += -lmlx5
diff --git a/src/spdk/examples/nvme/perf/perf.c b/src/spdk/examples/nvme/perf/perf.c
new file mode 100644
index 000000000..9e8cf6793
--- /dev/null
+++ b/src/spdk/examples/nvme/perf/perf.c
@@ -0,0 +1,2308 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+
+#include "spdk/env.h"
+#include "spdk/fd.h"
+#include "spdk/nvme.h"
+#include "spdk/vmd.h"
+#include "spdk/queue.h"
+#include "spdk/string.h"
+#include "spdk/nvme_intel.h"
+#include "spdk/histogram_data.h"
+#include "spdk/endian.h"
+#include "spdk/dif.h"
+#include "spdk/util.h"
+#include "spdk/log.h"
+#include "spdk/likely.h"
+
+#ifdef SPDK_CONFIG_URING
+#include <liburing.h>
+#endif
+
+#if HAVE_LIBAIO
+#include <libaio.h>
+#endif
+
+struct ctrlr_entry {
+ struct spdk_nvme_ctrlr *ctrlr;
+ enum spdk_nvme_transport_type trtype;
+ struct spdk_nvme_intel_rw_latency_page *latency_page;
+
+ struct spdk_nvme_qpair **unused_qpairs;
+
+ struct ctrlr_entry *next;
+ char name[1024];
+};
+
+enum entry_type {
+ ENTRY_TYPE_NVME_NS,
+ ENTRY_TYPE_AIO_FILE,
+ ENTRY_TYPE_URING_FILE,
+};
+
+struct ns_fn_table;
+
+struct ns_entry {
+ enum entry_type type;
+ const struct ns_fn_table *fn_table;
+
+ union {
+ struct {
+ struct spdk_nvme_ctrlr *ctrlr;
+ struct spdk_nvme_ns *ns;
+ } nvme;
+#ifdef SPDK_CONFIG_URING
+ struct {
+ int fd;
+ } uring;
+#endif
+#if HAVE_LIBAIO
+ struct {
+ int fd;
+ } aio;
+#endif
+ } u;
+
+ struct ns_entry *next;
+ uint32_t io_size_blocks;
+ uint32_t num_io_requests;
+ uint64_t size_in_ios;
+ uint32_t block_size;
+ uint32_t md_size;
+ bool md_interleave;
+ bool pi_loc;
+ enum spdk_nvme_pi_type pi_type;
+ uint32_t io_flags;
+ char name[1024];
+};
+
+static const double g_latency_cutoffs[] = {
+ 0.01,
+ 0.10,
+ 0.25,
+ 0.50,
+ 0.75,
+ 0.90,
+ 0.95,
+ 0.98,
+ 0.99,
+ 0.995,
+ 0.999,
+ 0.9999,
+ 0.99999,
+ 0.999999,
+ 0.9999999,
+ -1,
+};
+
+struct ns_worker_ctx {
+ struct ns_entry *entry;
+ uint64_t io_completed;
+ uint64_t last_io_completed;
+ uint64_t total_tsc;
+ uint64_t min_tsc;
+ uint64_t max_tsc;
+ uint64_t current_queue_depth;
+ uint64_t offset_in_ios;
+ bool is_draining;
+
+ union {
+ struct {
+ int num_active_qpairs;
+ int num_all_qpairs;
+ struct spdk_nvme_qpair **qpair;
+ struct spdk_nvme_poll_group *group;
+ int last_qpair;
+ } nvme;
+
+#ifdef SPDK_CONFIG_URING
+ struct {
+ struct io_uring ring;
+ uint64_t io_inflight;
+ uint64_t io_pending;
+ struct io_uring_cqe **cqes;
+
+ } uring;
+#endif
+#if HAVE_LIBAIO
+ struct {
+ struct io_event *events;
+ io_context_t ctx;
+ } aio;
+#endif
+ } u;
+
+ struct ns_worker_ctx *next;
+
+ struct spdk_histogram_data *histogram;
+};
+
+struct perf_task {
+ struct ns_worker_ctx *ns_ctx;
+ struct iovec iov;
+ struct iovec md_iov;
+ uint64_t submit_tsc;
+ bool is_read;
+ struct spdk_dif_ctx dif_ctx;
+#if HAVE_LIBAIO
+ struct iocb iocb;
+#endif
+};
+
+struct worker_thread {
+ struct ns_worker_ctx *ns_ctx;
+ struct worker_thread *next;
+ unsigned lcore;
+};
+
+struct ns_fn_table {
+ void (*setup_payload)(struct perf_task *task, uint8_t pattern);
+
+ int (*submit_io)(struct perf_task *task, struct ns_worker_ctx *ns_ctx,
+ struct ns_entry *entry, uint64_t offset_in_ios);
+
+ void (*check_io)(struct ns_worker_ctx *ns_ctx);
+
+ void (*verify_io)(struct perf_task *task, struct ns_entry *entry);
+
+ int (*init_ns_worker_ctx)(struct ns_worker_ctx *ns_ctx);
+
+ void (*cleanup_ns_worker_ctx)(struct ns_worker_ctx *ns_ctx);
+};
+
+static int g_outstanding_commands;
+
+static bool g_latency_ssd_tracking_enable;
+static int g_latency_sw_tracking_level;
+
+static bool g_vmd;
+static const char *g_workload_type;
+static struct ctrlr_entry *g_controllers;
+static struct ns_entry *g_namespaces;
+static int g_num_namespaces;
+static struct worker_thread *g_workers;
+static int g_num_workers;
+static uint32_t g_master_core;
+
+static uint64_t g_tsc_rate;
+
+static uint32_t g_io_align = 0x200;
+static uint32_t g_io_size_bytes;
+static uint32_t g_max_io_md_size;
+static uint32_t g_max_io_size_blocks;
+static uint32_t g_metacfg_pract_flag;
+static uint32_t g_metacfg_prchk_flags;
+static int g_rw_percentage = -1;
+static int g_is_random;
+static int g_queue_depth;
+static int g_nr_io_queues_per_ns = 1;
+static int g_nr_unused_io_queues;
+static int g_time_in_sec;
+static uint32_t g_max_completions;
+static int g_dpdk_mem;
+static int g_shm_id = -1;
+static uint32_t g_disable_sq_cmb;
+static bool g_use_uring;
+static bool g_no_pci;
+static bool g_warn;
+static bool g_header_digest;
+static bool g_data_digest;
+static bool g_no_shn_notification;
+static bool g_mix_specified;
+/* Default to 10 seconds for the keep alive value. This value is arbitrary. */
+static uint32_t g_keep_alive_timeout_in_ms = 10000;
+
+static const char *g_core_mask;
+
+struct trid_entry {
+ struct spdk_nvme_transport_id trid;
+ uint16_t nsid;
+ TAILQ_ENTRY(trid_entry) tailq;
+};
+
+static TAILQ_HEAD(, trid_entry) g_trid_list = TAILQ_HEAD_INITIALIZER(g_trid_list);
+
+static int g_file_optind; /* Index of first filename in argv */
+
+static inline void
+task_complete(struct perf_task *task);
+
+#ifdef SPDK_CONFIG_URING
+
+static void
+uring_setup_payload(struct perf_task *task, uint8_t pattern)
+{
+ task->iov.iov_base = spdk_dma_zmalloc(g_io_size_bytes, g_io_align, NULL);
+ task->iov.iov_len = g_io_size_bytes;
+ if (task->iov.iov_base == NULL) {
+ fprintf(stderr, "spdk_dma_zmalloc() for task->iov.iov_base failed\n");
+ exit(1);
+ }
+ memset(task->iov.iov_base, pattern, task->iov.iov_len);
+}
+
+static int
+uring_submit_io(struct perf_task *task, struct ns_worker_ctx *ns_ctx,
+ struct ns_entry *entry, uint64_t offset_in_ios)
+{
+ struct io_uring_sqe *sqe;
+
+ sqe = io_uring_get_sqe(&ns_ctx->u.uring.ring);
+ if (!sqe) {
+ fprintf(stderr, "Cannot get sqe\n");
+ return -1;
+ }
+
+ if (task->is_read) {
+ io_uring_prep_readv(sqe, entry->u.uring.fd, &task->iov, 1, offset_in_ios * task->iov.iov_len);
+ } else {
+ io_uring_prep_writev(sqe, entry->u.uring.fd, &task->iov, 1, offset_in_ios * task->iov.iov_len);
+ }
+
+ io_uring_sqe_set_data(sqe, task);
+ ns_ctx->u.uring.io_pending++;
+
+ return 0;
+}
+
+static void
+uring_check_io(struct ns_worker_ctx *ns_ctx)
+{
+ int i, count, to_complete, to_submit, ret = 0;
+ struct perf_task *task;
+
+ to_submit = ns_ctx->u.uring.io_pending;
+
+ if (to_submit > 0) {
+ /* If there are I/O to submit, use io_uring_submit here.
+ * It will automatically call spdk_io_uring_enter appropriately. */
+ ret = io_uring_submit(&ns_ctx->u.uring.ring);
+ if (ret < 0) {
+ return;
+ }
+ ns_ctx->u.uring.io_pending = 0;
+ ns_ctx->u.uring.io_inflight += to_submit;
+ }
+
+ to_complete = ns_ctx->u.uring.io_inflight;
+ if (to_complete > 0) {
+ count = io_uring_peek_batch_cqe(&ns_ctx->u.uring.ring, ns_ctx->u.uring.cqes, to_complete);
+ ns_ctx->u.uring.io_inflight -= count;
+ for (i = 0; i < count; i++) {
+ assert(ns_ctx->u.uring.cqes[i] != NULL);
+ task = (struct perf_task *)ns_ctx->u.uring.cqes[i]->user_data;
+ if (ns_ctx->u.uring.cqes[i]->res != (int)task->iov.iov_len) {
+ fprintf(stderr, "cqe[i]->status=%d\n", ns_ctx->u.uring.cqes[i]->res);
+ exit(0);
+ }
+ io_uring_cqe_seen(&ns_ctx->u.uring.ring, ns_ctx->u.uring.cqes[i]);
+ task_complete(task);
+ }
+ }
+}
+
+static void
+uring_verify_io(struct perf_task *task, struct ns_entry *entry)
+{
+}
+
+static int
+uring_init_ns_worker_ctx(struct ns_worker_ctx *ns_ctx)
+{
+ if (io_uring_queue_init(g_queue_depth, &ns_ctx->u.uring.ring, 0) < 0) {
+ SPDK_ERRLOG("uring I/O context setup failure\n");
+ return -1;
+ }
+
+ ns_ctx->u.uring.cqes = calloc(g_queue_depth, sizeof(struct io_uring_cqe *));
+ if (!ns_ctx->u.uring.cqes) {
+ io_uring_queue_exit(&ns_ctx->u.uring.ring);
+ return -1;
+ }
+
+ return 0;
+}
+
+static void
+uring_cleanup_ns_worker_ctx(struct ns_worker_ctx *ns_ctx)
+{
+ io_uring_queue_exit(&ns_ctx->u.uring.ring);
+ free(ns_ctx->u.uring.cqes);
+}
+
+static const struct ns_fn_table uring_fn_table = {
+ .setup_payload = uring_setup_payload,
+ .submit_io = uring_submit_io,
+ .check_io = uring_check_io,
+ .verify_io = uring_verify_io,
+ .init_ns_worker_ctx = uring_init_ns_worker_ctx,
+ .cleanup_ns_worker_ctx = uring_cleanup_ns_worker_ctx,
+};
+
+#endif
+
+#ifdef HAVE_LIBAIO
+static void
+aio_setup_payload(struct perf_task *task, uint8_t pattern)
+{
+ task->iov.iov_base = spdk_dma_zmalloc(g_io_size_bytes, g_io_align, NULL);
+ task->iov.iov_len = g_io_size_bytes;
+ if (task->iov.iov_base == NULL) {
+ fprintf(stderr, "spdk_dma_zmalloc() for task->buf failed\n");
+ exit(1);
+ }
+ memset(task->iov.iov_base, pattern, task->iov.iov_len);
+}
+
+static int
+aio_submit(io_context_t aio_ctx, struct iocb *iocb, int fd, enum io_iocb_cmd cmd,
+ struct iovec *iov, uint64_t offset, void *cb_ctx)
+{
+ iocb->aio_fildes = fd;
+ iocb->aio_reqprio = 0;
+ iocb->aio_lio_opcode = cmd;
+ iocb->u.c.buf = iov->iov_base;
+ iocb->u.c.nbytes = iov->iov_len;
+ iocb->u.c.offset = offset * iov->iov_len;
+ iocb->data = cb_ctx;
+
+ if (io_submit(aio_ctx, 1, &iocb) < 0) {
+ printf("io_submit");
+ return -1;
+ }
+
+ return 0;
+}
+
+static int
+aio_submit_io(struct perf_task *task, struct ns_worker_ctx *ns_ctx,
+ struct ns_entry *entry, uint64_t offset_in_ios)
+{
+ if (task->is_read) {
+ return aio_submit(ns_ctx->u.aio.ctx, &task->iocb, entry->u.aio.fd, IO_CMD_PREAD,
+ &task->iov, offset_in_ios, task);
+ } else {
+ return aio_submit(ns_ctx->u.aio.ctx, &task->iocb, entry->u.aio.fd, IO_CMD_PWRITE,
+ &task->iov, offset_in_ios, task);
+ }
+}
+
+static void
+aio_check_io(struct ns_worker_ctx *ns_ctx)
+{
+ int count, i;
+ struct timespec timeout;
+
+ timeout.tv_sec = 0;
+ timeout.tv_nsec = 0;
+
+ count = io_getevents(ns_ctx->u.aio.ctx, 1, g_queue_depth, ns_ctx->u.aio.events, &timeout);
+ if (count < 0) {
+ fprintf(stderr, "io_getevents error\n");
+ exit(1);
+ }
+
+ for (i = 0; i < count; i++) {
+ task_complete(ns_ctx->u.aio.events[i].data);
+ }
+}
+
+static void
+aio_verify_io(struct perf_task *task, struct ns_entry *entry)
+{
+}
+
+static int
+aio_init_ns_worker_ctx(struct ns_worker_ctx *ns_ctx)
+{
+ ns_ctx->u.aio.events = calloc(g_queue_depth, sizeof(struct io_event));
+ if (!ns_ctx->u.aio.events) {
+ return -1;
+ }
+ ns_ctx->u.aio.ctx = 0;
+ if (io_setup(g_queue_depth, &ns_ctx->u.aio.ctx) < 0) {
+ free(ns_ctx->u.aio.events);
+ perror("io_setup");
+ return -1;
+ }
+ return 0;
+}
+
+static void
+aio_cleanup_ns_worker_ctx(struct ns_worker_ctx *ns_ctx)
+{
+ io_destroy(ns_ctx->u.aio.ctx);
+ free(ns_ctx->u.aio.events);
+}
+
+static const struct ns_fn_table aio_fn_table = {
+ .setup_payload = aio_setup_payload,
+ .submit_io = aio_submit_io,
+ .check_io = aio_check_io,
+ .verify_io = aio_verify_io,
+ .init_ns_worker_ctx = aio_init_ns_worker_ctx,
+ .cleanup_ns_worker_ctx = aio_cleanup_ns_worker_ctx,
+};
+
+#endif /* HAVE_LIBAIO */
+
+#if defined(HAVE_LIBAIO) || defined(SPDK_CONFIG_URING)
+
+static int
+register_file(const char *path)
+{
+ struct ns_entry *entry;
+
+ int flags, fd;
+ uint64_t size;
+ uint32_t blklen;
+
+ if (g_rw_percentage == 100) {
+ flags = O_RDONLY;
+ } else if (g_rw_percentage == 0) {
+ flags = O_WRONLY;
+ } else {
+ flags = O_RDWR;
+ }
+
+ flags |= O_DIRECT;
+
+ fd = open(path, flags);
+ if (fd < 0) {
+ fprintf(stderr, "Could not open device %s: %s\n", path, strerror(errno));
+ return -1;
+ }
+
+ size = spdk_fd_get_size(fd);
+ if (size == 0) {
+ fprintf(stderr, "Could not determine size of device %s\n", path);
+ close(fd);
+ return -1;
+ }
+
+ blklen = spdk_fd_get_blocklen(fd);
+ if (blklen == 0) {
+ fprintf(stderr, "Could not determine block size of device %s\n", path);
+ close(fd);
+ return -1;
+ }
+
+ /*
+ * TODO: This should really calculate the LCM of the current g_io_align and blklen.
+ * For now, it's fairly safe to just assume all block sizes are powers of 2.
+ */
+ if (g_io_align < blklen) {
+ g_io_align = blklen;
+ }
+
+ entry = malloc(sizeof(struct ns_entry));
+ if (entry == NULL) {
+ close(fd);
+ perror("ns_entry malloc");
+ return -1;
+ }
+
+ if (g_use_uring) {
+#ifdef SPDK_CONFIG_URING
+ entry->type = ENTRY_TYPE_URING_FILE;
+ entry->fn_table = &uring_fn_table;
+ entry->u.uring.fd = fd;
+#endif
+ } else {
+#if HAVE_LIBAIO
+ entry->type = ENTRY_TYPE_AIO_FILE;
+ entry->fn_table = &aio_fn_table;
+ entry->u.aio.fd = fd;
+#endif
+ }
+ entry->size_in_ios = size / g_io_size_bytes;
+ entry->io_size_blocks = g_io_size_bytes / blklen;
+
+ snprintf(entry->name, sizeof(entry->name), "%s", path);
+
+ g_num_namespaces++;
+ entry->next = g_namespaces;
+ g_namespaces = entry;
+
+ return 0;
+}
+
+static int
+register_files(int argc, char **argv)
+{
+ int i;
+
+ /* Treat everything after the options as files for AIO/URING */
+ for (i = g_file_optind; i < argc; i++) {
+ if (register_file(argv[i]) != 0) {
+ return 1;
+ }
+ }
+
+ return 0;
+}
+#endif
+
+static void io_complete(void *ctx, const struct spdk_nvme_cpl *cpl);
+
+static void
+nvme_setup_payload(struct perf_task *task, uint8_t pattern)
+{
+ uint32_t max_io_size_bytes, max_io_md_size;
+
+ /* maximum extended lba format size from all active namespace,
+ * it's same with g_io_size_bytes for namespace without metadata.
+ */
+ max_io_size_bytes = g_io_size_bytes + g_max_io_md_size * g_max_io_size_blocks;
+ task->iov.iov_base = spdk_dma_zmalloc(max_io_size_bytes, g_io_align, NULL);
+ task->iov.iov_len = max_io_size_bytes;
+ if (task->iov.iov_base == NULL) {
+ fprintf(stderr, "task->buf spdk_dma_zmalloc failed\n");
+ exit(1);
+ }
+ memset(task->iov.iov_base, pattern, task->iov.iov_len);
+
+ max_io_md_size = g_max_io_md_size * g_max_io_size_blocks;
+ if (max_io_md_size != 0) {
+ task->md_iov.iov_base = spdk_dma_zmalloc(max_io_md_size, g_io_align, NULL);
+ task->md_iov.iov_len = max_io_md_size;
+ if (task->md_iov.iov_base == NULL) {
+ fprintf(stderr, "task->md_buf spdk_dma_zmalloc failed\n");
+ spdk_dma_free(task->iov.iov_base);
+ exit(1);
+ }
+ }
+}
+
+static int
+nvme_submit_io(struct perf_task *task, struct ns_worker_ctx *ns_ctx,
+ struct ns_entry *entry, uint64_t offset_in_ios)
+{
+ uint64_t lba;
+ int rc;
+ int qp_num;
+
+ enum dif_mode {
+ DIF_MODE_NONE = 0,
+ DIF_MODE_DIF = 1,
+ DIF_MODE_DIX = 2,
+ } mode = DIF_MODE_NONE;
+
+ lba = offset_in_ios * entry->io_size_blocks;
+
+ if (entry->md_size != 0 && !(entry->io_flags & SPDK_NVME_IO_FLAGS_PRACT)) {
+ if (entry->md_interleave) {
+ mode = DIF_MODE_DIF;
+ } else {
+ mode = DIF_MODE_DIX;
+ }
+ }
+
+ qp_num = ns_ctx->u.nvme.last_qpair;
+ ns_ctx->u.nvme.last_qpair++;
+ if (ns_ctx->u.nvme.last_qpair == ns_ctx->u.nvme.num_active_qpairs) {
+ ns_ctx->u.nvme.last_qpair = 0;
+ }
+
+ if (mode != DIF_MODE_NONE) {
+ rc = spdk_dif_ctx_init(&task->dif_ctx, entry->block_size, entry->md_size,
+ entry->md_interleave, entry->pi_loc,
+ (enum spdk_dif_type)entry->pi_type, entry->io_flags,
+ lba, 0xFFFF, (uint16_t)entry->io_size_blocks, 0, 0);
+ if (rc != 0) {
+ fprintf(stderr, "Initialization of DIF context failed\n");
+ exit(1);
+ }
+ }
+
+ if (task->is_read) {
+ return spdk_nvme_ns_cmd_read_with_md(entry->u.nvme.ns, ns_ctx->u.nvme.qpair[qp_num],
+ task->iov.iov_base, task->md_iov.iov_base,
+ lba,
+ entry->io_size_blocks, io_complete,
+ task, entry->io_flags,
+ task->dif_ctx.apptag_mask, task->dif_ctx.app_tag);
+ } else {
+ switch (mode) {
+ case DIF_MODE_DIF:
+ rc = spdk_dif_generate(&task->iov, 1, entry->io_size_blocks, &task->dif_ctx);
+ if (rc != 0) {
+ fprintf(stderr, "Generation of DIF failed\n");
+ return rc;
+ }
+ break;
+ case DIF_MODE_DIX:
+ rc = spdk_dix_generate(&task->iov, 1, &task->md_iov, entry->io_size_blocks,
+ &task->dif_ctx);
+ if (rc != 0) {
+ fprintf(stderr, "Generation of DIX failed\n");
+ return rc;
+ }
+ break;
+ default:
+ break;
+ }
+
+ return spdk_nvme_ns_cmd_write_with_md(entry->u.nvme.ns, ns_ctx->u.nvme.qpair[qp_num],
+ task->iov.iov_base, task->md_iov.iov_base,
+ lba,
+ entry->io_size_blocks, io_complete,
+ task, entry->io_flags,
+ task->dif_ctx.apptag_mask, task->dif_ctx.app_tag);
+ }
+}
+
+static void
+perf_disconnect_cb(struct spdk_nvme_qpair *qpair, void *ctx)
+{
+
+}
+
+static void
+nvme_check_io(struct ns_worker_ctx *ns_ctx)
+{
+ int64_t rc;
+
+ rc = spdk_nvme_poll_group_process_completions(ns_ctx->u.nvme.group, 0, perf_disconnect_cb);
+ if (rc < 0) {
+ fprintf(stderr, "NVMe io qpair process completion error\n");
+ exit(1);
+ }
+}
+
+static void
+nvme_verify_io(struct perf_task *task, struct ns_entry *entry)
+{
+ struct spdk_dif_error err_blk = {};
+ int rc;
+
+ if (!task->is_read || (entry->io_flags & SPDK_NVME_IO_FLAGS_PRACT)) {
+ return;
+ }
+
+ if (entry->md_interleave) {
+ rc = spdk_dif_verify(&task->iov, 1, entry->io_size_blocks, &task->dif_ctx,
+ &err_blk);
+ if (rc != 0) {
+ fprintf(stderr, "DIF error detected. type=%d, offset=%" PRIu32 "\n",
+ err_blk.err_type, err_blk.err_offset);
+ }
+ } else {
+ rc = spdk_dix_verify(&task->iov, 1, &task->md_iov, entry->io_size_blocks,
+ &task->dif_ctx, &err_blk);
+ if (rc != 0) {
+ fprintf(stderr, "DIX error detected. type=%d, offset=%" PRIu32 "\n",
+ err_blk.err_type, err_blk.err_offset);
+ }
+ }
+}
+
+/*
+ * TODO: If a controller has multiple namespaces, they could all use the same queue.
+ * For now, give each namespace/thread combination its own queue.
+ */
+static int
+nvme_init_ns_worker_ctx(struct ns_worker_ctx *ns_ctx)
+{
+ struct spdk_nvme_io_qpair_opts opts;
+ struct ns_entry *entry = ns_ctx->entry;
+ struct spdk_nvme_poll_group *group;
+ struct spdk_nvme_qpair *qpair;
+ int i;
+
+ ns_ctx->u.nvme.num_active_qpairs = g_nr_io_queues_per_ns;
+ ns_ctx->u.nvme.num_all_qpairs = g_nr_io_queues_per_ns + g_nr_unused_io_queues;
+ ns_ctx->u.nvme.qpair = calloc(ns_ctx->u.nvme.num_all_qpairs, sizeof(struct spdk_nvme_qpair *));
+ if (!ns_ctx->u.nvme.qpair) {
+ return -1;
+ }
+
+ spdk_nvme_ctrlr_get_default_io_qpair_opts(entry->u.nvme.ctrlr, &opts, sizeof(opts));
+ if (opts.io_queue_requests < entry->num_io_requests) {
+ opts.io_queue_requests = entry->num_io_requests;
+ }
+ opts.delay_cmd_submit = true;
+ opts.create_only = true;
+
+ ns_ctx->u.nvme.group = spdk_nvme_poll_group_create(NULL);
+ if (ns_ctx->u.nvme.group == NULL) {
+ goto poll_group_failed;
+ }
+
+ group = ns_ctx->u.nvme.group;
+ for (i = 0; i < ns_ctx->u.nvme.num_all_qpairs; i++) {
+ ns_ctx->u.nvme.qpair[i] = spdk_nvme_ctrlr_alloc_io_qpair(entry->u.nvme.ctrlr, &opts,
+ sizeof(opts));
+ qpair = ns_ctx->u.nvme.qpair[i];
+ if (!qpair) {
+ printf("ERROR: spdk_nvme_ctrlr_alloc_io_qpair failed\n");
+ goto qpair_failed;
+ }
+
+ if (spdk_nvme_poll_group_add(group, qpair)) {
+ printf("ERROR: unable to add I/O qpair to poll group.\n");
+ spdk_nvme_ctrlr_free_io_qpair(qpair);
+ goto qpair_failed;
+ }
+
+ if (spdk_nvme_ctrlr_connect_io_qpair(entry->u.nvme.ctrlr, qpair)) {
+ printf("ERROR: unable to connect I/O qpair.\n");
+ spdk_nvme_poll_group_remove(group, qpair);
+ spdk_nvme_ctrlr_free_io_qpair(qpair);
+ goto qpair_failed;
+ }
+ }
+
+ return 0;
+
+qpair_failed:
+ for (; i > 0; --i) {
+ spdk_nvme_poll_group_remove(ns_ctx->u.nvme.group, ns_ctx->u.nvme.qpair[i - 1]);
+ spdk_nvme_ctrlr_free_io_qpair(ns_ctx->u.nvme.qpair[i - 1]);
+ }
+
+ spdk_nvme_poll_group_destroy(ns_ctx->u.nvme.group);
+poll_group_failed:
+ free(ns_ctx->u.nvme.qpair);
+ return -1;
+}
+
+static void
+nvme_cleanup_ns_worker_ctx(struct ns_worker_ctx *ns_ctx)
+{
+ int i;
+
+ for (i = 0; i < ns_ctx->u.nvme.num_all_qpairs; i++) {
+ spdk_nvme_poll_group_remove(ns_ctx->u.nvme.group, ns_ctx->u.nvme.qpair[i]);
+ spdk_nvme_ctrlr_free_io_qpair(ns_ctx->u.nvme.qpair[i]);
+ }
+
+ spdk_nvme_poll_group_destroy(ns_ctx->u.nvme.group);
+ free(ns_ctx->u.nvme.qpair);
+}
+
+static const struct ns_fn_table nvme_fn_table = {
+ .setup_payload = nvme_setup_payload,
+ .submit_io = nvme_submit_io,
+ .check_io = nvme_check_io,
+ .verify_io = nvme_verify_io,
+ .init_ns_worker_ctx = nvme_init_ns_worker_ctx,
+ .cleanup_ns_worker_ctx = nvme_cleanup_ns_worker_ctx,
+};
+
+static int
+build_nvme_name(char *name, size_t length, struct spdk_nvme_ctrlr *ctrlr)
+{
+ const struct spdk_nvme_transport_id *trid;
+ int res = 0;
+
+ trid = spdk_nvme_ctrlr_get_transport_id(ctrlr);
+
+ switch (trid->trtype) {
+ case SPDK_NVME_TRANSPORT_PCIE:
+ res = snprintf(name, length, "PCIE (%s)", trid->traddr);
+ break;
+ case SPDK_NVME_TRANSPORT_RDMA:
+ res = snprintf(name, length, "RDMA (addr:%s subnqn:%s)", trid->traddr, trid->subnqn);
+ break;
+ case SPDK_NVME_TRANSPORT_TCP:
+ res = snprintf(name, length, "TCP (addr:%s subnqn:%s)", trid->traddr, trid->subnqn);
+ break;
+
+ default:
+ fprintf(stderr, "Unknown transport type %d\n", trid->trtype);
+ break;
+ }
+ return res;
+}
+
+static void
+build_nvme_ns_name(char *name, size_t length, struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid)
+{
+ int res = 0;
+
+ res = build_nvme_name(name, length, ctrlr);
+ if (res > 0) {
+ snprintf(name + res, length - res, " NSID %u", nsid);
+ }
+
+}
+
+static void
+register_ns(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns *ns)
+{
+ struct ns_entry *entry;
+ const struct spdk_nvme_ctrlr_data *cdata;
+ uint32_t max_xfer_size, entries, sector_size;
+ uint64_t ns_size;
+ struct spdk_nvme_io_qpair_opts opts;
+
+ cdata = spdk_nvme_ctrlr_get_data(ctrlr);
+
+ if (!spdk_nvme_ns_is_active(ns)) {
+ printf("Controller %-20.20s (%-20.20s): Skipping inactive NS %u\n",
+ cdata->mn, cdata->sn,
+ spdk_nvme_ns_get_id(ns));
+ g_warn = true;
+ return;
+ }
+
+ ns_size = spdk_nvme_ns_get_size(ns);
+ sector_size = spdk_nvme_ns_get_sector_size(ns);
+
+ if (ns_size < g_io_size_bytes || sector_size > g_io_size_bytes) {
+ printf("WARNING: controller %-20.20s (%-20.20s) ns %u has invalid "
+ "ns size %" PRIu64 " / block size %u for I/O size %u\n",
+ cdata->mn, cdata->sn, spdk_nvme_ns_get_id(ns),
+ ns_size, spdk_nvme_ns_get_sector_size(ns), g_io_size_bytes);
+ g_warn = true;
+ return;
+ }
+
+ max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns);
+ spdk_nvme_ctrlr_get_default_io_qpair_opts(ctrlr, &opts, sizeof(opts));
+ /* NVMe driver may add additional entries based on
+ * stripe size and maximum transfer size, we assume
+ * 1 more entry be used for stripe.
+ */
+ entries = (g_io_size_bytes - 1) / max_xfer_size + 2;
+ if ((g_queue_depth * entries) > opts.io_queue_size) {
+ printf("controller IO queue size %u less than required\n",
+ opts.io_queue_size);
+ printf("Consider using lower queue depth or small IO size because "
+ "IO requests may be queued at the NVMe driver.\n");
+ }
+ /* For requests which have children requests, parent request itself
+ * will also occupy 1 entry.
+ */
+ entries += 1;
+
+ entry = calloc(1, sizeof(struct ns_entry));
+ if (entry == NULL) {
+ perror("ns_entry malloc");
+ exit(1);
+ }
+
+ entry->type = ENTRY_TYPE_NVME_NS;
+ entry->fn_table = &nvme_fn_table;
+ entry->u.nvme.ctrlr = ctrlr;
+ entry->u.nvme.ns = ns;
+ entry->num_io_requests = g_queue_depth * entries;
+
+ entry->size_in_ios = ns_size / g_io_size_bytes;
+ entry->io_size_blocks = g_io_size_bytes / sector_size;
+
+ entry->block_size = spdk_nvme_ns_get_extended_sector_size(ns);
+ entry->md_size = spdk_nvme_ns_get_md_size(ns);
+ entry->md_interleave = spdk_nvme_ns_supports_extended_lba(ns);
+ entry->pi_loc = spdk_nvme_ns_get_data(ns)->dps.md_start;
+ entry->pi_type = spdk_nvme_ns_get_pi_type(ns);
+
+ if (spdk_nvme_ns_get_flags(ns) & SPDK_NVME_NS_DPS_PI_SUPPORTED) {
+ entry->io_flags = g_metacfg_pract_flag | g_metacfg_prchk_flags;
+ }
+
+ /* If metadata size = 8 bytes, PI is stripped (read) or inserted (write),
+ * and so reduce metadata size from block size. (If metadata size > 8 bytes,
+ * PI is passed (read) or replaced (write). So block size is not necessary
+ * to change.)
+ */
+ if ((entry->io_flags & SPDK_NVME_IO_FLAGS_PRACT) && (entry->md_size == 8)) {
+ entry->block_size = spdk_nvme_ns_get_sector_size(ns);
+ }
+
+ if (g_max_io_md_size < entry->md_size) {
+ g_max_io_md_size = entry->md_size;
+ }
+
+ if (g_max_io_size_blocks < entry->io_size_blocks) {
+ g_max_io_size_blocks = entry->io_size_blocks;
+ }
+
+ build_nvme_ns_name(entry->name, sizeof(entry->name), ctrlr, spdk_nvme_ns_get_id(ns));
+
+ g_num_namespaces++;
+ entry->next = g_namespaces;
+ g_namespaces = entry;
+}
+
+static void
+unregister_namespaces(void)
+{
+ struct ns_entry *entry = g_namespaces;
+
+ while (entry) {
+ struct ns_entry *next = entry->next;
+ free(entry);
+ entry = next;
+ }
+}
+
+static void
+enable_latency_tracking_complete(void *cb_arg, const struct spdk_nvme_cpl *cpl)
+{
+ if (spdk_nvme_cpl_is_error(cpl)) {
+ printf("enable_latency_tracking_complete failed\n");
+ }
+ g_outstanding_commands--;
+}
+
+static void
+set_latency_tracking_feature(struct spdk_nvme_ctrlr *ctrlr, bool enable)
+{
+ int res;
+ union spdk_nvme_intel_feat_latency_tracking latency_tracking;
+
+ if (enable) {
+ latency_tracking.bits.enable = 0x01;
+ } else {
+ latency_tracking.bits.enable = 0x00;
+ }
+
+ res = spdk_nvme_ctrlr_cmd_set_feature(ctrlr, SPDK_NVME_INTEL_FEAT_LATENCY_TRACKING,
+ latency_tracking.raw, 0, NULL, 0, enable_latency_tracking_complete, NULL);
+ if (res) {
+ printf("fail to allocate nvme request.\n");
+ return;
+ }
+ g_outstanding_commands++;
+
+ while (g_outstanding_commands) {
+ spdk_nvme_ctrlr_process_admin_completions(ctrlr);
+ }
+}
+
+static void
+register_ctrlr(struct spdk_nvme_ctrlr *ctrlr, struct trid_entry *trid_entry)
+{
+ struct spdk_nvme_ns *ns;
+ struct ctrlr_entry *entry = malloc(sizeof(struct ctrlr_entry));
+ uint32_t nsid;
+
+ if (entry == NULL) {
+ perror("ctrlr_entry malloc");
+ exit(1);
+ }
+
+ entry->latency_page = spdk_dma_zmalloc(sizeof(struct spdk_nvme_intel_rw_latency_page),
+ 4096, NULL);
+ if (entry->latency_page == NULL) {
+ printf("Allocation error (latency page)\n");
+ exit(1);
+ }
+
+ build_nvme_name(entry->name, sizeof(entry->name), ctrlr);
+
+ entry->ctrlr = ctrlr;
+ entry->trtype = trid_entry->trid.trtype;
+ entry->next = g_controllers;
+ g_controllers = entry;
+
+ if (g_latency_ssd_tracking_enable &&
+ spdk_nvme_ctrlr_is_feature_supported(ctrlr, SPDK_NVME_INTEL_FEAT_LATENCY_TRACKING)) {
+ set_latency_tracking_feature(ctrlr, true);
+ }
+
+ if (trid_entry->nsid == 0) {
+ for (nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr);
+ nsid != 0; nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid)) {
+ ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid);
+ if (ns == NULL) {
+ continue;
+ }
+ register_ns(ctrlr, ns);
+ }
+ } else {
+ ns = spdk_nvme_ctrlr_get_ns(ctrlr, trid_entry->nsid);
+ if (!ns) {
+ perror("Namespace does not exist.");
+ exit(1);
+ }
+
+ register_ns(ctrlr, ns);
+ }
+}
+
+static __thread unsigned int seed = 0;
+
+static inline void
+submit_single_io(struct perf_task *task)
+{
+ uint64_t offset_in_ios;
+ int rc;
+ struct ns_worker_ctx *ns_ctx = task->ns_ctx;
+ struct ns_entry *entry = ns_ctx->entry;
+
+ if (g_is_random) {
+ offset_in_ios = rand_r(&seed) % entry->size_in_ios;
+ } else {
+ offset_in_ios = ns_ctx->offset_in_ios++;
+ if (ns_ctx->offset_in_ios == entry->size_in_ios) {
+ ns_ctx->offset_in_ios = 0;
+ }
+ }
+
+ task->submit_tsc = spdk_get_ticks();
+
+ if ((g_rw_percentage == 100) ||
+ (g_rw_percentage != 0 && ((rand_r(&seed) % 100) < g_rw_percentage))) {
+ task->is_read = true;
+ } else {
+ task->is_read = false;
+ }
+
+ rc = entry->fn_table->submit_io(task, ns_ctx, entry, offset_in_ios);
+
+ if (spdk_unlikely(rc != 0)) {
+ fprintf(stderr, "starting I/O failed\n");
+ } else {
+ ns_ctx->current_queue_depth++;
+ }
+}
+
+static inline void
+task_complete(struct perf_task *task)
+{
+ struct ns_worker_ctx *ns_ctx;
+ uint64_t tsc_diff;
+ struct ns_entry *entry;
+
+ ns_ctx = task->ns_ctx;
+ entry = ns_ctx->entry;
+ ns_ctx->current_queue_depth--;
+ ns_ctx->io_completed++;
+ tsc_diff = spdk_get_ticks() - task->submit_tsc;
+ ns_ctx->total_tsc += tsc_diff;
+ if (spdk_unlikely(ns_ctx->min_tsc > tsc_diff)) {
+ ns_ctx->min_tsc = tsc_diff;
+ }
+ if (spdk_unlikely(ns_ctx->max_tsc < tsc_diff)) {
+ ns_ctx->max_tsc = tsc_diff;
+ }
+ if (spdk_unlikely(g_latency_sw_tracking_level > 0)) {
+ spdk_histogram_data_tally(ns_ctx->histogram, tsc_diff);
+ }
+
+ if (spdk_unlikely(entry->md_size > 0)) {
+ /* add application level verification for end-to-end data protection */
+ entry->fn_table->verify_io(task, entry);
+ }
+
+ /*
+ * is_draining indicates when time has expired for the test run
+ * and we are just waiting for the previously submitted I/O
+ * to complete. In this case, do not submit a new I/O to replace
+ * the one just completed.
+ */
+ if (spdk_unlikely(ns_ctx->is_draining)) {
+ spdk_dma_free(task->iov.iov_base);
+ spdk_dma_free(task->md_iov.iov_base);
+ free(task);
+ } else {
+ submit_single_io(task);
+ }
+}
+
+static void
+io_complete(void *ctx, const struct spdk_nvme_cpl *cpl)
+{
+ struct perf_task *task = ctx;
+
+ if (spdk_unlikely(spdk_nvme_cpl_is_error(cpl))) {
+ fprintf(stderr, "%s completed with error (sct=%d, sc=%d)\n",
+ task->is_read ? "Read" : "Write",
+ cpl->status.sct, cpl->status.sc);
+ }
+
+ task_complete(task);
+}
+
+static struct perf_task *
+allocate_task(struct ns_worker_ctx *ns_ctx, int queue_depth)
+{
+ struct perf_task *task;
+
+ task = calloc(1, sizeof(*task));
+ if (task == NULL) {
+ fprintf(stderr, "Out of memory allocating tasks\n");
+ exit(1);
+ }
+
+ ns_ctx->entry->fn_table->setup_payload(task, queue_depth % 8 + 1);
+
+ task->ns_ctx = ns_ctx;
+
+ return task;
+}
+
+static void
+submit_io(struct ns_worker_ctx *ns_ctx, int queue_depth)
+{
+ struct perf_task *task;
+
+ while (queue_depth-- > 0) {
+ task = allocate_task(ns_ctx, queue_depth);
+ submit_single_io(task);
+ }
+}
+
+static int
+init_ns_worker_ctx(struct ns_worker_ctx *ns_ctx)
+{
+ return ns_ctx->entry->fn_table->init_ns_worker_ctx(ns_ctx);
+}
+
+static void
+cleanup_ns_worker_ctx(struct ns_worker_ctx *ns_ctx)
+{
+ ns_ctx->entry->fn_table->cleanup_ns_worker_ctx(ns_ctx);
+}
+
+static void
+print_periodic_performance(void)
+{
+ uint64_t io_this_second;
+ double mb_this_second;
+ struct worker_thread *worker;
+ struct ns_worker_ctx *ns_ctx;
+
+ if (!isatty(STDOUT_FILENO)) {
+ /* Don't print periodic stats if output is not going
+ * to a terminal.
+ */
+ return;
+ }
+
+ io_this_second = 0;
+ worker = g_workers;
+ while (worker) {
+ ns_ctx = worker->ns_ctx;
+ while (ns_ctx) {
+ io_this_second += ns_ctx->io_completed - ns_ctx->last_io_completed;
+ ns_ctx->last_io_completed = ns_ctx->io_completed;
+ ns_ctx = ns_ctx->next;
+ }
+ worker = worker->next;
+ }
+
+ mb_this_second = (double)io_this_second * g_io_size_bytes / (1024 * 1024);
+ printf("%9ju IOPS, %8.2f MiB/s\r", io_this_second, mb_this_second);
+ fflush(stdout);
+}
+
+static int
+work_fn(void *arg)
+{
+ uint64_t tsc_end, tsc_current, tsc_next_print;
+ struct worker_thread *worker = (struct worker_thread *)arg;
+ struct ns_worker_ctx *ns_ctx = NULL;
+ uint32_t unfinished_ns_ctx;
+
+ /* Allocate queue pairs for each namespace. */
+ ns_ctx = worker->ns_ctx;
+ while (ns_ctx != NULL) {
+ if (init_ns_worker_ctx(ns_ctx) != 0) {
+ printf("ERROR: init_ns_worker_ctx() failed\n");
+ return 1;
+ }
+ ns_ctx = ns_ctx->next;
+ }
+
+ tsc_current = spdk_get_ticks();
+ tsc_end = tsc_current + g_time_in_sec * g_tsc_rate;
+ tsc_next_print = tsc_current + g_tsc_rate;
+
+ /* Submit initial I/O for each namespace. */
+ ns_ctx = worker->ns_ctx;
+ while (ns_ctx != NULL) {
+ submit_io(ns_ctx, g_queue_depth);
+ ns_ctx = ns_ctx->next;
+ }
+
+ while (1) {
+ /*
+ * Check for completed I/O for each controller. A new
+ * I/O will be submitted in the io_complete callback
+ * to replace each I/O that is completed.
+ */
+ ns_ctx = worker->ns_ctx;
+ while (ns_ctx != NULL) {
+ ns_ctx->entry->fn_table->check_io(ns_ctx);
+ ns_ctx = ns_ctx->next;
+ }
+
+ tsc_current = spdk_get_ticks();
+
+ if (worker->lcore == g_master_core && tsc_current > tsc_next_print) {
+ tsc_next_print += g_tsc_rate;
+ print_periodic_performance();
+ }
+
+ if (tsc_current > tsc_end) {
+ break;
+ }
+ }
+
+ /* drain the io of each ns_ctx in round robin to make the fairness */
+ do {
+ unfinished_ns_ctx = 0;
+ ns_ctx = worker->ns_ctx;
+ while (ns_ctx != NULL) {
+ /* first time will enter into this if case */
+ if (!ns_ctx->is_draining) {
+ ns_ctx->is_draining = true;
+ }
+
+ if (ns_ctx->current_queue_depth > 0) {
+ ns_ctx->entry->fn_table->check_io(ns_ctx);
+ if (ns_ctx->current_queue_depth == 0) {
+ cleanup_ns_worker_ctx(ns_ctx);
+ } else {
+ unfinished_ns_ctx++;
+ }
+ }
+ ns_ctx = ns_ctx->next;
+ }
+ } while (unfinished_ns_ctx > 0);
+
+ return 0;
+}
+
+static void usage(char *program_name)
+{
+ printf("%s options", program_name);
+#if defined(SPDK_CONFIG_URING) || defined(HAVE_LIBAIO)
+ printf(" [Kernel device(s)]...");
+#endif
+ printf("\n");
+ printf("\t[-q io depth]\n");
+ printf("\t[-o io size in bytes]\n");
+ printf("\t[-P number of io queues per namespace. default: 1]\n");
+ printf("\t[-U number of unused io queues per controller. default: 0]\n");
+ printf("\t[-w io pattern type, must be one of\n");
+ printf("\t\t(read, write, randread, randwrite, rw, randrw)]\n");
+ printf("\t[-M rwmixread (100 for reads, 0 for writes)]\n");
+ printf("\t[-L enable latency tracking via sw, default: disabled]\n");
+ printf("\t\t-L for latency summary, -LL for detailed histogram\n");
+ printf("\t[-l enable latency tracking via ssd (if supported), default: disabled]\n");
+ printf("\t[-t time in seconds]\n");
+ printf("\t[-c core mask for I/O submission/completion.]\n");
+ printf("\t\t(default: 1)\n");
+ printf("\t[-D disable submission queue in controller memory buffer, default: enabled]\n");
+ printf("\t[-H enable header digest for TCP transport, default: disabled]\n");
+ printf("\t[-I enable data digest for TCP transport, default: disabled]\n");
+ printf("\t[-N no shutdown notification process for controllers, default: disabled]\n");
+ printf("\t[-r Transport ID for local PCIe NVMe or NVMeoF]\n");
+ printf("\t Format: 'key:value [key:value] ...'\n");
+ printf("\t Keys:\n");
+ printf("\t trtype Transport type (e.g. PCIe, RDMA)\n");
+ printf("\t adrfam Address family (e.g. IPv4, IPv6)\n");
+ printf("\t traddr Transport address (e.g. 0000:04:00.0 for PCIe or 192.168.100.8 for RDMA)\n");
+ printf("\t trsvcid Transport service identifier (e.g. 4420)\n");
+ printf("\t subnqn Subsystem NQN (default: %s)\n", SPDK_NVMF_DISCOVERY_NQN);
+ printf("\t Example: -r 'trtype:PCIe traddr:0000:04:00.0' for PCIe or\n");
+ printf("\t -r 'trtype:RDMA adrfam:IPv4 traddr:192.168.100.8 trsvcid:4420' for NVMeoF\n");
+ printf("\t[-e metadata configuration]\n");
+ printf("\t Keys:\n");
+ printf("\t PRACT Protection Information Action bit (PRACT=1 or PRACT=0)\n");
+ printf("\t PRCHK Control of Protection Information Checking (PRCHK=GUARD|REFTAG|APPTAG)\n");
+ printf("\t Example: -e 'PRACT=0,PRCHK=GUARD|REFTAG|APPTAG'\n");
+ printf("\t -e 'PRACT=1,PRCHK=GUARD'\n");
+ printf("\t[-k keep alive timeout period in millisecond]\n");
+ printf("\t[-s DPDK huge memory size in MB.]\n");
+ printf("\t[-C max completions per poll]\n");
+ printf("\t\t(default: 0 - unlimited)\n");
+ printf("\t[-i shared memory group ID]\n");
+ printf("\t");
+ spdk_log_usage(stdout, "-T");
+#ifdef SPDK_CONFIG_URING
+ printf("\t[-R enable using liburing to drive kernel devices (Default: libaio)]\n");
+#endif
+#ifdef DEBUG
+ printf("\t[-G enable debug logging]\n");
+#else
+ printf("\t[-G enable debug logging (flag disabled, must reconfigure with --enable-debug)\n");
+#endif
+}
+
+static void
+check_cutoff(void *ctx, uint64_t start, uint64_t end, uint64_t count,
+ uint64_t total, uint64_t so_far)
+{
+ double so_far_pct;
+ double **cutoff = ctx;
+
+ if (count == 0) {
+ return;
+ }
+
+ so_far_pct = (double)so_far / total;
+ while (so_far_pct >= **cutoff && **cutoff > 0) {
+ printf("%9.5f%% : %9.3fus\n", **cutoff * 100, (double)end * 1000 * 1000 / g_tsc_rate);
+ (*cutoff)++;
+ }
+}
+
+static void
+print_bucket(void *ctx, uint64_t start, uint64_t end, uint64_t count,
+ uint64_t total, uint64_t so_far)
+{
+ double so_far_pct;
+
+ if (count == 0) {
+ return;
+ }
+
+ so_far_pct = (double)so_far * 100 / total;
+ printf("%9.3f - %9.3f: %9.4f%% (%9ju)\n",
+ (double)start * 1000 * 1000 / g_tsc_rate,
+ (double)end * 1000 * 1000 / g_tsc_rate,
+ so_far_pct, count);
+}
+
+static void
+print_performance(void)
+{
+ uint64_t total_io_completed, total_io_tsc;
+ double io_per_second, mb_per_second, average_latency, min_latency, max_latency;
+ double sum_ave_latency, min_latency_so_far, max_latency_so_far;
+ double total_io_per_second, total_mb_per_second;
+ int ns_count;
+ struct worker_thread *worker;
+ struct ns_worker_ctx *ns_ctx;
+ uint32_t max_strlen;
+
+ total_io_per_second = 0;
+ total_mb_per_second = 0;
+ total_io_completed = 0;
+ total_io_tsc = 0;
+ min_latency_so_far = (double)UINT64_MAX;
+ max_latency_so_far = 0;
+ ns_count = 0;
+
+ max_strlen = 0;
+ worker = g_workers;
+ while (worker) {
+ ns_ctx = worker->ns_ctx;
+ while (ns_ctx) {
+ max_strlen = spdk_max(strlen(ns_ctx->entry->name), max_strlen);
+ ns_ctx = ns_ctx->next;
+ }
+ worker = worker->next;
+ }
+
+ printf("========================================================\n");
+ printf("%*s\n", max_strlen + 60, "Latency(us)");
+ printf("%-*s: %10s %10s %10s %10s %10s\n",
+ max_strlen + 13, "Device Information", "IOPS", "MiB/s", "Average", "min", "max");
+
+ worker = g_workers;
+ while (worker) {
+ ns_ctx = worker->ns_ctx;
+ while (ns_ctx) {
+ if (ns_ctx->io_completed != 0) {
+ io_per_second = (double)ns_ctx->io_completed / g_time_in_sec;
+ mb_per_second = io_per_second * g_io_size_bytes / (1024 * 1024);
+ average_latency = ((double)ns_ctx->total_tsc / ns_ctx->io_completed) * 1000 * 1000 / g_tsc_rate;
+ min_latency = (double)ns_ctx->min_tsc * 1000 * 1000 / g_tsc_rate;
+ if (min_latency < min_latency_so_far) {
+ min_latency_so_far = min_latency;
+ }
+
+ max_latency = (double)ns_ctx->max_tsc * 1000 * 1000 / g_tsc_rate;
+ if (max_latency > max_latency_so_far) {
+ max_latency_so_far = max_latency;
+ }
+
+ printf("%-*.*s from core %2u: %10.2f %10.2f %10.2f %10.2f %10.2f\n",
+ max_strlen, max_strlen, ns_ctx->entry->name, worker->lcore,
+ io_per_second, mb_per_second,
+ average_latency, min_latency, max_latency);
+ total_io_per_second += io_per_second;
+ total_mb_per_second += mb_per_second;
+ total_io_completed += ns_ctx->io_completed;
+ total_io_tsc += ns_ctx->total_tsc;
+ ns_count++;
+ }
+ ns_ctx = ns_ctx->next;
+ }
+ worker = worker->next;
+ }
+
+ if (ns_count != 0 && total_io_completed) {
+ sum_ave_latency = ((double)total_io_tsc / total_io_completed) * 1000 * 1000 / g_tsc_rate;
+ printf("========================================================\n");
+ printf("%-*s: %10.2f %10.2f %10.2f %10.2f %10.2f\n",
+ max_strlen + 13, "Total", total_io_per_second, total_mb_per_second,
+ sum_ave_latency, min_latency_so_far, max_latency_so_far);
+ printf("\n");
+ }
+
+ if (g_latency_sw_tracking_level == 0 || total_io_completed == 0) {
+ return;
+ }
+
+ worker = g_workers;
+ while (worker) {
+ ns_ctx = worker->ns_ctx;
+ while (ns_ctx) {
+ const double *cutoff = g_latency_cutoffs;
+
+ printf("Summary latency data for %-43.43s from core %u:\n", ns_ctx->entry->name, worker->lcore);
+ printf("=================================================================================\n");
+
+ spdk_histogram_data_iterate(ns_ctx->histogram, check_cutoff, &cutoff);
+
+ printf("\n");
+ ns_ctx = ns_ctx->next;
+ }
+ worker = worker->next;
+ }
+
+ if (g_latency_sw_tracking_level == 1) {
+ return;
+ }
+
+ worker = g_workers;
+ while (worker) {
+ ns_ctx = worker->ns_ctx;
+ while (ns_ctx) {
+ printf("Latency histogram for %-43.43s from core %u:\n", ns_ctx->entry->name, worker->lcore);
+ printf("==============================================================================\n");
+ printf(" Range in us Cumulative IO count\n");
+
+ spdk_histogram_data_iterate(ns_ctx->histogram, print_bucket, NULL);
+ printf("\n");
+ ns_ctx = ns_ctx->next;
+ }
+ worker = worker->next;
+ }
+
+}
+
+static void
+print_latency_page(struct ctrlr_entry *entry)
+{
+ int i;
+
+ printf("\n");
+ printf("%s\n", entry->name);
+ printf("--------------------------------------------------------\n");
+
+ for (i = 0; i < 32; i++) {
+ if (entry->latency_page->buckets_32us[i]) {
+ printf("Bucket %dus - %dus: %d\n", i * 32, (i + 1) * 32, entry->latency_page->buckets_32us[i]);
+ }
+ }
+ for (i = 0; i < 31; i++) {
+ if (entry->latency_page->buckets_1ms[i]) {
+ printf("Bucket %dms - %dms: %d\n", i + 1, i + 2, entry->latency_page->buckets_1ms[i]);
+ }
+ }
+ for (i = 0; i < 31; i++) {
+ if (entry->latency_page->buckets_32ms[i])
+ printf("Bucket %dms - %dms: %d\n", (i + 1) * 32, (i + 2) * 32,
+ entry->latency_page->buckets_32ms[i]);
+ }
+}
+
+static void
+print_latency_statistics(const char *op_name, enum spdk_nvme_intel_log_page log_page)
+{
+ struct ctrlr_entry *ctrlr;
+
+ printf("%s Latency Statistics:\n", op_name);
+ printf("========================================================\n");
+ ctrlr = g_controllers;
+ while (ctrlr) {
+ if (spdk_nvme_ctrlr_is_log_page_supported(ctrlr->ctrlr, log_page)) {
+ if (spdk_nvme_ctrlr_cmd_get_log_page(ctrlr->ctrlr, log_page, SPDK_NVME_GLOBAL_NS_TAG,
+ ctrlr->latency_page, sizeof(struct spdk_nvme_intel_rw_latency_page), 0,
+ enable_latency_tracking_complete,
+ NULL)) {
+ printf("nvme_ctrlr_cmd_get_log_page() failed\n");
+ exit(1);
+ }
+
+ g_outstanding_commands++;
+ } else {
+ printf("Controller %s: %s latency statistics not supported\n", ctrlr->name, op_name);
+ }
+ ctrlr = ctrlr->next;
+ }
+
+ while (g_outstanding_commands) {
+ ctrlr = g_controllers;
+ while (ctrlr) {
+ spdk_nvme_ctrlr_process_admin_completions(ctrlr->ctrlr);
+ ctrlr = ctrlr->next;
+ }
+ }
+
+ ctrlr = g_controllers;
+ while (ctrlr) {
+ if (spdk_nvme_ctrlr_is_log_page_supported(ctrlr->ctrlr, log_page)) {
+ print_latency_page(ctrlr);
+ }
+ ctrlr = ctrlr->next;
+ }
+ printf("\n");
+}
+
+static void
+print_stats(void)
+{
+ print_performance();
+ if (g_latency_ssd_tracking_enable) {
+ if (g_rw_percentage != 0) {
+ print_latency_statistics("Read", SPDK_NVME_INTEL_LOG_READ_CMD_LATENCY);
+ }
+ if (g_rw_percentage != 100) {
+ print_latency_statistics("Write", SPDK_NVME_INTEL_LOG_WRITE_CMD_LATENCY);
+ }
+ }
+}
+
+static void
+unregister_trids(void)
+{
+ struct trid_entry *trid_entry, *tmp;
+
+ TAILQ_FOREACH_SAFE(trid_entry, &g_trid_list, tailq, tmp) {
+ TAILQ_REMOVE(&g_trid_list, trid_entry, tailq);
+ free(trid_entry);
+ }
+}
+
+static int
+add_trid(const char *trid_str)
+{
+ struct trid_entry *trid_entry;
+ struct spdk_nvme_transport_id *trid;
+ char *ns;
+
+ trid_entry = calloc(1, sizeof(*trid_entry));
+ if (trid_entry == NULL) {
+ return -1;
+ }
+
+ trid = &trid_entry->trid;
+ trid->trtype = SPDK_NVME_TRANSPORT_PCIE;
+ snprintf(trid->subnqn, sizeof(trid->subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN);
+
+ if (spdk_nvme_transport_id_parse(trid, trid_str) != 0) {
+ fprintf(stderr, "Invalid transport ID format '%s'\n", trid_str);
+ free(trid_entry);
+ return 1;
+ }
+
+ spdk_nvme_transport_id_populate_trstring(trid,
+ spdk_nvme_transport_id_trtype_str(trid->trtype));
+
+ ns = strcasestr(trid_str, "ns:");
+ if (ns) {
+ char nsid_str[6]; /* 5 digits maximum in an nsid */
+ int len;
+ int nsid;
+
+ ns += 3;
+
+ len = strcspn(ns, " \t\n");
+ if (len > 5) {
+ fprintf(stderr, "NVMe namespace IDs must be 5 digits or less\n");
+ free(trid_entry);
+ return 1;
+ }
+
+ memcpy(nsid_str, ns, len);
+ nsid_str[len] = '\0';
+
+ nsid = spdk_strtol(nsid_str, 10);
+ if (nsid <= 0 || nsid > 65535) {
+ fprintf(stderr, "NVMe namespace IDs must be less than 65536 and greater than 0\n");
+ free(trid_entry);
+ return 1;
+ }
+
+ trid_entry->nsid = (uint16_t)nsid;
+ }
+
+ TAILQ_INSERT_TAIL(&g_trid_list, trid_entry, tailq);
+ return 0;
+}
+
+static size_t
+parse_next_key(const char **str, char *key, char *val, size_t key_buf_size,
+ size_t val_buf_size)
+{
+ const char *sep;
+ const char *separator = ", \t\n";
+ size_t key_len, val_len;
+
+ *str += strspn(*str, separator);
+
+ sep = strchr(*str, '=');
+ if (!sep) {
+ fprintf(stderr, "Key without '=' separator\n");
+ return 0;
+ }
+
+ key_len = sep - *str;
+ if (key_len >= key_buf_size) {
+ fprintf(stderr, "Key length %zu is greater than maximum allowed %zu\n",
+ key_len, key_buf_size - 1);
+ return 0;
+ }
+
+ memcpy(key, *str, key_len);
+ key[key_len] = '\0';
+
+ *str += key_len + 1; /* Skip key */
+ val_len = strcspn(*str, separator);
+ if (val_len == 0) {
+ fprintf(stderr, "Key without value\n");
+ return 0;
+ }
+
+ if (val_len >= val_buf_size) {
+ fprintf(stderr, "Value length %zu is greater than maximum allowed %zu\n",
+ val_len, val_buf_size - 1);
+ return 0;
+ }
+
+ memcpy(val, *str, val_len);
+ val[val_len] = '\0';
+
+ *str += val_len;
+
+ return val_len;
+}
+
+static int
+parse_metadata(const char *metacfg_str)
+{
+ const char *str;
+ size_t val_len;
+ char key[32];
+ char val[1024];
+
+ if (metacfg_str == NULL) {
+ return -EINVAL;
+ }
+
+ str = metacfg_str;
+
+ while (*str != '\0') {
+ val_len = parse_next_key(&str, key, val, sizeof(key), sizeof(val));
+ if (val_len == 0) {
+ fprintf(stderr, "Failed to parse metadata\n");
+ return -EINVAL;
+ }
+
+ if (strcmp(key, "PRACT") == 0) {
+ if (*val == '1') {
+ g_metacfg_pract_flag = SPDK_NVME_IO_FLAGS_PRACT;
+ }
+ } else if (strcmp(key, "PRCHK") == 0) {
+ if (strstr(val, "GUARD") != NULL) {
+ g_metacfg_prchk_flags |= SPDK_NVME_IO_FLAGS_PRCHK_GUARD;
+ }
+ if (strstr(val, "REFTAG") != NULL) {
+ g_metacfg_prchk_flags |= SPDK_NVME_IO_FLAGS_PRCHK_REFTAG;
+ }
+ if (strstr(val, "APPTAG") != NULL) {
+ g_metacfg_prchk_flags |= SPDK_NVME_IO_FLAGS_PRCHK_APPTAG;
+ }
+ } else {
+ fprintf(stderr, "Unknown key '%s'\n", key);
+ }
+ }
+
+ return 0;
+}
+
+static int
+parse_args(int argc, char **argv)
+{
+ int op;
+ long int val;
+ int rc;
+
+ while ((op = getopt(argc, argv, "c:e:i:lo:q:r:k:s:t:w:C:DGHILM:NP:RT:U:V")) != -1) {
+ switch (op) {
+ case 'i':
+ case 'C':
+ case 'P':
+ case 'o':
+ case 'q':
+ case 'k':
+ case 's':
+ case 't':
+ case 'M':
+ case 'U':
+ val = spdk_strtol(optarg, 10);
+ if (val < 0) {
+ fprintf(stderr, "Converting a string to integer failed\n");
+ return val;
+ }
+ switch (op) {
+ case 'i':
+ g_shm_id = val;
+ break;
+ case 'C':
+ g_max_completions = val;
+ break;
+ case 'P':
+ g_nr_io_queues_per_ns = val;
+ break;
+ case 'o':
+ g_io_size_bytes = val;
+ break;
+ case 'q':
+ g_queue_depth = val;
+ break;
+ case 'k':
+ g_keep_alive_timeout_in_ms = val;
+ break;
+ case 's':
+ g_dpdk_mem = val;
+ break;
+ case 't':
+ g_time_in_sec = val;
+ break;
+ case 'M':
+ g_rw_percentage = val;
+ g_mix_specified = true;
+ break;
+ case 'U':
+ g_nr_unused_io_queues = val;
+ break;
+ }
+ break;
+ case 'c':
+ g_core_mask = optarg;
+ break;
+ case 'e':
+ if (parse_metadata(optarg)) {
+ usage(argv[0]);
+ return 1;
+ }
+ break;
+ case 'l':
+ g_latency_ssd_tracking_enable = true;
+ break;
+ case 'r':
+ if (add_trid(optarg)) {
+ usage(argv[0]);
+ return 1;
+ }
+ break;
+ case 'w':
+ g_workload_type = optarg;
+ break;
+ case 'D':
+ g_disable_sq_cmb = 1;
+ break;
+ case 'G':
+#ifndef DEBUG
+ fprintf(stderr, "%s must be configured with --enable-debug for -G flag\n",
+ argv[0]);
+ usage(argv[0]);
+ return 1;
+#else
+ spdk_log_set_flag("nvme");
+ spdk_log_set_print_level(SPDK_LOG_DEBUG);
+ break;
+#endif
+ case 'H':
+ g_header_digest = 1;
+ break;
+ case 'I':
+ g_data_digest = 1;
+ break;
+ case 'L':
+ g_latency_sw_tracking_level++;
+ break;
+ case 'N':
+ g_no_shn_notification = true;
+ break;
+ case 'R':
+#ifndef SPDK_CONFIG_URING
+ fprintf(stderr, "%s must be rebuilt with CONFIG_URING=y for -R flag.\n",
+ argv[0]);
+ usage(argv[0]);
+ return 0;
+#endif
+ g_use_uring = true;
+ break;
+ case 'T':
+ rc = spdk_log_set_flag(optarg);
+ if (rc < 0) {
+ fprintf(stderr, "unknown flag\n");
+ usage(argv[0]);
+ exit(EXIT_FAILURE);
+ }
+ spdk_log_set_print_level(SPDK_LOG_DEBUG);
+#ifndef DEBUG
+ fprintf(stderr, "%s must be rebuilt with CONFIG_DEBUG=y for -T flag.\n",
+ argv[0]);
+ usage(argv[0]);
+ return 0;
+#endif
+ break;
+ case 'V':
+ g_vmd = true;
+ break;
+ default:
+ usage(argv[0]);
+ return 1;
+ }
+ }
+
+ if (!g_nr_io_queues_per_ns) {
+ usage(argv[0]);
+ return 1;
+ }
+
+ if (!g_queue_depth) {
+ fprintf(stderr, "missing -q (queue size) operand\n");
+ usage(argv[0]);
+ return 1;
+ }
+ if (!g_io_size_bytes) {
+ fprintf(stderr, "missing -o (block size) operand\n");
+ usage(argv[0]);
+ return 1;
+ }
+ if (!g_workload_type) {
+ fprintf(stderr, "missing -w (io pattern type) operand\n");
+ usage(argv[0]);
+ return 1;
+ }
+ if (!g_time_in_sec) {
+ fprintf(stderr, "missing -t (test time in seconds) operand\n");
+ usage(argv[0]);
+ return 1;
+ }
+
+ if (strncmp(g_workload_type, "rand", 4) == 0) {
+ g_is_random = 1;
+ g_workload_type = &g_workload_type[4];
+ }
+
+ if (strcmp(g_workload_type, "read") == 0 || strcmp(g_workload_type, "write") == 0) {
+ g_rw_percentage = strcmp(g_workload_type, "read") == 0 ? 100 : 0;
+ if (g_mix_specified) {
+ fprintf(stderr, "Ignoring -M option... Please use -M option"
+ " only when using rw or randrw.\n");
+ }
+ } else if (strcmp(g_workload_type, "rw") == 0) {
+ if (g_rw_percentage < 0 || g_rw_percentage > 100) {
+ fprintf(stderr,
+ "-M must be specified to value from 0 to 100 "
+ "for rw or randrw.\n");
+ return 1;
+ }
+ } else {
+ fprintf(stderr,
+ "io pattern type must be one of\n"
+ "(read, write, randread, randwrite, rw, randrw)\n");
+ return 1;
+ }
+
+ if (TAILQ_EMPTY(&g_trid_list)) {
+ /* If no transport IDs specified, default to enumerating all local PCIe devices */
+ add_trid("trtype:PCIe");
+ } else {
+ struct trid_entry *trid_entry, *trid_entry_tmp;
+
+ g_no_pci = true;
+ /* check whether there is local PCIe type */
+ TAILQ_FOREACH_SAFE(trid_entry, &g_trid_list, tailq, trid_entry_tmp) {
+ if (trid_entry->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) {
+ g_no_pci = false;
+ break;
+ }
+ }
+ }
+
+ g_file_optind = optind;
+
+ return 0;
+}
+
+static int
+register_workers(void)
+{
+ uint32_t i;
+ struct worker_thread *worker;
+
+ g_workers = NULL;
+ g_num_workers = 0;
+
+ SPDK_ENV_FOREACH_CORE(i) {
+ worker = calloc(1, sizeof(*worker));
+ if (worker == NULL) {
+ fprintf(stderr, "Unable to allocate worker\n");
+ return -1;
+ }
+
+ worker->lcore = i;
+ worker->next = g_workers;
+ g_workers = worker;
+ g_num_workers++;
+ }
+
+ return 0;
+}
+
+static void
+unregister_workers(void)
+{
+ struct worker_thread *worker = g_workers;
+
+ /* Free namespace context and worker thread */
+ while (worker) {
+ struct worker_thread *next_worker = worker->next;
+ struct ns_worker_ctx *ns_ctx = worker->ns_ctx;
+
+ while (ns_ctx) {
+ struct ns_worker_ctx *next_ns_ctx = ns_ctx->next;
+ spdk_histogram_data_free(ns_ctx->histogram);
+ free(ns_ctx);
+ ns_ctx = next_ns_ctx;
+ }
+
+ free(worker);
+ worker = next_worker;
+ }
+}
+
+static bool
+probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
+ struct spdk_nvme_ctrlr_opts *opts)
+{
+ if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) {
+ if (g_disable_sq_cmb) {
+ opts->use_cmb_sqs = false;
+ }
+ if (g_no_shn_notification) {
+ opts->no_shn_notification = true;
+ }
+ }
+
+ /* Set io_queue_size to UINT16_MAX, NVMe driver
+ * will then reduce this to MQES to maximize
+ * the io_queue_size as much as possible.
+ */
+ opts->io_queue_size = UINT16_MAX;
+
+ /* Set the header and data_digest */
+ opts->header_digest = g_header_digest;
+ opts->data_digest = g_data_digest;
+ opts->keep_alive_timeout_ms = g_keep_alive_timeout_in_ms;
+
+ return true;
+}
+
+static void
+attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
+ struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts)
+{
+ struct trid_entry *trid_entry = cb_ctx;
+ struct spdk_pci_addr pci_addr;
+ struct spdk_pci_device *pci_dev;
+ struct spdk_pci_id pci_id;
+
+ if (trid->trtype != SPDK_NVME_TRANSPORT_PCIE) {
+ printf("Attached to NVMe over Fabrics controller at %s:%s: %s\n",
+ trid->traddr, trid->trsvcid,
+ trid->subnqn);
+ } else {
+ if (spdk_pci_addr_parse(&pci_addr, trid->traddr)) {
+ return;
+ }
+
+ pci_dev = spdk_nvme_ctrlr_get_pci_device(ctrlr);
+ if (!pci_dev) {
+ return;
+ }
+
+ pci_id = spdk_pci_device_get_id(pci_dev);
+
+ printf("Attached to NVMe Controller at %s [%04x:%04x]\n",
+ trid->traddr,
+ pci_id.vendor_id, pci_id.device_id);
+ }
+
+ register_ctrlr(ctrlr, trid_entry);
+}
+
+static int
+register_controllers(void)
+{
+ struct trid_entry *trid_entry;
+
+ printf("Initializing NVMe Controllers\n");
+
+ if (g_vmd && spdk_vmd_init()) {
+ fprintf(stderr, "Failed to initialize VMD."
+ " Some NVMe devices can be unavailable.\n");
+ }
+
+ TAILQ_FOREACH(trid_entry, &g_trid_list, tailq) {
+ if (spdk_nvme_probe(&trid_entry->trid, trid_entry, probe_cb, attach_cb, NULL) != 0) {
+ fprintf(stderr, "spdk_nvme_probe() failed for transport address '%s'\n",
+ trid_entry->trid.traddr);
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
+static void
+unregister_controllers(void)
+{
+ struct ctrlr_entry *entry = g_controllers;
+
+ while (entry) {
+ struct ctrlr_entry *next = entry->next;
+ spdk_dma_free(entry->latency_page);
+ if (g_latency_ssd_tracking_enable &&
+ spdk_nvme_ctrlr_is_feature_supported(entry->ctrlr, SPDK_NVME_INTEL_FEAT_LATENCY_TRACKING)) {
+ set_latency_tracking_feature(entry->ctrlr, false);
+ }
+
+ if (g_nr_unused_io_queues) {
+ int i;
+
+ for (i = 0; i < g_nr_unused_io_queues; i++) {
+ spdk_nvme_ctrlr_free_io_qpair(entry->unused_qpairs[i]);
+ }
+
+ free(entry->unused_qpairs);
+ }
+
+ spdk_nvme_detach(entry->ctrlr);
+ free(entry);
+ entry = next;
+ }
+
+ if (g_vmd) {
+ spdk_vmd_fini();
+ }
+}
+
+static int
+associate_workers_with_ns(void)
+{
+ struct ns_entry *entry = g_namespaces;
+ struct worker_thread *worker = g_workers;
+ struct ns_worker_ctx *ns_ctx;
+ int i, count;
+
+ count = g_num_namespaces > g_num_workers ? g_num_namespaces : g_num_workers;
+
+ for (i = 0; i < count; i++) {
+ if (entry == NULL) {
+ break;
+ }
+
+ ns_ctx = calloc(1, sizeof(struct ns_worker_ctx));
+ if (!ns_ctx) {
+ return -1;
+ }
+
+ printf("Associating %s with lcore %d\n", entry->name, worker->lcore);
+ ns_ctx->min_tsc = UINT64_MAX;
+ ns_ctx->entry = entry;
+ ns_ctx->next = worker->ns_ctx;
+ ns_ctx->histogram = spdk_histogram_data_alloc();
+ worker->ns_ctx = ns_ctx;
+
+ worker = worker->next;
+ if (worker == NULL) {
+ worker = g_workers;
+ }
+
+ entry = entry->next;
+ if (entry == NULL) {
+ entry = g_namespaces;
+ }
+
+ }
+
+ return 0;
+}
+
+static void *
+nvme_poll_ctrlrs(void *arg)
+{
+ struct ctrlr_entry *entry;
+ int oldstate;
+
+ spdk_unaffinitize_thread();
+
+ while (true) {
+ pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate);
+
+ entry = g_controllers;
+ while (entry) {
+ if (entry->trtype != SPDK_NVME_TRANSPORT_PCIE) {
+ spdk_nvme_ctrlr_process_admin_completions(entry->ctrlr);
+ }
+ entry = entry->next;
+ }
+
+ pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &oldstate);
+
+ /* This is a pthread cancellation point and cannot be removed. */
+ sleep(1);
+ }
+
+ return NULL;
+}
+
+int main(int argc, char **argv)
+{
+ int rc;
+ struct worker_thread *worker, *master_worker;
+ struct spdk_env_opts opts;
+ pthread_t thread_id = 0;
+
+ rc = parse_args(argc, argv);
+ if (rc != 0) {
+ return rc;
+ }
+
+ spdk_env_opts_init(&opts);
+ opts.name = "perf";
+ opts.shm_id = g_shm_id;
+ if (g_core_mask) {
+ opts.core_mask = g_core_mask;
+ }
+
+ if (g_dpdk_mem) {
+ opts.mem_size = g_dpdk_mem;
+ }
+ if (g_no_pci) {
+ opts.no_pci = g_no_pci;
+ }
+ if (spdk_env_init(&opts) < 0) {
+ fprintf(stderr, "Unable to initialize SPDK env\n");
+ rc = -1;
+ goto cleanup;
+ }
+
+ g_tsc_rate = spdk_get_ticks_hz();
+
+ if (register_workers() != 0) {
+ rc = -1;
+ goto cleanup;
+ }
+
+#if defined(HAVE_LIBAIO) || defined(SPDK_CONFIG_URING)
+ if (register_files(argc, argv) != 0) {
+ rc = -1;
+ goto cleanup;
+ }
+#endif
+
+ if (register_controllers() != 0) {
+ rc = -1;
+ goto cleanup;
+ }
+
+ if (g_warn) {
+ printf("WARNING: Some requested NVMe devices were skipped\n");
+ }
+
+ if (g_num_namespaces == 0) {
+ fprintf(stderr, "No valid NVMe controllers or AIO or URING devices found\n");
+ goto cleanup;
+ }
+
+ rc = pthread_create(&thread_id, NULL, &nvme_poll_ctrlrs, NULL);
+ if (rc != 0) {
+ fprintf(stderr, "Unable to spawn a thread to poll admin queues.\n");
+ goto cleanup;
+ }
+
+ if (associate_workers_with_ns() != 0) {
+ rc = -1;
+ goto cleanup;
+ }
+
+ printf("Initialization complete. Launching workers.\n");
+
+ /* Launch all of the slave workers */
+ g_master_core = spdk_env_get_current_core();
+ master_worker = NULL;
+ worker = g_workers;
+ while (worker != NULL) {
+ if (worker->lcore != g_master_core) {
+ spdk_env_thread_launch_pinned(worker->lcore, work_fn, worker);
+ } else {
+ assert(master_worker == NULL);
+ master_worker = worker;
+ }
+ worker = worker->next;
+ }
+
+ assert(master_worker != NULL);
+ rc = work_fn(master_worker);
+
+ spdk_env_thread_wait_all();
+
+ print_stats();
+
+cleanup:
+ if (thread_id && pthread_cancel(thread_id) == 0) {
+ pthread_join(thread_id, NULL);
+ }
+ unregister_trids();
+ unregister_namespaces();
+ unregister_controllers();
+ unregister_workers();
+
+ if (rc != 0) {
+ fprintf(stderr, "%s: errors occured\n", argv[0]);
+ }
+
+ return rc;
+}
diff --git a/src/spdk/examples/nvme/reconnect/.gitignore b/src/spdk/examples/nvme/reconnect/.gitignore
new file mode 100644
index 000000000..efe3eada4
--- /dev/null
+++ b/src/spdk/examples/nvme/reconnect/.gitignore
@@ -0,0 +1 @@
+reconnect
diff --git a/src/spdk/examples/nvme/reconnect/Makefile b/src/spdk/examples/nvme/reconnect/Makefile
new file mode 100644
index 000000000..880ae76c0
--- /dev/null
+++ b/src/spdk/examples/nvme/reconnect/Makefile
@@ -0,0 +1,38 @@
+#
+# BSD LICENSE
+#
+# Copyright (c) Intel Corporation.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Intel Corporation nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
+
+APP = reconnect
+
+include $(SPDK_ROOT_DIR)/mk/nvme.libtest.mk
diff --git a/src/spdk/examples/nvme/reconnect/reconnect.c b/src/spdk/examples/nvme/reconnect/reconnect.c
new file mode 100644
index 000000000..74c5f3657
--- /dev/null
+++ b/src/spdk/examples/nvme/reconnect/reconnect.c
@@ -0,0 +1,1185 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation. All rights reserved.
+ * Copyright (c) 2020 Mellanox Technologies LTD. All rights reserved.
+ *
+ * Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+
+#include "spdk/env.h"
+#include "spdk/nvme.h"
+#include "spdk/queue.h"
+#include "spdk/string.h"
+#include "spdk/util.h"
+#include "spdk/log.h"
+#include "spdk/likely.h"
+
+struct ctrlr_entry {
+ struct spdk_nvme_ctrlr *ctrlr;
+ struct spdk_nvme_transport_id failover_trid;
+ enum spdk_nvme_transport_type trtype;
+ struct ctrlr_entry *next;
+ char name[1024];
+ int num_resets;
+};
+
+struct ns_entry {
+ struct spdk_nvme_ctrlr *ctrlr;
+ struct spdk_nvme_ns *ns;
+
+ struct ns_entry *next;
+ uint32_t io_size_blocks;
+ uint32_t num_io_requests;
+ uint64_t size_in_ios;
+ uint32_t block_size;
+ uint32_t io_flags;
+ char name[1024];
+};
+
+struct ns_worker_ctx {
+ struct ns_entry *entry;
+ uint64_t io_completed;
+ uint64_t current_queue_depth;
+ uint64_t offset_in_ios;
+ bool is_draining;
+
+ int num_qpairs;
+ struct spdk_nvme_qpair **qpair;
+ int last_qpair;
+
+ struct ns_worker_ctx *next;
+};
+
+struct perf_task {
+ struct ns_worker_ctx *ns_ctx;
+ struct iovec iov;
+ bool is_read;
+};
+
+struct worker_thread {
+ struct ns_worker_ctx *ns_ctx;
+ struct worker_thread *next;
+ unsigned lcore;
+};
+
+/* For basic reset handling. */
+static int g_max_ctrlr_resets = 15;
+
+static struct ctrlr_entry *g_controllers = NULL;
+static struct ns_entry *g_namespaces = NULL;
+static int g_num_namespaces = 0;
+static struct worker_thread *g_workers = NULL;
+static int g_num_workers = 0;
+
+static uint64_t g_tsc_rate;
+
+static uint32_t g_io_align = 0x200;
+static uint32_t g_io_size_bytes;
+static uint32_t g_max_io_size_blocks;
+static int g_rw_percentage;
+static int g_is_random;
+static int g_queue_depth;
+static int g_time_in_sec;
+static uint32_t g_max_completions;
+static int g_dpdk_mem;
+static bool g_warn;
+static uint32_t g_keep_alive_timeout_in_ms = 0;
+static uint8_t g_transport_retry_count = 4;
+static uint8_t g_transport_ack_timeout = 0; /* disabled */
+
+static const char *g_core_mask;
+
+struct trid_entry {
+ struct spdk_nvme_transport_id trid;
+ struct spdk_nvme_transport_id failover_trid;
+ TAILQ_ENTRY(trid_entry) tailq;
+};
+
+static TAILQ_HEAD(, trid_entry) g_trid_list = TAILQ_HEAD_INITIALIZER(g_trid_list);
+
+static inline void
+task_complete(struct perf_task *task);
+static void submit_io(struct ns_worker_ctx *ns_ctx, int queue_depth);
+
+static void io_complete(void *ctx, const struct spdk_nvme_cpl *cpl);
+
+static void
+nvme_setup_payload(struct perf_task *task)
+{
+ /* maximum extended lba format size from all active namespace,
+ * it's same with g_io_size_bytes for namespace without metadata.
+ */
+ task->iov.iov_base = spdk_dma_zmalloc(g_io_size_bytes, g_io_align, NULL);
+ task->iov.iov_len = g_io_size_bytes;
+ if (task->iov.iov_base == NULL) {
+ fprintf(stderr, "task->buf spdk_dma_zmalloc failed\n");
+ exit(1);
+ }
+}
+
+static int
+nvme_submit_io(struct perf_task *task, struct ns_worker_ctx *ns_ctx,
+ struct ns_entry *entry, uint64_t offset_in_ios)
+{
+ uint64_t lba;
+ int qp_num;
+
+ lba = offset_in_ios * entry->io_size_blocks;
+
+ qp_num = ns_ctx->last_qpair;
+ ns_ctx->last_qpair++;
+ if (ns_ctx->last_qpair == ns_ctx->num_qpairs) {
+ ns_ctx->last_qpair = 0;
+ }
+
+ if (task->is_read) {
+ return spdk_nvme_ns_cmd_read(entry->ns, ns_ctx->qpair[qp_num],
+ task->iov.iov_base, lba,
+ entry->io_size_blocks, io_complete,
+ task, entry->io_flags);
+ }
+
+ return spdk_nvme_ns_cmd_write(entry->ns, ns_ctx->qpair[qp_num],
+ task->iov.iov_base, lba,
+ entry->io_size_blocks, io_complete,
+ task, entry->io_flags);
+}
+
+static void
+nvme_check_io(struct ns_worker_ctx *ns_ctx)
+{
+ int i, rc;
+
+ for (i = 0; i < ns_ctx->num_qpairs; i++) {
+ rc = spdk_nvme_qpair_process_completions(ns_ctx->qpair[i], g_max_completions);
+ /* The transport level qpair is failed and we need to reconnect it. */
+ if (spdk_unlikely(rc == -ENXIO)) {
+ rc = spdk_nvme_ctrlr_reconnect_io_qpair(ns_ctx->qpair[i]);
+ /* successful reconnect */
+ if (rc == 0) {
+ continue;
+ } else if (rc == -ENXIO) {
+ /* This means the controller is failed. Defer to it to restore the qpair. */
+ continue;
+ } else {
+ /*
+ * We were unable to restore the qpair on this attempt. We don't
+ * really know why. For naive handling, just keep trying.
+ * TODO: add a retry limit, and destroy the qpair after x iterations.
+ */
+ fprintf(stderr, "qpair failed and we were unable to recover it.\n");
+ }
+ } else if (spdk_unlikely(rc < 0)) {
+ fprintf(stderr, "Received an unknown error processing completions.\n");
+ exit(1);
+ }
+ }
+}
+
+/*
+ * TODO: If a controller has multiple namespaces, they could all use the same queue.
+ * For now, give each namespace/thread combination its own queue.
+ */
+static int
+nvme_init_ns_worker_ctx(struct ns_worker_ctx *ns_ctx)
+{
+ struct spdk_nvme_io_qpair_opts opts;
+ struct ns_entry *entry = ns_ctx->entry;
+ int i;
+
+ ns_ctx->num_qpairs = 1;
+ ns_ctx->qpair = calloc(ns_ctx->num_qpairs, sizeof(struct spdk_nvme_qpair *));
+ if (!ns_ctx->qpair) {
+ return -1;
+ }
+
+ spdk_nvme_ctrlr_get_default_io_qpair_opts(entry->ctrlr, &opts, sizeof(opts));
+ if (opts.io_queue_requests < entry->num_io_requests) {
+ opts.io_queue_requests = entry->num_io_requests;
+ }
+
+ for (i = 0; i < ns_ctx->num_qpairs; i++) {
+ ns_ctx->qpair[i] = spdk_nvme_ctrlr_alloc_io_qpair(entry->ctrlr, &opts,
+ sizeof(opts));
+ if (!ns_ctx->qpair[i]) {
+ printf("ERROR: spdk_nvme_ctrlr_alloc_io_qpair failed\n");
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
+static void
+nvme_cleanup_ns_worker_ctx(struct ns_worker_ctx *ns_ctx)
+{
+ int i;
+
+ for (i = 0; i < ns_ctx->num_qpairs; i++) {
+ spdk_nvme_ctrlr_free_io_qpair(ns_ctx->qpair[i]);
+ }
+
+ free(ns_ctx->qpair);
+}
+
+static void
+build_nvme_name(char *name, size_t length, struct spdk_nvme_ctrlr *ctrlr)
+{
+ const struct spdk_nvme_transport_id *trid;
+
+ trid = spdk_nvme_ctrlr_get_transport_id(ctrlr);
+
+ switch (trid->trtype) {
+ case SPDK_NVME_TRANSPORT_RDMA:
+ snprintf(name, length, "RDMA (addr:%s subnqn:%s)", trid->traddr, trid->subnqn);
+ break;
+ case SPDK_NVME_TRANSPORT_TCP:
+ snprintf(name, length, "TCP (addr:%s subnqn:%s)", trid->traddr, trid->subnqn);
+ break;
+ default:
+ fprintf(stderr, "Unknown transport type %d\n", trid->trtype);
+ break;
+ }
+}
+
+static void
+register_ns(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns *ns)
+{
+ struct ns_entry *entry;
+ const struct spdk_nvme_ctrlr_data *cdata;
+ uint32_t max_xfer_size, entries, sector_size;
+ uint64_t ns_size;
+ struct spdk_nvme_io_qpair_opts opts;
+
+ cdata = spdk_nvme_ctrlr_get_data(ctrlr);
+
+ if (!spdk_nvme_ns_is_active(ns)) {
+ printf("Controller %-20.20s (%-20.20s): Skipping inactive NS %u\n",
+ cdata->mn, cdata->sn,
+ spdk_nvme_ns_get_id(ns));
+ g_warn = true;
+ return;
+ }
+
+ ns_size = spdk_nvme_ns_get_size(ns);
+ sector_size = spdk_nvme_ns_get_sector_size(ns);
+
+ if (ns_size < g_io_size_bytes || sector_size > g_io_size_bytes) {
+ printf("WARNING: controller %-20.20s (%-20.20s) ns %u has invalid "
+ "ns size %" PRIu64 " / block size %u for I/O size %u\n",
+ cdata->mn, cdata->sn, spdk_nvme_ns_get_id(ns),
+ ns_size, spdk_nvme_ns_get_sector_size(ns), g_io_size_bytes);
+ g_warn = true;
+ return;
+ }
+
+ max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns);
+ spdk_nvme_ctrlr_get_default_io_qpair_opts(ctrlr, &opts, sizeof(opts));
+ /* NVMe driver may add additional entries based on
+ * stripe size and maximum transfer size, we assume
+ * 1 more entry be used for stripe.
+ */
+ entries = (g_io_size_bytes - 1) / max_xfer_size + 2;
+ if ((g_queue_depth * entries) > opts.io_queue_size) {
+ printf("controller IO queue size %u less than required\n",
+ opts.io_queue_size);
+ printf("Consider using lower queue depth or small IO size because "
+ "IO requests may be queued at the NVMe driver.\n");
+ g_warn = true;
+ }
+ /* For requests which have children requests, parent request itself
+ * will also occupy 1 entry.
+ */
+ entries += 1;
+
+ entry = calloc(1, sizeof(struct ns_entry));
+ if (entry == NULL) {
+ perror("ns_entry malloc");
+ exit(1);
+ }
+
+ entry->ctrlr = ctrlr;
+ entry->ns = ns;
+ entry->num_io_requests = g_queue_depth * entries;
+
+ entry->size_in_ios = ns_size / g_io_size_bytes;
+ entry->io_size_blocks = g_io_size_bytes / sector_size;
+
+ entry->block_size = spdk_nvme_ns_get_sector_size(ns);
+
+
+ if (g_max_io_size_blocks < entry->io_size_blocks) {
+ g_max_io_size_blocks = entry->io_size_blocks;
+ }
+
+ build_nvme_name(entry->name, sizeof(entry->name), ctrlr);
+
+ g_num_namespaces++;
+ entry->next = g_namespaces;
+ g_namespaces = entry;
+}
+
+static void
+unregister_namespaces(void)
+{
+ struct ns_entry *entry = g_namespaces;
+
+ while (entry) {
+ struct ns_entry *next = entry->next;
+ free(entry);
+ entry = next;
+ }
+}
+
+static void
+register_ctrlr(struct spdk_nvme_ctrlr *ctrlr, struct trid_entry *trid_entry)
+{
+ struct spdk_nvme_ns *ns;
+ struct ctrlr_entry *entry = calloc(1, sizeof(struct ctrlr_entry));
+ const struct spdk_nvme_transport_id *ctrlr_trid;
+ uint32_t nsid;
+
+ if (entry == NULL) {
+ perror("ctrlr_entry malloc");
+ exit(1);
+ }
+
+ ctrlr_trid = spdk_nvme_ctrlr_get_transport_id(ctrlr);
+ assert(ctrlr_trid != NULL);
+
+ /* each controller needs a unique failover trid. */
+ entry->failover_trid = trid_entry->failover_trid;
+
+ /*
+ * Users are allowed to leave the trid subnqn blank or specify a discovery controller subnqn.
+ * In those cases, the controller subnqn will not equal the trid_entry subnqn and, by association,
+ * the failover_trid subnqn.
+ * When we do failover, we want to reconnect to the same nqn so explicitly set the failover nqn to
+ * the ctrlr nqn here.
+ */
+ snprintf(entry->failover_trid.subnqn, SPDK_NVMF_NQN_MAX_LEN + 1, "%s", ctrlr_trid->subnqn);
+
+
+ build_nvme_name(entry->name, sizeof(entry->name), ctrlr);
+
+ entry->ctrlr = ctrlr;
+ entry->trtype = trid_entry->trid.trtype;
+ entry->next = g_controllers;
+ g_controllers = entry;
+
+ for (nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr);
+ nsid != 0; nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid)) {
+ ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid);
+ if (ns == NULL) {
+ continue;
+ }
+ register_ns(ctrlr, ns);
+ }
+}
+
+static __thread unsigned int seed = 0;
+
+static inline void
+submit_single_io(struct perf_task *task)
+{
+ uint64_t offset_in_ios;
+ int rc;
+ struct ns_worker_ctx *ns_ctx = task->ns_ctx;
+ struct ns_entry *entry = ns_ctx->entry;
+
+ if (g_is_random) {
+ offset_in_ios = rand_r(&seed) % entry->size_in_ios;
+ } else {
+ offset_in_ios = ns_ctx->offset_in_ios++;
+ if (ns_ctx->offset_in_ios == entry->size_in_ios) {
+ ns_ctx->offset_in_ios = 0;
+ }
+ }
+
+ if ((g_rw_percentage == 100) ||
+ (g_rw_percentage != 0 && ((rand_r(&seed) % 100) < g_rw_percentage))) {
+ task->is_read = true;
+ } else {
+ task->is_read = false;
+ }
+
+ rc = nvme_submit_io(task, ns_ctx, entry, offset_in_ios);
+
+ if (spdk_unlikely(rc != 0)) {
+ fprintf(stderr, "starting I/O failed\n");
+ } else {
+ ns_ctx->current_queue_depth++;
+ }
+}
+
+static inline void
+task_complete(struct perf_task *task)
+{
+ struct ns_worker_ctx *ns_ctx;
+
+ ns_ctx = task->ns_ctx;
+ ns_ctx->current_queue_depth--;
+ ns_ctx->io_completed++;
+
+ /*
+ * is_draining indicates when time has expired for the test run
+ * and we are just waiting for the previously submitted I/O
+ * to complete. In this case, do not submit a new I/O to replace
+ * the one just completed.
+ */
+ if (spdk_unlikely(ns_ctx->is_draining)) {
+ spdk_dma_free(task->iov.iov_base);
+ free(task);
+ } else {
+ submit_single_io(task);
+ }
+}
+
+static void
+io_complete(void *ctx, const struct spdk_nvme_cpl *cpl)
+{
+ struct perf_task *task = ctx;
+
+ if (spdk_unlikely(spdk_nvme_cpl_is_error(cpl))) {
+ fprintf(stderr, "%s completed with error (sct=%d, sc=%d)\n",
+ task->is_read ? "Read" : "Write",
+ cpl->status.sct, cpl->status.sc);
+ }
+
+ task_complete(task);
+}
+
+static void
+check_io(struct ns_worker_ctx *ns_ctx)
+{
+ nvme_check_io(ns_ctx);
+}
+
+static struct perf_task *
+allocate_task(struct ns_worker_ctx *ns_ctx, int queue_depth)
+{
+ struct perf_task *task;
+
+ task = calloc(1, sizeof(*task));
+ if (task == NULL) {
+ fprintf(stderr, "Out of memory allocating tasks\n");
+ exit(1);
+ }
+
+ nvme_setup_payload(task);
+
+ task->ns_ctx = ns_ctx;
+
+ return task;
+}
+
+static void
+submit_io(struct ns_worker_ctx *ns_ctx, int queue_depth)
+{
+ struct perf_task *task;
+
+ while (queue_depth-- > 0) {
+ task = allocate_task(ns_ctx, queue_depth);
+ submit_single_io(task);
+ }
+}
+
+static int
+work_fn(void *arg)
+{
+ uint64_t tsc_end;
+ struct worker_thread *worker = (struct worker_thread *)arg;
+ struct ns_worker_ctx *ns_ctx = NULL;
+ uint32_t unfinished_ns_ctx;
+
+ printf("Starting thread on core %u\n", worker->lcore);
+
+ /* Allocate queue pairs for each namespace. */
+ ns_ctx = worker->ns_ctx;
+ while (ns_ctx != NULL) {
+ if (nvme_init_ns_worker_ctx(ns_ctx) != 0) {
+ printf("ERROR: init_ns_worker_ctx() failed\n");
+ return 1;
+ }
+ ns_ctx = ns_ctx->next;
+ }
+
+ tsc_end = spdk_get_ticks() + g_time_in_sec * g_tsc_rate;
+
+ /* Submit initial I/O for each namespace. */
+ ns_ctx = worker->ns_ctx;
+ while (ns_ctx != NULL) {
+ submit_io(ns_ctx, g_queue_depth);
+ ns_ctx = ns_ctx->next;
+ }
+
+ while (1) {
+ /*
+ * Check for completed I/O for each controller. A new
+ * I/O will be submitted in the io_complete callback
+ * to replace each I/O that is completed.
+ */
+ ns_ctx = worker->ns_ctx;
+ while (ns_ctx != NULL) {
+ check_io(ns_ctx);
+ ns_ctx = ns_ctx->next;
+ }
+
+ if (spdk_get_ticks() > tsc_end) {
+ break;
+ }
+ }
+
+ /* drain the io of each ns_ctx in round robin to make the fairness */
+ do {
+ unfinished_ns_ctx = 0;
+ ns_ctx = worker->ns_ctx;
+ while (ns_ctx != NULL) {
+ /* first time will enter into this if case */
+ if (!ns_ctx->is_draining) {
+ ns_ctx->is_draining = true;
+ }
+
+ if (ns_ctx->current_queue_depth > 0) {
+ check_io(ns_ctx);
+ if (ns_ctx->current_queue_depth == 0) {
+ nvme_cleanup_ns_worker_ctx(ns_ctx);
+ } else {
+ unfinished_ns_ctx++;
+ }
+ }
+ ns_ctx = ns_ctx->next;
+ }
+ } while (unfinished_ns_ctx > 0);
+
+ return 0;
+}
+
+static void usage(char *program_name)
+{
+ printf("%s options", program_name);
+ printf("\n");
+ printf("\t[-q io depth]\n");
+ printf("\t[-o io size in bytes]\n");
+ printf("\t[-w io pattern type, must be one of\n");
+ printf("\t\t(read, write, randread, randwrite, rw, randrw)]\n");
+ printf("\t[-M rwmixread (100 for reads, 0 for writes)]\n");
+ printf("\t[-t time in seconds]\n");
+ printf("\t[-c core mask for I/O submission/completion.]\n");
+ printf("\t\t(default: 1)\n");
+ printf("\t[-r Transport ID for NVMeoF]\n");
+ printf("\t Format: 'key:value [key:value] ...'\n");
+ printf("\t Keys:\n");
+ printf("\t trtype Transport type (e.g. RDMA)\n");
+ printf("\t adrfam Address family (e.g. IPv4, IPv6)\n");
+ printf("\t traddr Transport address (e.g. 192.168.100.8 for RDMA)\n");
+ printf("\t trsvcid Transport service identifier (e.g. 4420)\n");
+ printf("\t subnqn Subsystem NQN (default: %s)\n", SPDK_NVMF_DISCOVERY_NQN);
+ printf("\t alt_traddr (Optional) Alternative Transport address for failover.\n");
+ printf("\t Example: -r 'trtype:RDMA adrfam:IPv4 traddr:192.168.100.8 trsvcid:4420' for NVMeoF\n");
+ printf("\t[-k keep alive timeout period in millisecond]\n");
+ printf("\t[-s DPDK huge memory size in MB.]\n");
+ printf("\t[-m max completions per poll]\n");
+ printf("\t\t(default: 0 - unlimited)\n");
+ printf("\t[-i shared memory group ID]\n");
+ printf("\t[-A transport ACK timeout]\n");
+ printf("\t[-R transport retry count]\n");
+ printf("\t");
+ spdk_log_usage(stdout, "-T");
+#ifdef DEBUG
+ printf("\t[-G enable debug logging]\n");
+#else
+ printf("\t[-G enable debug logging (flag disabled, must reconfigure with --enable-debug)\n");
+#endif
+}
+
+static void
+unregister_trids(void)
+{
+ struct trid_entry *trid_entry, *tmp;
+
+ TAILQ_FOREACH_SAFE(trid_entry, &g_trid_list, tailq, tmp) {
+ TAILQ_REMOVE(&g_trid_list, trid_entry, tailq);
+ free(trid_entry);
+ }
+}
+
+static int
+add_trid(const char *trid_str)
+{
+ struct trid_entry *trid_entry;
+ struct spdk_nvme_transport_id *trid;
+ char *alt_traddr;
+ int len;
+
+ trid_entry = calloc(1, sizeof(*trid_entry));
+ if (trid_entry == NULL) {
+ return -1;
+ }
+
+ trid = &trid_entry->trid;
+ snprintf(trid->subnqn, sizeof(trid->subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN);
+
+ if (spdk_nvme_transport_id_parse(trid, trid_str) != 0) {
+ fprintf(stderr, "Invalid transport ID format '%s'\n", trid_str);
+ free(trid_entry);
+ return 1;
+ }
+
+ trid_entry->failover_trid = trid_entry->trid;
+
+ alt_traddr = strcasestr(trid_str, "alt_traddr:");
+ if (alt_traddr) {
+ alt_traddr += strlen("alt_traddr:");
+ len = strcspn(alt_traddr, " \t\n");
+ if (len > SPDK_NVMF_TRADDR_MAX_LEN) {
+ fprintf(stderr, "The failover traddr %s is too long.\n", alt_traddr);
+ free(trid_entry);
+ return -1;
+ }
+ snprintf(trid_entry->failover_trid.traddr, SPDK_NVMF_TRADDR_MAX_LEN + 1, "%s", alt_traddr);
+ }
+
+ TAILQ_INSERT_TAIL(&g_trid_list, trid_entry, tailq);
+ return 0;
+}
+
+static int
+parse_args(int argc, char **argv)
+{
+ struct trid_entry *trid_entry, *trid_entry_tmp;
+ const char *workload_type;
+ int op;
+ bool mix_specified = false;
+ long int val;
+ int rc;
+
+ /* default value */
+ g_queue_depth = 0;
+ g_io_size_bytes = 0;
+ workload_type = NULL;
+ g_time_in_sec = 0;
+ g_rw_percentage = -1;
+ g_core_mask = NULL;
+ g_max_completions = 0;
+
+ while ((op = getopt(argc, argv, "c:m:o:q:r:k:s:t:w:A:GM:R:T:")) != -1) {
+ switch (op) {
+ case 'm':
+ case 'o':
+ case 'q':
+ case 'k':
+ case 's':
+ case 't':
+ case 'A':
+ case 'M':
+ case 'R':
+ val = spdk_strtol(optarg, 10);
+ if (val < 0) {
+ fprintf(stderr, "Converting a string to integer failed\n");
+ return val;
+ }
+ switch (op) {
+ case 'm':
+ g_max_completions = val;
+ break;
+ case 'o':
+ g_io_size_bytes = val;
+ break;
+ case 'q':
+ g_queue_depth = val;
+ break;
+ case 'k':
+ g_keep_alive_timeout_in_ms = val;
+ break;
+ case 's':
+ g_dpdk_mem = val;
+ break;
+ case 't':
+ g_time_in_sec = val;
+ break;
+ case 'A':
+ g_transport_ack_timeout = val;
+ break;
+ case 'M':
+ g_rw_percentage = val;
+ mix_specified = true;
+ break;
+ case 'R':
+ g_transport_retry_count = val;
+ break;
+ }
+ break;
+ case 'c':
+ g_core_mask = optarg;
+ break;
+ case 'r':
+ if (add_trid(optarg)) {
+ usage(argv[0]);
+ return 1;
+ }
+ break;
+ case 'w':
+ workload_type = optarg;
+ break;
+ case 'G':
+#ifndef DEBUG
+ fprintf(stderr, "%s must be configured with --enable-debug for -G flag\n",
+ argv[0]);
+ usage(argv[0]);
+ return 1;
+#else
+ spdk_log_set_flag("nvme");
+ spdk_log_set_print_level(SPDK_LOG_DEBUG);
+ break;
+#endif
+ case 'T':
+ rc = spdk_log_set_flag(optarg);
+ if (rc < 0) {
+ fprintf(stderr, "unknown flag\n");
+ usage(argv[0]);
+ exit(EXIT_FAILURE);
+ }
+ spdk_log_set_print_level(SPDK_LOG_DEBUG);
+#ifndef DEBUG
+ fprintf(stderr, "%s must be rebuilt with CONFIG_DEBUG=y for -T flag.\n",
+ argv[0]);
+ usage(argv[0]);
+ return 0;
+#endif
+ break;
+ default:
+ usage(argv[0]);
+ return 1;
+ }
+ }
+
+ if (!g_queue_depth) {
+ usage(argv[0]);
+ return 1;
+ }
+ if (!g_io_size_bytes) {
+ usage(argv[0]);
+ return 1;
+ }
+ if (!workload_type) {
+ usage(argv[0]);
+ return 1;
+ }
+ if (!g_time_in_sec) {
+ usage(argv[0]);
+ return 1;
+ }
+
+ if (strcmp(workload_type, "read") &&
+ strcmp(workload_type, "write") &&
+ strcmp(workload_type, "randread") &&
+ strcmp(workload_type, "randwrite") &&
+ strcmp(workload_type, "rw") &&
+ strcmp(workload_type, "randrw")) {
+ fprintf(stderr,
+ "io pattern type must be one of\n"
+ "(read, write, randread, randwrite, rw, randrw)\n");
+ return 1;
+ }
+
+ if (!strcmp(workload_type, "read") ||
+ !strcmp(workload_type, "randread")) {
+ g_rw_percentage = 100;
+ }
+
+ if (!strcmp(workload_type, "write") ||
+ !strcmp(workload_type, "randwrite")) {
+ g_rw_percentage = 0;
+ }
+
+ if (!strcmp(workload_type, "read") ||
+ !strcmp(workload_type, "randread") ||
+ !strcmp(workload_type, "write") ||
+ !strcmp(workload_type, "randwrite")) {
+ if (mix_specified) {
+ fprintf(stderr, "Ignoring -M option... Please use -M option"
+ " only when using rw or randrw.\n");
+ }
+ }
+
+ if (!strcmp(workload_type, "rw") ||
+ !strcmp(workload_type, "randrw")) {
+ if (g_rw_percentage < 0 || g_rw_percentage > 100) {
+ fprintf(stderr,
+ "-M must be specified to value from 0 to 100 "
+ "for rw or randrw.\n");
+ return 1;
+ }
+ }
+
+ if (!strcmp(workload_type, "read") ||
+ !strcmp(workload_type, "write") ||
+ !strcmp(workload_type, "rw")) {
+ g_is_random = 0;
+ } else {
+ g_is_random = 1;
+ }
+
+ if (TAILQ_EMPTY(&g_trid_list)) {
+ fprintf(stderr, "You must specify at least one fabrics TRID.\n");
+ return -1;
+ }
+
+ /* check whether there is local PCIe type and fail. */
+ TAILQ_FOREACH_SAFE(trid_entry, &g_trid_list, tailq, trid_entry_tmp) {
+ if (trid_entry->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) {
+ fprintf(stderr, "This application was not intended to be run on PCIe controllers.\n");
+ return 1;
+ }
+ }
+
+ return 0;
+}
+
+static int
+register_workers(void)
+{
+ uint32_t i;
+ struct worker_thread *worker;
+
+ g_workers = NULL;
+ g_num_workers = 0;
+
+ SPDK_ENV_FOREACH_CORE(i) {
+ worker = calloc(1, sizeof(*worker));
+ if (worker == NULL) {
+ fprintf(stderr, "Unable to allocate worker\n");
+ return -1;
+ }
+
+ worker->lcore = i;
+ worker->next = g_workers;
+ g_workers = worker;
+ g_num_workers++;
+ }
+
+ return 0;
+}
+
+static void
+unregister_workers(void)
+{
+ struct worker_thread *worker = g_workers;
+
+ /* Free namespace context and worker thread */
+ while (worker) {
+ struct worker_thread *next_worker = worker->next;
+ struct ns_worker_ctx *ns_ctx = worker->ns_ctx;
+
+ while (ns_ctx) {
+ struct ns_worker_ctx *next_ns_ctx = ns_ctx->next;
+ free(ns_ctx);
+ ns_ctx = next_ns_ctx;
+ }
+
+ free(worker);
+ worker = next_worker;
+ }
+}
+
+static bool
+probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
+ struct spdk_nvme_ctrlr_opts *opts)
+{
+ /* These should have been weeded out earlier. */
+ assert(trid->trtype != SPDK_NVME_TRANSPORT_PCIE);
+
+ printf("Attaching to NVMe over Fabrics controller at %s:%s: %s\n",
+ trid->traddr, trid->trsvcid,
+ trid->subnqn);
+
+ /* Set io_queue_size to UINT16_MAX, NVMe driver
+ * will then reduce this to MQES to maximize
+ * the io_queue_size as much as possible.
+ */
+ opts->io_queue_size = UINT16_MAX;
+
+ opts->keep_alive_timeout_ms = spdk_max(opts->keep_alive_timeout_ms,
+ g_keep_alive_timeout_in_ms);
+
+ opts->transport_retry_count = g_transport_retry_count;
+ opts->transport_ack_timeout = g_transport_ack_timeout;
+
+ return true;
+}
+
+static void
+attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
+ struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts)
+{
+ struct trid_entry *trid_entry = cb_ctx;
+
+ printf("Attached to NVMe over Fabrics controller at %s:%s: %s\n",
+ trid->traddr, trid->trsvcid,
+ trid->subnqn);
+
+ register_ctrlr(ctrlr, trid_entry);
+}
+
+static int
+register_controllers(void)
+{
+ struct trid_entry *trid_entry;
+
+ printf("Initializing NVMe Controllers\n");
+
+ TAILQ_FOREACH(trid_entry, &g_trid_list, tailq) {
+ if (spdk_nvme_probe(&trid_entry->trid, trid_entry, probe_cb, attach_cb, NULL) != 0) {
+ fprintf(stderr, "spdk_nvme_probe() failed for transport address '%s'\n",
+ trid_entry->trid.traddr);
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
+static void
+unregister_controllers(void)
+{
+ struct ctrlr_entry *entry = g_controllers;
+
+ while (entry) {
+ struct ctrlr_entry *next = entry->next;
+
+ spdk_nvme_detach(entry->ctrlr);
+ free(entry);
+ entry = next;
+ }
+}
+
+static int
+associate_workers_with_ns(void)
+{
+ struct ns_entry *entry = g_namespaces;
+ struct worker_thread *worker = g_workers;
+ struct ns_worker_ctx *ns_ctx;
+ int i, count;
+
+ count = g_num_namespaces > g_num_workers ? g_num_namespaces : g_num_workers;
+
+ for (i = 0; i < count; i++) {
+ if (entry == NULL) {
+ break;
+ }
+
+ ns_ctx = calloc(1, sizeof(struct ns_worker_ctx));
+ if (!ns_ctx) {
+ return -1;
+ }
+
+ printf("Associating %s with lcore %d\n", entry->name, worker->lcore);
+ ns_ctx->entry = entry;
+ ns_ctx->next = worker->ns_ctx;
+ worker->ns_ctx = ns_ctx;
+
+ worker = worker->next;
+ if (worker == NULL) {
+ worker = g_workers;
+ }
+
+ entry = entry->next;
+ if (entry == NULL) {
+ entry = g_namespaces;
+ }
+
+ }
+
+ return 0;
+}
+
+static void *
+nvme_poll_ctrlrs(void *arg)
+{
+ struct ctrlr_entry *entry;
+ const struct spdk_nvme_transport_id *old_trid;
+ int oldstate;
+ int rc;
+
+
+ spdk_unaffinitize_thread();
+
+ while (true) {
+ pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate);
+
+ entry = g_controllers;
+ while (entry) {
+ rc = spdk_nvme_ctrlr_process_admin_completions(entry->ctrlr);
+ /* This controller has encountered a failure at the transport level. reset it. */
+ if (rc == -ENXIO) {
+ if (entry->num_resets == 0) {
+ old_trid = spdk_nvme_ctrlr_get_transport_id(entry->ctrlr);
+ fprintf(stderr, "A controller has encountered a failure and is being reset.\n");
+ if (spdk_nvme_transport_id_compare(old_trid, &entry->failover_trid)) {
+ fprintf(stderr, "Resorting to new failover address %s\n", entry->failover_trid.traddr);
+ spdk_nvme_ctrlr_fail(entry->ctrlr);
+ rc = spdk_nvme_ctrlr_set_trid(entry->ctrlr, &entry->failover_trid);
+ if (rc != 0) {
+ fprintf(stderr, "Unable to fail over to back up trid.\n");
+ }
+ }
+ }
+
+ rc = spdk_nvme_ctrlr_reset(entry->ctrlr);
+ if (rc != 0) {
+ entry->num_resets++;
+ fprintf(stderr, "Unable to reset the controller.\n");
+
+ if (entry->num_resets > g_max_ctrlr_resets) {
+ fprintf(stderr, "Controller cannot be recovered. Exiting.\n");
+ exit(1);
+ }
+ } else {
+ fprintf(stderr, "Controller properly reset.\n");
+ }
+ }
+ entry = entry->next;
+ }
+
+ pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &oldstate);
+
+ /* This is a pthread cancellation point and cannot be removed. */
+ sleep(1);
+ }
+
+ return NULL;
+}
+
+int main(int argc, char **argv)
+{
+ int rc;
+ struct worker_thread *worker, *master_worker;
+ unsigned master_core;
+ struct spdk_env_opts opts;
+ pthread_t thread_id = 0;
+
+ rc = parse_args(argc, argv);
+ if (rc != 0) {
+ return rc;
+ }
+
+ spdk_env_opts_init(&opts);
+ opts.name = "reconnect";
+ if (g_core_mask) {
+ opts.core_mask = g_core_mask;
+ }
+
+ if (g_dpdk_mem) {
+ opts.mem_size = g_dpdk_mem;
+ }
+ if (spdk_env_init(&opts) < 0) {
+ fprintf(stderr, "Unable to initialize SPDK env\n");
+ rc = 1;
+ goto cleanup;
+ }
+
+ g_tsc_rate = spdk_get_ticks_hz();
+
+ if (register_workers() != 0) {
+ rc = 1;
+ goto cleanup;
+ }
+
+ if (register_controllers() != 0) {
+ rc = 1;
+ goto cleanup;
+ }
+
+ if (g_warn) {
+ printf("WARNING: Some requested NVMe devices were skipped\n");
+ }
+
+ if (g_num_namespaces == 0) {
+ fprintf(stderr, "No valid NVMe controllers found\n");
+ goto cleanup;
+ }
+
+ rc = pthread_create(&thread_id, NULL, &nvme_poll_ctrlrs, NULL);
+ if (rc != 0) {
+ fprintf(stderr, "Unable to spawn a thread to poll admin queues.\n");
+ goto cleanup;
+ }
+
+ if (associate_workers_with_ns() != 0) {
+ rc = 1;
+ goto cleanup;
+ }
+
+ printf("Initialization complete. Launching workers.\n");
+
+ /* Launch all of the slave workers */
+ master_core = spdk_env_get_current_core();
+ master_worker = NULL;
+ worker = g_workers;
+ while (worker != NULL) {
+ if (worker->lcore != master_core) {
+ spdk_env_thread_launch_pinned(worker->lcore, work_fn, worker);
+ } else {
+ assert(master_worker == NULL);
+ master_worker = worker;
+ }
+ worker = worker->next;
+ }
+
+ assert(master_worker != NULL);
+ rc = work_fn(master_worker);
+
+ spdk_env_thread_wait_all();
+
+cleanup:
+ if (thread_id && pthread_cancel(thread_id) == 0) {
+ pthread_join(thread_id, NULL);
+ }
+ unregister_trids();
+ unregister_namespaces();
+ unregister_controllers();
+ unregister_workers();
+
+ if (rc != 0) {
+ fprintf(stderr, "%s: errors occured\n", argv[0]);
+ /*
+ * return a generic error to the caller. This allows us to
+ * distinguish between a failure in the script and something
+ * like a segfault or an invalid access which causes the program
+ * to crash.
+ */
+ rc = 1;
+ }
+
+ return rc;
+}
diff --git a/src/spdk/examples/nvmf/Makefile b/src/spdk/examples/nvmf/Makefile
new file mode 100644
index 000000000..f34027406
--- /dev/null
+++ b/src/spdk/examples/nvmf/Makefile
@@ -0,0 +1,44 @@
+#
+# BSD LICENSE
+#
+# Copyright (c) Intel Corporation.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Intel Corporation nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+DIRS-y += nvmf
+
+.PHONY: all clean $(DIRS-y)
+
+all: $(DIRS-y)
+clean: $(DIRS-y)
+
+include $(SPDK_ROOT_DIR)/mk/spdk.subdirs.mk
diff --git a/src/spdk/examples/nvmf/nvmf/.gitignore b/src/spdk/examples/nvmf/nvmf/.gitignore
new file mode 100644
index 000000000..461c83676
--- /dev/null
+++ b/src/spdk/examples/nvmf/nvmf/.gitignore
@@ -0,0 +1 @@
+nvmf
diff --git a/src/spdk/examples/nvmf/nvmf/Makefile b/src/spdk/examples/nvmf/nvmf/Makefile
new file mode 100644
index 000000000..ccf5273be
--- /dev/null
+++ b/src/spdk/examples/nvmf/nvmf/Makefile
@@ -0,0 +1,52 @@
+#
+# BSD LICENSE
+#
+# Copyright (c) Intel Corporation.
+# Copyright (c) 2015-2016, Micron Technology, Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Intel Corporation nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+include $(SPDK_ROOT_DIR)/mk/spdk.modules.mk
+
+APP := nvmf
+
+C_SRCS := nvmf.c
+SPDK_LIB_LIST = $(ALL_MODULES_LIST)
+SPDK_LIB_LIST += nvmf thread util bdev conf accel rpc jsonrpc json log sock trace notify
+SPDK_LIB_LIST += event $(EVENT_BDEV_SUBSYSTEM)
+
+ifeq ($(CONFIG_FC),y)
+ifneq ($(strip $(CONFIG_FC_PATH)),)
+SYS_LIBS += -L$(CONFIG_FC_PATH)
+endif
+SYS_LIBS += -lufc
+endif
+
+include $(SPDK_ROOT_DIR)/mk/spdk.app.mk
diff --git a/src/spdk/examples/nvmf/nvmf/README.md b/src/spdk/examples/nvmf/nvmf/README.md
new file mode 100644
index 000000000..c7432a6b7
--- /dev/null
+++ b/src/spdk/examples/nvmf/nvmf/README.md
@@ -0,0 +1,31 @@
+# NVMe-oF target without SPDK event framework
+
+## Overview
+
+This example is used to show how to use the nvmf lib. In this example we want to encourage user
+to use RPC cmd so we would only support RPC style.
+
+## Usage
+
+This example's usage is very similar with nvmf_tgt, difference is that you must use the RPC cmd
+to setup the nvmf target.
+
+First, start this example app. You can use the -m to specify how many cores you want to use.
+The other parameters you can use -h to show.
+ ./nvmf -m 0xf -r /var/tmp/spdk.sock
+
+Then, you need to use the RPC cmd to config the nvmf target. You can use the -h to get how many
+RPC cmd you can use. As this example is about nvmf so I think you can focus on the nvmf cmds and
+the bdev cmds.
+ ./scripts/rpc.py -h
+
+Next, You should use the RPC cmd to setup nvmf target.
+ ./scripts/rpc.py nvmf_create_transport -t RDMA -g nvmf_example
+ ./scripts/rpc.py nvmf_create_subsystem -t nvmf_example -s SPDK00000000000001 -a -m 32 nqn.2016-06.io.spdk:cnode1
+ ./scripts/rpc.py bdev_malloc_create -b Malloc1 128 512
+ ./scripts/rpc.py nvmf_subsystem_add_ns -t nvmf_example nqn.2016-06.io.spdk:cnode1 Malloc1
+ ./scripts/rpc.py nvmf_subsystem_add_listener -t rdma -f Ipv4 -a 192.168.0.10 -s 4420 -p nvmf_example nqn.2016-06.io.spdk:cnode1
+
+Last, start the initiator to connect the nvmf example target and test the IOs
+ $ROOT_SPDK/example/nvme/perf/perf -q 64 -o 4095 -w randrw -M 30 -l -t 60 \
+ -r "trtype:RDMA adrfam:IPv4 traddr:192.168.0.10 trsvcid:4420 subnqn:nqn.2016-06.io.spdk:cnode1"
diff --git a/src/spdk/examples/nvmf/nvmf/nvmf.c b/src/spdk/examples/nvmf/nvmf/nvmf.c
new file mode 100644
index 000000000..22e271fb3
--- /dev/null
+++ b/src/spdk/examples/nvmf/nvmf/nvmf.c
@@ -0,0 +1,905 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+#include "spdk/env.h"
+#include "spdk/event.h"
+#include "spdk/string.h"
+#include "spdk/thread.h"
+#include "spdk/bdev.h"
+#include "spdk/rpc.h"
+#include "spdk/nvmf.h"
+#include "spdk/likely.h"
+
+#include "spdk_internal/event.h"
+
+#define NVMF_DEFAULT_SUBSYSTEMS 32
+#define ACCEPT_TIMEOUT_US 10000 /* 10ms */
+
+static const char *g_rpc_addr = SPDK_DEFAULT_RPC_ADDR;
+static uint32_t g_acceptor_poll_rate = ACCEPT_TIMEOUT_US;
+
+enum nvmf_target_state {
+ NVMF_INIT_SUBSYSTEM = 0,
+ NVMF_INIT_TARGET,
+ NVMF_INIT_POLL_GROUPS,
+ NVMF_INIT_START_SUBSYSTEMS,
+ NVMF_INIT_START_ACCEPTOR,
+ NVMF_RUNNING,
+ NVMF_FINI_STOP_SUBSYSTEMS,
+ NVMF_FINI_POLL_GROUPS,
+ NVMF_FINI_STOP_ACCEPTOR,
+ NVMF_FINI_TARGET,
+ NVMF_FINI_SUBSYSTEM,
+};
+
+struct nvmf_lw_thread {
+ TAILQ_ENTRY(nvmf_lw_thread) link;
+ bool resched;
+};
+
+struct nvmf_reactor {
+ uint32_t core;
+
+ struct spdk_ring *threads;
+ TAILQ_ENTRY(nvmf_reactor) link;
+};
+
+struct nvmf_target_poll_group {
+ struct spdk_nvmf_poll_group *group;
+ struct spdk_thread *thread;
+
+ TAILQ_ENTRY(nvmf_target_poll_group) link;
+};
+
+struct nvmf_target {
+ struct spdk_nvmf_tgt *tgt;
+
+ int max_subsystems;
+};
+
+TAILQ_HEAD(, nvmf_reactor) g_reactors = TAILQ_HEAD_INITIALIZER(g_reactors);
+TAILQ_HEAD(, nvmf_target_poll_group) g_poll_groups = TAILQ_HEAD_INITIALIZER(g_poll_groups);
+static uint32_t g_num_poll_groups = 0;
+
+static struct nvmf_reactor *g_master_reactor = NULL;
+static struct nvmf_reactor *g_next_reactor = NULL;
+static struct spdk_thread *g_init_thread = NULL;
+static struct spdk_thread *g_fini_thread = NULL;
+static struct nvmf_target g_nvmf_tgt = {
+ .max_subsystems = NVMF_DEFAULT_SUBSYSTEMS,
+};
+static struct spdk_poller *g_acceptor_poller = NULL;
+static struct nvmf_target_poll_group *g_next_pg = NULL;
+static pthread_mutex_t g_mutex = PTHREAD_MUTEX_INITIALIZER;
+static bool g_reactors_exit = false;
+static enum nvmf_target_state g_target_state;
+static bool g_intr_received = false;
+
+static uint32_t g_migrate_pg_period_us = 0;
+static struct spdk_poller *g_migrate_pg_poller = NULL;
+
+static void nvmf_target_advance_state(void);
+static int nvmf_schedule_spdk_thread(struct spdk_thread *thread);
+
+static void
+usage(char *program_name)
+{
+ printf("%s options", program_name);
+ printf("\n");
+ printf("\t[-g period of round robin poll group migration (us) (default: 0 (disabled))]\n");
+ printf("\t[-h show this usage]\n");
+ printf("\t[-i shared memory ID (optional)]\n");
+ printf("\t[-m core mask for DPDK]\n");
+ printf("\t[-n max subsystems for target(default: 32)]\n");
+ printf("\t[-p acceptor poller rate in us for target(default: 10000us)]\n");
+ printf("\t[-r RPC listen address (default /var/tmp/spdk.sock)]\n");
+ printf("\t[-s memory size in MB for DPDK (default: 0MB)]\n");
+ printf("\t[-u disable PCI access]\n");
+}
+
+static int
+parse_args(int argc, char **argv, struct spdk_env_opts *opts)
+{
+ int op;
+ long int value;
+
+ while ((op = getopt(argc, argv, "g:i:m:n:p:r:s:u:h")) != -1) {
+ switch (op) {
+ case 'g':
+ value = spdk_strtol(optarg, 10);
+ if (value < 0) {
+ fprintf(stderr, "converting a string to integer failed\n");
+ return -EINVAL;
+ }
+ g_migrate_pg_period_us = value;
+ break;
+ case 'i':
+ value = spdk_strtol(optarg, 10);
+ if (value < 0) {
+ fprintf(stderr, "converting a string to integer failed\n");
+ return -EINVAL;
+ }
+ opts->shm_id = value;
+ break;
+ case 'm':
+ opts->core_mask = optarg;
+ break;
+ case 'n':
+ g_nvmf_tgt.max_subsystems = spdk_strtol(optarg, 10);
+ if (g_nvmf_tgt.max_subsystems < 0) {
+ fprintf(stderr, "converting a string to integer failed\n");
+ return -EINVAL;
+ }
+ break;
+ case 'p':
+ value = spdk_strtol(optarg, 10);
+ if (value < 0) {
+ fprintf(stderr, "converting a string to integer failed\n");
+ return -EINVAL;
+ }
+ g_acceptor_poll_rate = value;
+ break;
+ case 'r':
+ g_rpc_addr = optarg;
+ break;
+ case 's':
+ value = spdk_strtol(optarg, 10);
+ if (value < 0) {
+ fprintf(stderr, "converting a string to integer failed\n");
+ return -EINVAL;
+ }
+ opts->mem_size = value;
+ break;
+ case 'u':
+ opts->no_pci = true;
+ break;
+ case 'h':
+ default:
+ usage(argv[0]);
+ return 1;
+ }
+ }
+
+ return 0;
+}
+
+static int
+nvmf_reactor_run(void *arg)
+{
+ struct nvmf_reactor *nvmf_reactor = arg;
+ struct nvmf_lw_thread *lw_thread;
+ struct spdk_thread *thread;
+
+ /* run all the lightweight threads in this nvmf_reactor by FIFO. */
+ do {
+ if (spdk_ring_dequeue(nvmf_reactor->threads, (void **)&lw_thread, 1)) {
+ thread = spdk_thread_get_from_ctx(lw_thread);
+
+ spdk_thread_poll(thread, 0, 0);
+
+ if (spdk_unlikely(spdk_thread_is_exited(thread) &&
+ spdk_thread_is_idle(thread))) {
+ spdk_thread_destroy(thread);
+ } else if (spdk_unlikely(lw_thread->resched)) {
+ lw_thread->resched = false;
+ nvmf_schedule_spdk_thread(thread);
+ } else {
+ spdk_ring_enqueue(nvmf_reactor->threads, (void **)&lw_thread, 1, NULL);
+ }
+ }
+ } while (!g_reactors_exit);
+
+ /* free all the lightweight threads */
+ while (spdk_ring_dequeue(nvmf_reactor->threads, (void **)&lw_thread, 1)) {
+ thread = spdk_thread_get_from_ctx(lw_thread);
+ spdk_set_thread(thread);
+
+ if (spdk_thread_is_exited(thread)) {
+ spdk_thread_destroy(thread);
+ } else {
+ /* This thread is not exited yet, and may need to communicate with other threads
+ * to be exited. So mark it as exiting, and check again after traversing other threads.
+ */
+ spdk_thread_exit(thread);
+ spdk_thread_poll(thread, 0, 0);
+ spdk_ring_enqueue(nvmf_reactor->threads, (void **)&lw_thread, 1, NULL);
+ }
+ }
+
+ return 0;
+}
+
+static int
+nvmf_schedule_spdk_thread(struct spdk_thread *thread)
+{
+ struct nvmf_reactor *nvmf_reactor;
+ struct nvmf_lw_thread *lw_thread;
+ struct spdk_cpuset *cpumask;
+ uint32_t i;
+
+ /* Lightweight threads may have a requested cpumask.
+ * This is a request only - the scheduler does not have to honor it.
+ * For this scheduler implementation, each reactor is pinned to
+ * a particular core so honoring the request is reasonably easy.
+ */
+ cpumask = spdk_thread_get_cpumask(thread);
+
+ lw_thread = spdk_thread_get_ctx(thread);
+ assert(lw_thread != NULL);
+ memset(lw_thread, 0, sizeof(*lw_thread));
+
+ /* assign lightweight threads to nvmf reactor(core)
+ * Here we use the mutex.The way the actual SPDK event framework
+ * solves this is by using internal rings for messages between reactors
+ */
+ pthread_mutex_lock(&g_mutex);
+ for (i = 0; i < spdk_env_get_core_count(); i++) {
+ if (g_next_reactor == NULL) {
+ g_next_reactor = TAILQ_FIRST(&g_reactors);
+ }
+ nvmf_reactor = g_next_reactor;
+ g_next_reactor = TAILQ_NEXT(g_next_reactor, link);
+
+ /* each spdk_thread has the core affinity */
+ if (spdk_cpuset_get_cpu(cpumask, nvmf_reactor->core)) {
+ spdk_ring_enqueue(nvmf_reactor->threads, (void **)&lw_thread, 1, NULL);
+ break;
+ }
+ }
+ pthread_mutex_unlock(&g_mutex);
+
+ if (i == spdk_env_get_core_count()) {
+ fprintf(stderr, "failed to schedule spdk thread\n");
+ return -1;
+ }
+ return 0;
+}
+
+static void
+nvmf_request_spdk_thread_reschedule(struct spdk_thread *thread)
+{
+ struct nvmf_lw_thread *lw_thread;
+
+ assert(thread == spdk_get_thread());
+
+ lw_thread = spdk_thread_get_ctx(thread);
+
+ assert(lw_thread != NULL);
+
+ lw_thread->resched = true;
+}
+
+static int
+nvmf_reactor_thread_op(struct spdk_thread *thread, enum spdk_thread_op op)
+{
+ switch (op) {
+ case SPDK_THREAD_OP_NEW:
+ return nvmf_schedule_spdk_thread(thread);
+ case SPDK_THREAD_OP_RESCHED:
+ nvmf_request_spdk_thread_reschedule(thread);
+ return 0;
+ default:
+ return -ENOTSUP;
+ }
+}
+
+static bool
+nvmf_reactor_thread_op_supported(enum spdk_thread_op op)
+{
+ switch (op) {
+ case SPDK_THREAD_OP_NEW:
+ case SPDK_THREAD_OP_RESCHED:
+ return true;
+ default:
+ return false;
+ }
+}
+
+static int
+nvmf_init_threads(void)
+{
+ int rc;
+ uint32_t i;
+ char thread_name[32];
+ struct nvmf_reactor *nvmf_reactor;
+ struct spdk_cpuset cpumask;
+ uint32_t master_core = spdk_env_get_current_core();
+
+ /* Whenever SPDK creates a new lightweight thread it will call
+ * nvmf_schedule_spdk_thread asking for the application to begin
+ * polling it via spdk_thread_poll(). Each lightweight thread in
+ * SPDK optionally allocates extra memory to be used by the application
+ * framework. The size of the extra memory allocated is the second parameter.
+ */
+ spdk_thread_lib_init_ext(nvmf_reactor_thread_op, nvmf_reactor_thread_op_supported,
+ sizeof(struct nvmf_lw_thread));
+
+ /* Spawn one system thread per CPU core. The system thread is called a reactor.
+ * SPDK will spawn lightweight threads that must be mapped to reactors in
+ * nvmf_schedule_spdk_thread. Using a single system thread per CPU core is a
+ * choice unique to this application. SPDK itself does not require this specific
+ * threading model. For example, another viable threading model would be
+ * dynamically scheduling the lightweight threads onto a thread pool using a
+ * work queue.
+ */
+ SPDK_ENV_FOREACH_CORE(i) {
+ nvmf_reactor = calloc(1, sizeof(struct nvmf_reactor));
+ if (!nvmf_reactor) {
+ fprintf(stderr, "failed to alloc nvmf reactor\n");
+ rc = -ENOMEM;
+ goto err_exit;
+ }
+
+ nvmf_reactor->core = i;
+
+ nvmf_reactor->threads = spdk_ring_create(SPDK_RING_TYPE_MP_SC, 1024, SPDK_ENV_SOCKET_ID_ANY);
+ if (!nvmf_reactor->threads) {
+ fprintf(stderr, "failed to alloc ring\n");
+ free(nvmf_reactor);
+ rc = -ENOMEM;
+ goto err_exit;
+ }
+
+ TAILQ_INSERT_TAIL(&g_reactors, nvmf_reactor, link);
+
+ if (i == master_core) {
+ g_master_reactor = nvmf_reactor;
+ g_next_reactor = g_master_reactor;
+ } else {
+ rc = spdk_env_thread_launch_pinned(i,
+ nvmf_reactor_run,
+ nvmf_reactor);
+ if (rc) {
+ fprintf(stderr, "failed to pin reactor launch\n");
+ goto err_exit;
+ }
+ }
+ }
+
+ /* Spawn a lightweight thread only on the current core to manage this application. */
+ spdk_cpuset_zero(&cpumask);
+ spdk_cpuset_set_cpu(&cpumask, master_core, true);
+ snprintf(thread_name, sizeof(thread_name), "nvmf_master_thread");
+ g_init_thread = spdk_thread_create(thread_name, &cpumask);
+ if (!g_init_thread) {
+ fprintf(stderr, "failed to create spdk thread\n");
+ return -1;
+ }
+
+ fprintf(stdout, "nvmf threads initlize successfully\n");
+ return 0;
+
+err_exit:
+ return rc;
+}
+
+static void
+nvmf_destroy_threads(void)
+{
+ struct nvmf_reactor *nvmf_reactor, *tmp;
+
+ TAILQ_FOREACH_SAFE(nvmf_reactor, &g_reactors, link, tmp) {
+ spdk_ring_free(nvmf_reactor->threads);
+ free(nvmf_reactor);
+ }
+
+ pthread_mutex_destroy(&g_mutex);
+ spdk_thread_lib_fini();
+ fprintf(stdout, "nvmf threads destroy successfully\n");
+}
+
+static void
+nvmf_tgt_destroy_done(void *ctx, int status)
+{
+ fprintf(stdout, "destroyed the nvmf target service\n");
+
+ g_target_state = NVMF_FINI_SUBSYSTEM;
+ nvmf_target_advance_state();
+}
+
+static void
+nvmf_destroy_nvmf_tgt(void)
+{
+ if (g_nvmf_tgt.tgt) {
+ spdk_nvmf_tgt_destroy(g_nvmf_tgt.tgt, nvmf_tgt_destroy_done, NULL);
+ } else {
+ g_target_state = NVMF_FINI_SUBSYSTEM;
+ }
+}
+
+static void
+nvmf_create_nvmf_tgt(void)
+{
+ struct spdk_nvmf_subsystem *subsystem;
+ struct spdk_nvmf_target_opts tgt_opts;
+
+ tgt_opts.max_subsystems = g_nvmf_tgt.max_subsystems;
+ snprintf(tgt_opts.name, sizeof(tgt_opts.name), "%s", "nvmf_example");
+ /* Construct the default NVMe-oF target
+ * An NVMe-oF target is a collection of subsystems, namespace, and poll
+ * groups, and defines the scope of the NVMe-oF discovery service.
+ */
+ g_nvmf_tgt.tgt = spdk_nvmf_tgt_create(&tgt_opts);
+ if (g_nvmf_tgt.tgt == NULL) {
+ fprintf(stderr, "spdk_nvmf_tgt_create() failed\n");
+ goto error;
+ }
+
+ /* Create and add discovery subsystem to the NVMe-oF target.
+ * NVMe-oF defines a discovery mechanism that a host uses to determine
+ * the NVM subsystems that expose namespaces that the host may access.
+ * It provides a host with following capabilities:
+ * 1,The ability to discover a list of NVM subsystems with namespaces
+ * that are accessible to the host.
+ * 2,The ability to discover multiple paths to an NVM subsystem.
+ * 3,The ability to discover controllers that are statically configured.
+ */
+ subsystem = spdk_nvmf_subsystem_create(g_nvmf_tgt.tgt, SPDK_NVMF_DISCOVERY_NQN,
+ SPDK_NVMF_SUBTYPE_DISCOVERY, 0);
+ if (subsystem == NULL) {
+ fprintf(stderr, "failed to create discovery nvmf library subsystem\n");
+ goto error;
+ }
+
+ /* Allow any host to access the discovery subsystem */
+ spdk_nvmf_subsystem_set_allow_any_host(subsystem, true);
+
+ fprintf(stdout, "created a nvmf target service\n");
+
+ g_target_state = NVMF_INIT_POLL_GROUPS;
+ return;
+
+error:
+ g_target_state = NVMF_FINI_TARGET;
+}
+
+static void
+nvmf_tgt_subsystem_stop_next(struct spdk_nvmf_subsystem *subsystem,
+ void *cb_arg, int status)
+{
+ subsystem = spdk_nvmf_subsystem_get_next(subsystem);
+ if (subsystem) {
+ spdk_nvmf_subsystem_stop(subsystem,
+ nvmf_tgt_subsystem_stop_next,
+ cb_arg);
+ return;
+ }
+
+ fprintf(stdout, "all subsystems of target stopped\n");
+
+ g_target_state = NVMF_FINI_POLL_GROUPS;
+ nvmf_target_advance_state();
+}
+
+static void
+nvmf_tgt_stop_subsystems(struct nvmf_target *nvmf_tgt)
+{
+ struct spdk_nvmf_subsystem *subsystem;
+
+ subsystem = spdk_nvmf_subsystem_get_first(nvmf_tgt->tgt);
+ if (spdk_likely(subsystem)) {
+ spdk_nvmf_subsystem_stop(subsystem,
+ nvmf_tgt_subsystem_stop_next,
+ NULL);
+ } else {
+ g_target_state = NVMF_FINI_POLL_GROUPS;
+ }
+}
+
+static int
+nvmf_tgt_acceptor_poll(void *arg)
+{
+ struct nvmf_target *nvmf_tgt = arg;
+
+ spdk_nvmf_tgt_accept(nvmf_tgt->tgt);
+
+ return -1;
+}
+
+static void
+nvmf_tgt_subsystem_start_next(struct spdk_nvmf_subsystem *subsystem,
+ void *cb_arg, int status)
+{
+ subsystem = spdk_nvmf_subsystem_get_next(subsystem);
+ if (subsystem) {
+ spdk_nvmf_subsystem_start(subsystem, nvmf_tgt_subsystem_start_next,
+ cb_arg);
+ return;
+ }
+
+ fprintf(stdout, "all subsystems of target started\n");
+
+ g_target_state = NVMF_INIT_START_ACCEPTOR;
+ nvmf_target_advance_state();
+}
+
+static void
+nvmf_tgt_start_subsystems(struct nvmf_target *nvmf_tgt)
+{
+ struct spdk_nvmf_subsystem *subsystem;
+
+ /* Subsystem is the NVM subsystem which is a combine of namespaces
+ * except the discovery subsystem which is used for discovery service.
+ * It also controls the hosts that means the subsystem determines whether
+ * the host can access this subsystem.
+ */
+ subsystem = spdk_nvmf_subsystem_get_first(nvmf_tgt->tgt);
+ if (spdk_likely(subsystem)) {
+ /* In SPDK there are three states in subsystem: Inactive, Active, Paused.
+ * Start subsystem means make it from inactive to active that means
+ * subsystem start to work or it can be accessed.
+ */
+ spdk_nvmf_subsystem_start(subsystem,
+ nvmf_tgt_subsystem_start_next,
+ NULL);
+ } else {
+ g_target_state = NVMF_INIT_START_ACCEPTOR;
+ }
+}
+
+static void
+nvmf_tgt_create_poll_groups_done(void *ctx)
+{
+ struct nvmf_target_poll_group *pg = ctx;
+
+ if (!g_next_pg) {
+ g_next_pg = pg;
+ }
+
+ TAILQ_INSERT_TAIL(&g_poll_groups, pg, link);
+
+ assert(g_num_poll_groups < spdk_env_get_core_count());
+
+ if (++g_num_poll_groups == spdk_env_get_core_count()) {
+ fprintf(stdout, "create targets's poll groups done\n");
+
+ g_target_state = NVMF_INIT_START_SUBSYSTEMS;
+ nvmf_target_advance_state();
+ }
+}
+
+static void
+nvmf_tgt_create_poll_group(void *ctx)
+{
+ struct nvmf_target_poll_group *pg;
+
+ pg = calloc(1, sizeof(struct nvmf_target_poll_group));
+ if (!pg) {
+ fprintf(stderr, "failed to allocate poll group\n");
+ assert(false);
+ return;
+ }
+
+ pg->thread = spdk_get_thread();
+ pg->group = spdk_nvmf_poll_group_create(g_nvmf_tgt.tgt);
+ if (!pg->group) {
+ fprintf(stderr, "failed to create poll group of the target\n");
+ free(pg);
+ assert(false);
+ return;
+ }
+
+ spdk_thread_send_msg(g_init_thread, nvmf_tgt_create_poll_groups_done, pg);
+}
+
+/* Create a lightweight thread per poll group instead of assuming a pool of lightweight
+ * threads already exist at start up time. A poll group is a collection of unrelated NVMe-oF
+ * connections. Each poll group is only accessed from the associated lightweight thread.
+ */
+static void
+nvmf_poll_groups_create(void)
+{
+ struct spdk_cpuset tmp_cpumask = {};
+ uint32_t i;
+ char thread_name[32];
+ struct spdk_thread *thread;
+
+ assert(g_init_thread != NULL);
+
+ SPDK_ENV_FOREACH_CORE(i) {
+ spdk_cpuset_zero(&tmp_cpumask);
+ spdk_cpuset_set_cpu(&tmp_cpumask, i, true);
+ snprintf(thread_name, sizeof(thread_name), "nvmf_tgt_poll_group_%u", i);
+
+ thread = spdk_thread_create(thread_name, &tmp_cpumask);
+ assert(thread != NULL);
+
+ spdk_thread_send_msg(thread, nvmf_tgt_create_poll_group, NULL);
+ }
+}
+
+static void
+_nvmf_tgt_destroy_poll_groups_done(void *ctx)
+{
+ assert(g_num_poll_groups > 0);
+
+ if (--g_num_poll_groups == 0) {
+ fprintf(stdout, "destroy targets's poll groups done\n");
+
+ g_target_state = NVMF_FINI_STOP_ACCEPTOR;
+ nvmf_target_advance_state();
+ }
+}
+
+static void
+nvmf_tgt_destroy_poll_groups_done(void *cb_arg, int status)
+{
+ struct nvmf_target_poll_group *pg = cb_arg;
+
+ free(pg);
+
+ spdk_thread_send_msg(g_fini_thread, _nvmf_tgt_destroy_poll_groups_done, NULL);
+
+ spdk_thread_exit(spdk_get_thread());
+}
+
+static void
+nvmf_tgt_destroy_poll_group(void *ctx)
+{
+ struct nvmf_target_poll_group *pg = ctx;
+
+ spdk_nvmf_poll_group_destroy(pg->group, nvmf_tgt_destroy_poll_groups_done, pg);
+}
+
+static void
+nvmf_poll_groups_destroy(void)
+{
+ struct nvmf_target_poll_group *pg, *tmp;
+
+ g_fini_thread = spdk_get_thread();
+ assert(g_fini_thread != NULL);
+
+ TAILQ_FOREACH_SAFE(pg, &g_poll_groups, link, tmp) {
+ TAILQ_REMOVE(&g_poll_groups, pg, link);
+ spdk_thread_send_msg(pg->thread, nvmf_tgt_destroy_poll_group, pg);
+ }
+}
+
+static void
+nvmf_subsystem_fini_done(void *cb_arg)
+{
+ fprintf(stdout, "bdev subsystem finish successfully\n");
+ spdk_rpc_finish();
+ g_reactors_exit = true;
+}
+
+static void
+nvmf_subsystem_init_done(int rc, void *cb_arg)
+{
+ fprintf(stdout, "bdev subsystem init successfully\n");
+ spdk_rpc_initialize(g_rpc_addr);
+ spdk_rpc_set_state(SPDK_RPC_RUNTIME);
+
+ g_target_state = NVMF_INIT_TARGET;
+ nvmf_target_advance_state();
+}
+
+static void
+migrate_poll_group_by_rr(void *ctx)
+{
+ uint32_t current_core, next_core;
+ struct spdk_cpuset cpumask = {};
+
+ current_core = spdk_env_get_current_core();
+ next_core = spdk_env_get_next_core(current_core);
+ if (next_core == UINT32_MAX) {
+ next_core = spdk_env_get_first_core();
+ }
+
+ spdk_cpuset_set_cpu(&cpumask, next_core, true);
+
+ spdk_thread_set_cpumask(&cpumask);
+}
+
+static int
+migrate_poll_groups_by_rr(void *ctx)
+{
+ struct nvmf_target_poll_group *pg;
+
+ TAILQ_FOREACH(pg, &g_poll_groups, link) {
+ spdk_thread_send_msg(pg->thread, migrate_poll_group_by_rr, NULL);
+ }
+
+ return 1;
+}
+
+static void
+nvmf_target_advance_state(void)
+{
+ enum nvmf_target_state prev_state;
+
+ do {
+ prev_state = g_target_state;
+
+ switch (g_target_state) {
+ case NVMF_INIT_SUBSYSTEM:
+ /* initlize the bdev layer */
+ spdk_subsystem_init(nvmf_subsystem_init_done, NULL);
+ return;
+ case NVMF_INIT_TARGET:
+ nvmf_create_nvmf_tgt();
+ break;
+ case NVMF_INIT_POLL_GROUPS:
+ nvmf_poll_groups_create();
+ break;
+ case NVMF_INIT_START_SUBSYSTEMS:
+ nvmf_tgt_start_subsystems(&g_nvmf_tgt);
+ break;
+ case NVMF_INIT_START_ACCEPTOR:
+ g_acceptor_poller = SPDK_POLLER_REGISTER(nvmf_tgt_acceptor_poll, &g_nvmf_tgt,
+ g_acceptor_poll_rate);
+ fprintf(stdout, "Acceptor running\n");
+ g_target_state = NVMF_RUNNING;
+ break;
+ case NVMF_RUNNING:
+ fprintf(stdout, "nvmf target is running\n");
+ if (g_migrate_pg_period_us != 0) {
+ g_migrate_pg_poller = SPDK_POLLER_REGISTER(migrate_poll_groups_by_rr, NULL,
+ g_migrate_pg_period_us);
+ }
+ break;
+ case NVMF_FINI_STOP_SUBSYSTEMS:
+ spdk_poller_unregister(&g_migrate_pg_poller);
+ nvmf_tgt_stop_subsystems(&g_nvmf_tgt);
+ break;
+ case NVMF_FINI_POLL_GROUPS:
+ nvmf_poll_groups_destroy();
+ break;
+ case NVMF_FINI_STOP_ACCEPTOR:
+ spdk_poller_unregister(&g_acceptor_poller);
+ g_target_state = NVMF_FINI_TARGET;
+ break;
+ case NVMF_FINI_TARGET:
+ nvmf_destroy_nvmf_tgt();
+ break;
+ case NVMF_FINI_SUBSYSTEM:
+ spdk_subsystem_fini(nvmf_subsystem_fini_done, NULL);
+ break;
+ }
+ } while (g_target_state != prev_state);
+}
+
+static void
+nvmf_target_app_start(void *arg)
+{
+ g_target_state = NVMF_INIT_SUBSYSTEM;
+ nvmf_target_advance_state();
+}
+
+static void
+_nvmf_shutdown_cb(void *ctx)
+{
+ /* Still in initialization state, defer shutdown operation */
+ if (g_target_state < NVMF_RUNNING) {
+ spdk_thread_send_msg(spdk_get_thread(), _nvmf_shutdown_cb, NULL);
+ return;
+ } else if (g_target_state > NVMF_RUNNING) {
+ /* Already in Shutdown status, ignore the signal */
+ return;
+ }
+
+ g_target_state = NVMF_FINI_STOP_SUBSYSTEMS;
+ nvmf_target_advance_state();
+}
+
+static void
+nvmf_shutdown_cb(int signo)
+{
+ if (!g_intr_received) {
+ g_intr_received = true;
+ spdk_thread_send_msg(g_init_thread, _nvmf_shutdown_cb, NULL);
+ }
+}
+
+static int
+nvmf_setup_signal_handlers(void)
+{
+ struct sigaction sigact;
+ sigset_t sigmask;
+ int signals[] = {SIGINT, SIGTERM};
+ int num_signals = sizeof(signals) / sizeof(int);
+ int rc, i;
+
+ rc = sigemptyset(&sigmask);
+ if (rc) {
+ fprintf(stderr, "errno:%d--failed to empty signal set\n", errno);
+ return rc;
+ }
+ memset(&sigact, 0, sizeof(sigact));
+ rc = sigemptyset(&sigact.sa_mask);
+ if (rc) {
+ fprintf(stderr, "errno:%d--failed to empty signal set\n", errno);
+ return rc;
+ }
+
+ /* Install the same handler for SIGINT and SIGTERM */
+ sigact.sa_handler = nvmf_shutdown_cb;
+
+ for (i = 0; i < num_signals; i++) {
+ rc = sigaction(signals[i], &sigact, NULL);
+ if (rc < 0) {
+ fprintf(stderr, "errno:%d--sigaction() failed\n", errno);
+ return rc;
+ }
+ rc = sigaddset(&sigmask, signals[i]);
+ if (rc) {
+ fprintf(stderr, "errno:%d--failed to add set\n", errno);
+ return rc;
+ }
+ }
+
+ pthread_sigmask(SIG_UNBLOCK, &sigmask, NULL);
+
+ return 0;
+}
+
+int main(int argc, char **argv)
+{
+ int rc;
+ struct spdk_env_opts opts;
+
+ spdk_env_opts_init(&opts);
+ opts.name = "nvmf-example";
+
+ rc = parse_args(argc, argv, &opts);
+ if (rc != 0) {
+ return rc;
+ }
+
+ if (spdk_env_init(&opts) < 0) {
+ fprintf(stderr, "unable to initialize SPDK env\n");
+ return -EINVAL;
+ }
+
+ /* Initialize the threads */
+ rc = nvmf_init_threads();
+ assert(rc == 0);
+
+ /* Send a message to the thread assigned to the master reactor
+ * that continues initialization. This is how we bootstrap the
+ * program so that all code from here on is running on an SPDK thread.
+ */
+ assert(g_init_thread != NULL);
+
+ rc = nvmf_setup_signal_handlers();
+ assert(rc == 0);
+
+ spdk_thread_send_msg(g_init_thread, nvmf_target_app_start, NULL);
+
+ nvmf_reactor_run(g_master_reactor);
+
+ spdk_env_thread_wait_all();
+ nvmf_destroy_threads();
+ return rc;
+}
diff --git a/src/spdk/examples/sock/Makefile b/src/spdk/examples/sock/Makefile
new file mode 100644
index 000000000..097061fd1
--- /dev/null
+++ b/src/spdk/examples/sock/Makefile
@@ -0,0 +1,47 @@
+#
+# BSD LICENSE
+#
+# Copyright (c) Intel Corporation.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Intel Corporation nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+DIRS-y += hello_world
+
+.PHONY: all clean $(DIRS-y)
+
+all: $(DIRS-y)
+ @:
+
+clean: $(DIRS-y)
+ @:
+
+include $(SPDK_ROOT_DIR)/mk/spdk.subdirs.mk
diff --git a/src/spdk/examples/sock/hello_world/.gitignore b/src/spdk/examples/sock/hello_world/.gitignore
new file mode 100644
index 000000000..95ffb143c
--- /dev/null
+++ b/src/spdk/examples/sock/hello_world/.gitignore
@@ -0,0 +1 @@
+hello_sock
diff --git a/src/spdk/examples/sock/hello_world/Makefile b/src/spdk/examples/sock/hello_world/Makefile
new file mode 100644
index 000000000..e326bf26b
--- /dev/null
+++ b/src/spdk/examples/sock/hello_world/Makefile
@@ -0,0 +1,43 @@
+#
+# Copyright (c) Intel Corporation.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Intel Corporation nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+include $(SPDK_ROOT_DIR)/mk/spdk.modules.mk
+
+APP = hello_sock
+
+C_SRCS := hello_sock.c
+
+SPDK_LIB_LIST = $(SOCK_MODULES_LIST)
+SPDK_LIB_LIST += event_net net event thread util conf trace log jsonrpc json rpc sock notify
+
+include $(SPDK_ROOT_DIR)/mk/spdk.app.mk
diff --git a/src/spdk/examples/sock/hello_world/hello_sock.c b/src/spdk/examples/sock/hello_world/hello_sock.c
new file mode 100644
index 000000000..75ae4e631
--- /dev/null
+++ b/src/spdk/examples/sock/hello_world/hello_sock.c
@@ -0,0 +1,442 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+#include "spdk/thread.h"
+#include "spdk/env.h"
+#include "spdk/event.h"
+#include "spdk/log.h"
+#include "spdk/string.h"
+
+#include "spdk/sock.h"
+#include "spdk/net.h"
+
+#define ACCEPT_TIMEOUT_US 1000
+#define CLOSE_TIMEOUT_US 1000000
+#define BUFFER_SIZE 1024
+#define ADDR_STR_LEN INET6_ADDRSTRLEN
+
+static bool g_is_running;
+
+static char *g_host;
+static char *g_sock_impl_name;
+static int g_port;
+static bool g_is_server;
+static bool g_verbose;
+
+/*
+ * We'll use this struct to gather housekeeping hello_context to pass between
+ * our events and callbacks.
+ */
+struct hello_context_t {
+ bool is_server;
+ char *host;
+ char *sock_impl_name;
+ int port;
+
+ bool verbose;
+ int bytes_in;
+ int bytes_out;
+
+ struct spdk_sock *sock;
+
+ struct spdk_sock_group *group;
+ struct spdk_poller *poller_in;
+ struct spdk_poller *poller_out;
+ struct spdk_poller *time_out;
+
+ int rc;
+};
+
+/*
+ * Usage function for printing parameters that are specific to this application
+ */
+static void
+hello_sock_usage(void)
+{
+ printf(" -H host_addr host address\n");
+ printf(" -P port port number\n");
+ printf(" -N sock_impl socket implementation, e.g., -N posix or -N vpp\n");
+ printf(" -S start in server mode\n");
+ printf(" -V print out additional informations");
+}
+
+/*
+ * This function is called to parse the parameters that are specific to this application
+ */
+static int hello_sock_parse_arg(int ch, char *arg)
+{
+ switch (ch) {
+ case 'H':
+ g_host = arg;
+ break;
+ case 'N':
+ g_sock_impl_name = arg;
+ break;
+ case 'P':
+ g_port = spdk_strtol(arg, 10);
+ if (g_port < 0) {
+ fprintf(stderr, "Invalid port ID\n");
+ return g_port;
+ }
+ break;
+ case 'S':
+ g_is_server = 1;
+ break;
+ case 'V':
+ g_verbose = true;
+ break;
+ default:
+ return -EINVAL;
+ }
+ return 0;
+}
+
+static int
+hello_sock_close_timeout_poll(void *arg)
+{
+ struct hello_context_t *ctx = arg;
+ SPDK_NOTICELOG("Connection closed\n");
+
+ spdk_poller_unregister(&ctx->time_out);
+ spdk_poller_unregister(&ctx->poller_in);
+ spdk_sock_close(&ctx->sock);
+ spdk_sock_group_close(&ctx->group);
+
+ spdk_app_stop(ctx->rc);
+ return 0;
+}
+
+static int
+hello_sock_quit(struct hello_context_t *ctx, int rc)
+{
+ ctx->rc = rc;
+ spdk_poller_unregister(&ctx->poller_out);
+ if (!ctx->time_out) {
+ ctx->time_out = SPDK_POLLER_REGISTER(hello_sock_close_timeout_poll, ctx,
+ CLOSE_TIMEOUT_US);
+ }
+ return 0;
+}
+
+static int
+hello_sock_recv_poll(void *arg)
+{
+ struct hello_context_t *ctx = arg;
+ int rc;
+ char buf_in[BUFFER_SIZE];
+
+ /*
+ * Get response
+ */
+ rc = spdk_sock_recv(ctx->sock, buf_in, sizeof(buf_in) - 1);
+
+ if (rc <= 0) {
+ if (errno == EAGAIN || errno == EWOULDBLOCK) {
+ return 0;
+ }
+
+ SPDK_ERRLOG("spdk_sock_recv() failed, errno %d: %s\n",
+ errno, spdk_strerror(errno));
+ return -1;
+ }
+
+ if (rc > 0) {
+ ctx->bytes_in += rc;
+ buf_in[rc] = '\0';
+ printf("%s", buf_in);
+ }
+
+ return 0;
+}
+
+static int
+hello_sock_writev_poll(void *arg)
+{
+ struct hello_context_t *ctx = arg;
+ int rc = 0;
+ char buf_out[BUFFER_SIZE];
+ struct iovec iov;
+ ssize_t n;
+
+ n = read(STDIN_FILENO, buf_out, sizeof(buf_out));
+ if (n == 0 || !g_is_running) {
+ /* EOF */
+ SPDK_NOTICELOG("Closing connection...\n");
+ hello_sock_quit(ctx, 0);
+ return 0;
+ }
+ if (n > 0) {
+ /*
+ * Send message to the server
+ */
+ iov.iov_base = buf_out;
+ iov.iov_len = n;
+ rc = spdk_sock_writev(ctx->sock, &iov, 1);
+ if (rc > 0) {
+ ctx->bytes_out += rc;
+ }
+ }
+ return rc;
+}
+
+static int
+hello_sock_connect(struct hello_context_t *ctx)
+{
+ int rc;
+ char saddr[ADDR_STR_LEN], caddr[ADDR_STR_LEN];
+ uint16_t cport, sport;
+
+ SPDK_NOTICELOG("Connecting to the server on %s:%d with sock_impl(%s)\n", ctx->host, ctx->port,
+ ctx->sock_impl_name);
+
+ ctx->sock = spdk_sock_connect(ctx->host, ctx->port, ctx->sock_impl_name);
+ if (ctx->sock == NULL) {
+ SPDK_ERRLOG("connect error(%d): %s\n", errno, spdk_strerror(errno));
+ return -1;
+ }
+
+ rc = spdk_sock_getaddr(ctx->sock, saddr, sizeof(saddr), &sport, caddr, sizeof(caddr), &cport);
+ if (rc < 0) {
+ SPDK_ERRLOG("Cannot get connection addresses\n");
+ spdk_sock_close(&ctx->sock);
+ return -1;
+ }
+
+ SPDK_NOTICELOG("Connection accepted from (%s, %hu) to (%s, %hu)\n", caddr, cport, saddr, sport);
+
+ fcntl(STDIN_FILENO, F_SETFL, fcntl(STDIN_FILENO, F_GETFL) | O_NONBLOCK);
+
+ g_is_running = true;
+ ctx->poller_in = SPDK_POLLER_REGISTER(hello_sock_recv_poll, ctx, 0);
+ ctx->poller_out = SPDK_POLLER_REGISTER(hello_sock_writev_poll, ctx, 0);
+
+ return 0;
+}
+
+static void
+hello_sock_cb(void *arg, struct spdk_sock_group *group, struct spdk_sock *sock)
+{
+ ssize_t n;
+ char buf[BUFFER_SIZE];
+ struct iovec iov;
+ struct hello_context_t *ctx = arg;
+
+ n = spdk_sock_recv(sock, buf, sizeof(buf));
+ if (n < 0) {
+ if (errno == EAGAIN || errno == EWOULDBLOCK) {
+ SPDK_ERRLOG("spdk_sock_recv() failed, errno %d: %s\n",
+ errno, spdk_strerror(errno));
+ return;
+ }
+
+ SPDK_ERRLOG("spdk_sock_recv() failed, errno %d: %s\n",
+ errno, spdk_strerror(errno));
+ }
+
+ if (n > 0) {
+ ctx->bytes_in += n;
+ iov.iov_base = buf;
+ iov.iov_len = n;
+ n = spdk_sock_writev(sock, &iov, 1);
+ if (n > 0) {
+ ctx->bytes_out += n;
+ }
+ return;
+ }
+
+ /* Connection closed */
+ SPDK_NOTICELOG("Connection closed\n");
+ spdk_sock_group_remove_sock(group, sock);
+ spdk_sock_close(&sock);
+}
+
+static int
+hello_sock_accept_poll(void *arg)
+{
+ struct hello_context_t *ctx = arg;
+ struct spdk_sock *sock;
+ int rc;
+ int count = 0;
+ char saddr[ADDR_STR_LEN], caddr[ADDR_STR_LEN];
+ uint16_t cport, sport;
+
+ if (!g_is_running) {
+ hello_sock_quit(ctx, 0);
+ return 0;
+ }
+
+ while (1) {
+ sock = spdk_sock_accept(ctx->sock);
+ if (sock != NULL) {
+ rc = spdk_sock_getaddr(sock, saddr, sizeof(saddr), &sport, caddr, sizeof(caddr), &cport);
+ if (rc < 0) {
+ SPDK_ERRLOG("Cannot get connection addresses\n");
+ spdk_sock_close(&ctx->sock);
+ return -1;
+ }
+
+ SPDK_NOTICELOG("Accepting a new connection from (%s, %hu) to (%s, %hu)\n",
+ caddr, cport, saddr, sport);
+
+ rc = spdk_sock_group_add_sock(ctx->group, sock,
+ hello_sock_cb, ctx);
+
+ if (rc < 0) {
+ spdk_sock_close(&sock);
+ SPDK_ERRLOG("failed\n");
+ break;
+ }
+
+ count++;
+ } else {
+ if (errno != EAGAIN && errno != EWOULDBLOCK) {
+ SPDK_ERRLOG("accept error(%d): %s\n", errno, spdk_strerror(errno));
+ }
+ break;
+ }
+ }
+
+ return count;
+}
+
+static int
+hello_sock_group_poll(void *arg)
+{
+ struct hello_context_t *ctx = arg;
+ int rc;
+
+ rc = spdk_sock_group_poll(ctx->group);
+ if (rc < 0) {
+ SPDK_ERRLOG("Failed to poll sock_group=%p\n", ctx->group);
+ }
+
+ return -1;
+}
+
+static int
+hello_sock_listen(struct hello_context_t *ctx)
+{
+ ctx->sock = spdk_sock_listen(ctx->host, ctx->port, ctx->sock_impl_name);
+ if (ctx->sock == NULL) {
+ SPDK_ERRLOG("Cannot create server socket\n");
+ return -1;
+ }
+
+ SPDK_NOTICELOG("Listening connection on %s:%d with sock_impl(%s)\n", ctx->host, ctx->port,
+ ctx->sock_impl_name);
+
+ /*
+ * Create sock group for server socket
+ */
+ ctx->group = spdk_sock_group_create(NULL);
+
+ g_is_running = true;
+
+ /*
+ * Start acceptor and group poller
+ */
+ ctx->poller_in = SPDK_POLLER_REGISTER(hello_sock_accept_poll, ctx,
+ ACCEPT_TIMEOUT_US);
+ ctx->poller_out = SPDK_POLLER_REGISTER(hello_sock_group_poll, ctx, 0);
+
+ return 0;
+}
+
+static void
+hello_sock_shutdown_cb(void)
+{
+ g_is_running = false;
+}
+
+/*
+ * Our initial event that kicks off everything from main().
+ */
+static void
+hello_start(void *arg1)
+{
+ struct hello_context_t *ctx = arg1;
+ int rc;
+
+ SPDK_NOTICELOG("Successfully started the application\n");
+
+ if (ctx->is_server) {
+ rc = hello_sock_listen(ctx);
+ } else {
+ rc = hello_sock_connect(ctx);
+ }
+
+ if (rc) {
+ spdk_app_stop(-1);
+ return;
+ }
+}
+
+int
+main(int argc, char **argv)
+{
+ struct spdk_app_opts opts = {};
+ int rc = 0;
+ struct hello_context_t hello_context = {};
+
+ /* Set default values in opts structure. */
+ spdk_app_opts_init(&opts);
+ opts.name = "hello_sock";
+ opts.shutdown_cb = hello_sock_shutdown_cb;
+
+ if ((rc = spdk_app_parse_args(argc, argv, &opts, "H:N:P:SV", NULL, hello_sock_parse_arg,
+ hello_sock_usage)) != SPDK_APP_PARSE_ARGS_SUCCESS) {
+ exit(rc);
+ }
+ hello_context.is_server = g_is_server;
+ hello_context.host = g_host;
+ hello_context.sock_impl_name = g_sock_impl_name;
+ hello_context.port = g_port;
+ hello_context.verbose = g_verbose;
+
+ rc = spdk_app_start(&opts, hello_start, &hello_context);
+ if (rc) {
+ SPDK_ERRLOG("ERROR starting application\n");
+ }
+
+ SPDK_NOTICELOG("Exiting from application\n");
+
+ if (hello_context.verbose) {
+ printf("** %d bytes received, %d bytes sent **\n",
+ hello_context.bytes_in, hello_context.bytes_out);
+ }
+
+ /* Gracefully close out all of the SPDK subsystems. */
+ spdk_app_fini();
+ return rc;
+}
diff --git a/src/spdk/examples/vmd/Makefile b/src/spdk/examples/vmd/Makefile
new file mode 100644
index 000000000..ca8fb6980
--- /dev/null
+++ b/src/spdk/examples/vmd/Makefile
@@ -0,0 +1,44 @@
+#
+# BSD LICENSE
+#
+# Copyright (c) Intel Corporation.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Intel Corporation nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+DIRS-y += lsvmd led
+
+.PHONY: all clean $(DIRS-y)
+
+all: $(DIRS-y)
+clean: $(DIRS-y)
+
+include $(SPDK_ROOT_DIR)/mk/spdk.subdirs.mk
diff --git a/src/spdk/examples/vmd/led/.gitignore b/src/spdk/examples/vmd/led/.gitignore
new file mode 100644
index 000000000..727c8bd27
--- /dev/null
+++ b/src/spdk/examples/vmd/led/.gitignore
@@ -0,0 +1 @@
+led
diff --git a/src/spdk/examples/vmd/led/Makefile b/src/spdk/examples/vmd/led/Makefile
new file mode 100644
index 000000000..0fa807bbc
--- /dev/null
+++ b/src/spdk/examples/vmd/led/Makefile
@@ -0,0 +1,44 @@
+#
+# BSD LICENSE
+#
+# Copyright (c) Intel Corporation.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Intel Corporation nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+include $(SPDK_ROOT_DIR)/mk/spdk.modules.mk
+
+APP = led
+
+C_SRCS := led.c
+
+SPDK_LIB_LIST = vmd log
+
+include $(SPDK_ROOT_DIR)/mk/spdk.app.mk
diff --git a/src/spdk/examples/vmd/led/led.c b/src/spdk/examples/vmd/led/led.c
new file mode 100644
index 000000000..e014de704
--- /dev/null
+++ b/src/spdk/examples/vmd/led/led.c
@@ -0,0 +1,214 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+#include "spdk/log.h"
+#include "spdk/likely.h"
+#include "spdk/env.h"
+#include "spdk/vmd.h"
+
+enum app_action {
+ APP_ACTION_SET,
+ APP_ACTION_GET,
+ APP_ACTION_NOP,
+};
+
+struct app_opts {
+ const char *app_name;
+ struct spdk_pci_addr pci_addr;
+ bool all_devices;
+ enum app_action action;
+ enum spdk_vmd_led_state led_state;
+};
+
+struct app_opts g_opts = {
+ .all_devices = true,
+ .action = APP_ACTION_GET,
+ .led_state = SPDK_VMD_LED_STATE_UNKNOWN,
+};
+
+static const char *g_led_states[] = {
+ [SPDK_VMD_LED_STATE_OFF] = "off",
+ [SPDK_VMD_LED_STATE_IDENTIFY] = "identify",
+ [SPDK_VMD_LED_STATE_FAULT] = "fault",
+ [SPDK_VMD_LED_STATE_REBUILD] = "rebuild",
+ [SPDK_VMD_LED_STATE_UNKNOWN] = "unknown",
+};
+
+static void
+usage(void)
+{
+ printf("Usage: %s [-d] [-s STATE] [-r TRADDR]\n", g_opts.app_name);
+ printf("\n");
+ printf("Options:\n");
+ printf(" -d enables debug logs from the VMD module\n");
+ printf(" -s STATE sets the state of the LEDs. Available states are:\n");
+ printf(" off, identify, fault, rebuild\n");
+ printf(" -r TRADDR uses device identified by TRADDR\n");
+ printf(" -h shows this help\n");
+}
+
+static int
+parse_args(int argc, char **argv)
+{
+ int led_state;
+ int op;
+
+ g_opts.app_name = argv[0];
+
+ while ((op = getopt(argc, argv, "dhr:s:")) != -1) {
+ switch (op) {
+ case 'r':
+ if (spdk_pci_addr_parse(&g_opts.pci_addr, optarg)) {
+ fprintf(stderr, "Unable to parse PCI address: %s\n", optarg);
+ return -EINVAL;
+ }
+
+ g_opts.all_devices = false;
+ break;
+
+ case 'd':
+#ifdef DEBUG
+ spdk_log_set_print_level(SPDK_LOG_DEBUG);
+ spdk_log_set_flag("vmd");
+ break;
+#else
+ fprintf(stderr, "%s must be rebuilt with --enable-debug for the -d flag\n",
+ argv[0]);
+ return -EINVAL;
+#endif
+ case 's':
+ for (led_state = SPDK_VMD_LED_STATE_OFF;
+ led_state <= SPDK_VMD_LED_STATE_REBUILD;
+ led_state++) {
+ if (strcmp(optarg, g_led_states[led_state]) == 0) {
+ g_opts.led_state = (enum spdk_vmd_led_state)led_state;
+ break;
+ }
+ }
+
+ if (g_opts.led_state == SPDK_VMD_LED_STATE_UNKNOWN) {
+ fprintf(stderr, "Invalid LED state\n");
+ return -EINVAL;
+ }
+
+ g_opts.action = APP_ACTION_SET;
+ break;
+
+ case 'h':
+ g_opts.action = APP_ACTION_NOP;
+ usage();
+ break;
+
+ default:
+ return -EINVAL;
+ }
+ }
+
+ return 0;
+}
+
+int
+main(int argc, char **argv)
+{
+ struct spdk_env_opts opts;
+ struct spdk_pci_device *pci_device;
+ enum spdk_vmd_led_state led_state;
+ char addr_buf[128];
+ int rc, status = 0;
+
+ if (parse_args(argc, argv) != 0) {
+ usage();
+ return 1;
+ }
+
+ if (g_opts.action == APP_ACTION_NOP) {
+ return 0;
+ }
+
+ spdk_env_opts_init(&opts);
+ opts.name = "led";
+
+ if (spdk_env_init(&opts) < 0) {
+ fprintf(stderr, "Unable to initialize SPDK environment\n");
+ return 1;
+ }
+
+ rc = spdk_vmd_init();
+ if (rc) {
+ fprintf(stderr, "Unable to initialize VMD subsystem\n");
+ return 1;
+ }
+
+ for (pci_device = spdk_pci_get_first_device(); pci_device != NULL;
+ pci_device = spdk_pci_get_next_device(pci_device)) {
+ if (strcmp(spdk_pci_device_get_type(pci_device), "vmd") != 0) {
+ continue;
+ }
+
+ if (!g_opts.all_devices &&
+ spdk_pci_addr_compare(&g_opts.pci_addr, &pci_device->addr) != 0) {
+ continue;
+ }
+
+ rc = spdk_pci_addr_fmt(addr_buf, sizeof(addr_buf), &pci_device->addr);
+ if (rc != 0) {
+ fprintf(stderr, "Failed to format VMD's PCI address\n");
+ status = 1;
+ break;
+ }
+
+ if (g_opts.action == APP_ACTION_GET) {
+ rc = spdk_vmd_get_led_state(pci_device, &led_state);
+ if (spdk_unlikely(rc != 0)) {
+ fprintf(stderr, "Failed to retrieve the state of the LED on %s\n",
+ addr_buf);
+ status = 1;
+ break;
+ }
+
+ printf("%s: %s\n", addr_buf, g_led_states[led_state]);
+ } else {
+ rc = spdk_vmd_set_led_state(pci_device, g_opts.led_state);
+ if (spdk_unlikely(rc != 0)) {
+ fprintf(stderr, "Failed to set LED state on %s\n", addr_buf);
+ status = 1;
+ break;
+ }
+ }
+ }
+
+ spdk_vmd_fini();
+
+ return status;
+}
diff --git a/src/spdk/examples/vmd/lsvmd/.gitignore b/src/spdk/examples/vmd/lsvmd/.gitignore
new file mode 100644
index 000000000..3956616f7
--- /dev/null
+++ b/src/spdk/examples/vmd/lsvmd/.gitignore
@@ -0,0 +1 @@
+lsvmd
diff --git a/src/spdk/examples/vmd/lsvmd/Makefile b/src/spdk/examples/vmd/lsvmd/Makefile
new file mode 100644
index 000000000..96fde0773
--- /dev/null
+++ b/src/spdk/examples/vmd/lsvmd/Makefile
@@ -0,0 +1,44 @@
+#
+# BSD LICENSE
+#
+# Copyright (c) Intel Corporation.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Intel Corporation nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+include $(SPDK_ROOT_DIR)/mk/spdk.modules.mk
+
+APP = lsvmd
+
+C_SRCS := lsvmd.c
+
+SPDK_LIB_LIST = vmd log
+
+include $(SPDK_ROOT_DIR)/mk/spdk.app.mk
diff --git a/src/spdk/examples/vmd/lsvmd/lsvmd.c b/src/spdk/examples/vmd/lsvmd/lsvmd.c
new file mode 100644
index 000000000..b083bc049
--- /dev/null
+++ b/src/spdk/examples/vmd/lsvmd/lsvmd.c
@@ -0,0 +1,110 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+#include "spdk/log.h"
+#include "spdk/env.h"
+#include "spdk/vmd.h"
+
+struct spdk_pci_addr g_probe_addr;
+
+static int
+parse_args(int argc, char **argv)
+{
+ int op;
+
+ while ((op = getopt(argc, argv, "r:d")) != -1) {
+ switch (op) {
+ case 'r':
+ if (spdk_pci_addr_parse(&g_probe_addr, optarg)) {
+ SPDK_ERRLOG("Error parsing PCI address\n");
+ return 1;
+ }
+
+ break;
+
+ case 'd':
+ spdk_log_set_print_level(SPDK_LOG_DEBUG);
+ spdk_log_set_flag("vmd");
+ break;
+
+ default:
+ return 1;
+ }
+ }
+
+ return 0;
+}
+
+int main(int argc, char **argv)
+{
+ struct spdk_env_opts opts;
+ struct spdk_pci_device *pci_device;
+ char addr_buf[128];
+ int rc;
+
+ rc = parse_args(argc, argv);
+ if (rc != 0) {
+ return rc;
+ }
+
+ spdk_env_opts_init(&opts);
+ opts.name = "lsvmd";
+
+ if (spdk_env_init(&opts) < 0) {
+ SPDK_ERRLOG("Unable to initialize SPDK env\n");
+ return 1;
+ }
+
+ rc = spdk_vmd_init();
+ if (rc) {
+ SPDK_ERRLOG("No VMD Controllers found\n");
+ }
+
+ for (pci_device = spdk_pci_get_first_device(); pci_device != NULL;
+ pci_device = spdk_pci_get_next_device(pci_device)) {
+ if (strcmp(spdk_pci_device_get_type(pci_device), "vmd") == 0) {
+ rc = spdk_pci_addr_fmt(addr_buf, sizeof(addr_buf), &pci_device->addr);
+ if (rc != 0) {
+ fprintf(stderr, "Failed to format VMD's PCI address\n");
+ continue;
+ }
+
+ printf("%s\n", addr_buf);
+ }
+ }
+
+ spdk_vmd_fini();
+
+ return rc;
+}