summaryrefslogtreecommitdiffstats
path: root/src/spdk/examples/nvme/hotplug/hotplug.c
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--src/spdk/examples/nvme/hotplug/hotplug.c491
1 files changed, 491 insertions, 0 deletions
diff --git a/src/spdk/examples/nvme/hotplug/hotplug.c b/src/spdk/examples/nvme/hotplug/hotplug.c
new file mode 100644
index 00000000..66d93a71
--- /dev/null
+++ b/src/spdk/examples/nvme/hotplug/hotplug.c
@@ -0,0 +1,491 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+
+#include "spdk/nvme.h"
+#include "spdk/queue.h"
+
+struct dev_ctx {
+ TAILQ_ENTRY(dev_ctx) tailq;
+ bool is_new;
+ bool is_removed;
+ bool is_draining;
+ struct spdk_nvme_ctrlr *ctrlr;
+ struct spdk_nvme_ns *ns;
+ struct spdk_nvme_qpair *qpair;
+ uint32_t io_size_blocks;
+ uint64_t size_in_ios;
+ uint64_t io_completed;
+ uint64_t prev_io_completed;
+ uint64_t current_queue_depth;
+ uint64_t offset_in_ios;
+ char name[1024];
+};
+
+struct perf_task {
+ struct dev_ctx *dev;
+ void *buf;
+};
+
+static TAILQ_HEAD(, dev_ctx) g_devs = TAILQ_HEAD_INITIALIZER(g_devs);
+
+static uint64_t g_tsc_rate;
+
+static uint32_t g_io_size_bytes = 4096;
+static int g_queue_depth = 4;
+static int g_time_in_sec;
+static int g_expected_insert_times = -1;
+static int g_expected_removal_times = -1;
+static int g_insert_times;
+static int g_removal_times;
+static int g_shm_id = -1;
+
+static void
+task_complete(struct perf_task *task);
+
+static void
+register_dev(struct spdk_nvme_ctrlr *ctrlr)
+{
+ struct dev_ctx *dev;
+ const struct spdk_nvme_ctrlr_data *cdata = spdk_nvme_ctrlr_get_data(ctrlr);
+
+ dev = calloc(1, sizeof(*dev));
+ if (dev == NULL) {
+ perror("dev_ctx malloc");
+ exit(1);
+ }
+
+ snprintf(dev->name, sizeof(dev->name), "%-20.20s (%-20.20s)", cdata->mn, cdata->sn);
+
+ dev->ctrlr = ctrlr;
+ dev->is_new = true;
+ dev->is_removed = false;
+ dev->is_draining = false;
+
+ dev->ns = spdk_nvme_ctrlr_get_ns(ctrlr, 1);
+
+ if (!dev->ns || !spdk_nvme_ns_is_active(dev->ns)) {
+ fprintf(stderr, "Controller %s: No active namespace; skipping\n", dev->name);
+ goto skip;
+ }
+
+ if (spdk_nvme_ns_get_size(dev->ns) < g_io_size_bytes ||
+ spdk_nvme_ns_get_sector_size(dev->ns) > g_io_size_bytes) {
+ fprintf(stderr, "Controller %s: Invalid "
+ "ns size %" PRIu64 " / block size %u for I/O size %u\n",
+ dev->name,
+ spdk_nvme_ns_get_size(dev->ns),
+ spdk_nvme_ns_get_sector_size(dev->ns),
+ g_io_size_bytes);
+ goto skip;
+ }
+
+ dev->size_in_ios = spdk_nvme_ns_get_size(dev->ns) / g_io_size_bytes;
+ dev->io_size_blocks = g_io_size_bytes / spdk_nvme_ns_get_sector_size(dev->ns);
+
+ dev->qpair = spdk_nvme_ctrlr_alloc_io_qpair(ctrlr, NULL, 0);
+ if (!dev->qpair) {
+ fprintf(stderr, "ERROR: spdk_nvme_ctrlr_alloc_io_qpair() failed\n");
+ goto skip;
+ }
+ g_insert_times++;
+ TAILQ_INSERT_TAIL(&g_devs, dev, tailq);
+ return;
+
+skip:
+ free(dev);
+}
+
+static void
+unregister_dev(struct dev_ctx *dev)
+{
+ fprintf(stderr, "unregister_dev: %s\n", dev->name);
+
+ spdk_nvme_ctrlr_free_io_qpair(dev->qpair);
+ spdk_nvme_detach(dev->ctrlr);
+
+ TAILQ_REMOVE(&g_devs, dev, tailq);
+ free(dev);
+}
+
+static struct perf_task *
+alloc_task(struct dev_ctx *dev)
+{
+ struct perf_task *task;
+
+ task = calloc(1, sizeof(*task));
+ if (task == NULL) {
+ return NULL;
+ }
+
+ task->buf = spdk_dma_zmalloc(g_io_size_bytes, 0x200, NULL);
+ if (task->buf == NULL) {
+ free(task);
+ return NULL;
+ }
+
+ task->dev = dev;
+
+ return task;
+}
+
+static void
+free_task(struct perf_task *task)
+{
+ spdk_dma_free(task->buf);
+ free(task);
+}
+
+static void io_complete(void *ctx, const struct spdk_nvme_cpl *completion);
+
+static void
+submit_single_io(struct perf_task *task)
+{
+ struct dev_ctx *dev = task->dev;
+ uint64_t offset_in_ios;
+ int rc;
+
+ offset_in_ios = dev->offset_in_ios++;
+ if (dev->offset_in_ios == dev->size_in_ios) {
+ dev->offset_in_ios = 0;
+ }
+
+ rc = spdk_nvme_ns_cmd_read(dev->ns, dev->qpair, task->buf,
+ offset_in_ios * dev->io_size_blocks,
+ dev->io_size_blocks, io_complete, task, 0);
+
+ if (rc != 0) {
+ fprintf(stderr, "starting I/O failed\n");
+ free_task(task);
+ } else {
+ dev->current_queue_depth++;
+ }
+}
+
+static void
+task_complete(struct perf_task *task)
+{
+ struct dev_ctx *dev;
+
+ dev = task->dev;
+ dev->current_queue_depth--;
+ dev->io_completed++;
+
+ /*
+ * is_draining indicates when time has expired for the test run
+ * and we are just waiting for the previously submitted I/O
+ * to complete. In this case, do not submit a new I/O to replace
+ * the one just completed.
+ */
+ if (!dev->is_draining && !dev->is_removed) {
+ submit_single_io(task);
+ } else {
+ free_task(task);
+ }
+}
+
+static void
+io_complete(void *ctx, const struct spdk_nvme_cpl *completion)
+{
+ task_complete((struct perf_task *)ctx);
+}
+
+static void
+check_io(struct dev_ctx *dev)
+{
+ spdk_nvme_qpair_process_completions(dev->qpair, 0);
+}
+
+static void
+submit_io(struct dev_ctx *dev, int queue_depth)
+{
+ struct perf_task *task;
+
+ while (queue_depth-- > 0) {
+ task = alloc_task(dev);
+ if (task == NULL) {
+ fprintf(stderr, "task allocation failed\n");
+ exit(1);
+ }
+
+ submit_single_io(task);
+ }
+}
+
+static void
+drain_io(struct dev_ctx *dev)
+{
+ dev->is_draining = true;
+ while (dev->current_queue_depth > 0) {
+ check_io(dev);
+ }
+}
+
+static void
+print_stats(void)
+{
+ struct dev_ctx *dev;
+
+ TAILQ_FOREACH(dev, &g_devs, tailq) {
+ fprintf(stderr, "%-43.43s: %10" PRIu64 " I/Os completed (+%" PRIu64 ")\n",
+ dev->name,
+ dev->io_completed,
+ dev->io_completed - dev->prev_io_completed);
+ dev->prev_io_completed = dev->io_completed;
+ }
+
+ fprintf(stderr, "\n");
+}
+
+static bool
+probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
+ struct spdk_nvme_ctrlr_opts *opts)
+{
+ fprintf(stderr, "Attaching to %s\n", trid->traddr);
+
+ return true;
+}
+
+static void
+attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
+ struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts)
+{
+ fprintf(stderr, "Attached to %s\n", trid->traddr);
+
+ register_dev(ctrlr);
+}
+
+static void
+remove_cb(void *cb_ctx, struct spdk_nvme_ctrlr *ctrlr)
+{
+ struct dev_ctx *dev;
+
+ TAILQ_FOREACH(dev, &g_devs, tailq) {
+ if (dev->ctrlr == ctrlr) {
+ /*
+ * Mark the device as removed, but don't detach yet.
+ *
+ * The I/O handling code will detach once it sees that
+ * is_removed is true and all outstanding I/O have been completed.
+ */
+ dev->is_removed = true;
+ fprintf(stderr, "Controller removed: %s\n", dev->name);
+ return;
+ }
+ }
+
+ /*
+ * If we get here, this remove_cb is for a controller that we are not tracking
+ * in g_devs (for example, because we skipped it during register_dev),
+ * so immediately detach it.
+ */
+ spdk_nvme_detach(ctrlr);
+}
+
+static void
+io_loop(void)
+{
+ struct dev_ctx *dev, *dev_tmp;
+ uint64_t tsc_end;
+ uint64_t next_stats_tsc;
+
+ tsc_end = spdk_get_ticks() + g_time_in_sec * g_tsc_rate;
+ next_stats_tsc = spdk_get_ticks();
+
+ while (1) {
+ uint64_t now;
+
+ /*
+ * Check for completed I/O for each controller. A new
+ * I/O will be submitted in the io_complete callback
+ * to replace each I/O that is completed.
+ */
+ TAILQ_FOREACH(dev, &g_devs, tailq) {
+ if (dev->is_new) {
+ /* Submit initial I/O for this controller. */
+ submit_io(dev, g_queue_depth);
+ dev->is_new = false;
+ }
+
+ check_io(dev);
+ }
+
+ /*
+ * Check for hotplug events.
+ */
+ if (spdk_nvme_probe(NULL, NULL, probe_cb, attach_cb, remove_cb) != 0) {
+ fprintf(stderr, "spdk_nvme_probe() failed\n");
+ break;
+ }
+
+ /*
+ * Check for devices which were hot-removed and have finished
+ * processing outstanding I/Os.
+ *
+ * unregister_dev() may remove devs from the list, so use the
+ * removal-safe iterator.
+ */
+ TAILQ_FOREACH_SAFE(dev, &g_devs, tailq, dev_tmp) {
+ if (dev->is_removed && dev->current_queue_depth == 0) {
+ g_removal_times++;
+ unregister_dev(dev);
+ }
+ }
+
+ now = spdk_get_ticks();
+ if (now > tsc_end) {
+ break;
+ }
+ if (now > next_stats_tsc) {
+ print_stats();
+ next_stats_tsc += g_tsc_rate;
+ }
+
+ if (g_insert_times == g_expected_insert_times && g_removal_times == g_expected_removal_times) {
+ break;
+ }
+ }
+
+ TAILQ_FOREACH_SAFE(dev, &g_devs, tailq, dev_tmp) {
+ drain_io(dev);
+ unregister_dev(dev);
+ }
+}
+
+static void usage(char *program_name)
+{
+ printf("%s options", program_name);
+ printf("\n");
+ printf("\t[-i shm id (optional)]\n");
+ printf("\t[-n expected hot insert times]\n");
+ printf("\t[-r expected hot removal times]\n");
+ printf("\t[-t time in seconds]\n");
+}
+
+static int
+parse_args(int argc, char **argv)
+{
+ int op;
+
+ /* default value */
+ g_time_in_sec = 0;
+
+ while ((op = getopt(argc, argv, "i:n:r:t:")) != -1) {
+ switch (op) {
+ case 'i':
+ g_shm_id = atoi(optarg);
+ break;
+ case 'n':
+ g_expected_insert_times = atoi(optarg);
+ break;
+ case 'r':
+ g_expected_removal_times = atoi(optarg);
+ break;
+ case 't':
+ g_time_in_sec = atoi(optarg);
+ break;
+ default:
+ usage(argv[0]);
+ return 1;
+ }
+ }
+
+ if (!g_time_in_sec) {
+ usage(argv[0]);
+ return 1;
+ }
+
+ return 0;
+}
+
+
+static int
+register_controllers(void)
+{
+ fprintf(stderr, "Initializing NVMe Controllers\n");
+
+ if (spdk_nvme_probe(NULL, NULL, probe_cb, attach_cb, remove_cb) != 0) {
+ fprintf(stderr, "spdk_nvme_probe() failed\n");
+ return 1;
+ }
+ /* Reset g_insert_times to 0 so that we do not count controllers attached at start as hotplug events. */
+ g_insert_times = 0;
+ return 0;
+}
+
+int main(int argc, char **argv)
+{
+ int rc;
+ struct spdk_env_opts opts;
+
+ rc = parse_args(argc, argv);
+ if (rc != 0) {
+ return rc;
+ }
+
+ spdk_env_opts_init(&opts);
+ opts.name = "hotplug";
+ opts.core_mask = "0x1";
+ if (g_shm_id > -1) {
+ opts.shm_id = g_shm_id;
+ }
+ if (spdk_env_init(&opts) < 0) {
+ fprintf(stderr, "Unable to initialize SPDK env\n");
+ return 1;
+ }
+
+ g_tsc_rate = spdk_get_ticks_hz();
+
+ /* Detect the controllers that are plugged in at startup. */
+ if (register_controllers() != 0) {
+ return 1;
+ }
+
+ fprintf(stderr, "Initialization complete. Starting I/O...\n");
+ io_loop();
+
+ if (g_expected_insert_times != -1 && g_insert_times != g_expected_insert_times) {
+ fprintf(stderr, "Expected inserts %d != actual inserts %d\n",
+ g_expected_insert_times, g_insert_times);
+ return 1;
+ }
+
+ if (g_expected_removal_times != -1 && g_removal_times != g_expected_removal_times) {
+ fprintf(stderr, "Expected removals %d != actual removals %d\n",
+ g_expected_removal_times, g_removal_times);
+ return 1;
+ }
+
+ return 0;
+}
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-22 04:37:42 +0000