diff options
Diffstat (limited to 'src/spdk/examples/nvme/perf')
| -rw-r--r-- | src/spdk/examples/nvme/perf/.gitignore | 1 | ||||
| -rw-r--r-- | src/spdk/examples/nvme/perf/Makefile | 49 | ||||
| -rw-r--r-- | src/spdk/examples/nvme/perf/README.md | 5 | ||||
| -rw-r--r-- | src/spdk/examples/nvme/perf/perf.c | 2308 |
4 files changed, 2363 insertions, 0 deletions
diff --git a/src/spdk/examples/nvme/perf/.gitignore b/src/spdk/examples/nvme/perf/.gitignore new file mode 100644 index 000000000..bd14107d8 --- /dev/null +++ b/src/spdk/examples/nvme/perf/.gitignore @@ -0,0 +1 @@ +perf diff --git a/src/spdk/examples/nvme/perf/Makefile b/src/spdk/examples/nvme/perf/Makefile new file mode 100644 index 000000000..0742f1842 --- /dev/null +++ b/src/spdk/examples/nvme/perf/Makefile @@ -0,0 +1,49 @@ +# +# BSD LICENSE +# +# Copyright (c) Intel Corporation. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..) + +APP = perf + +include $(SPDK_ROOT_DIR)/mk/nvme.libtest.mk + +ifeq ($(OS),Linux) +SYS_LIBS += -laio +CFLAGS += -DHAVE_LIBAIO +endif + +install: $(APP) + $(INSTALL_EXAMPLE) + +uninstall: + $(UNINSTALL_EXAMPLE) diff --git a/src/spdk/examples/nvme/perf/README.md b/src/spdk/examples/nvme/perf/README.md new file mode 100644 index 000000000..e5ec38d12 --- /dev/null +++ b/src/spdk/examples/nvme/perf/README.md @@ -0,0 +1,5 @@ +# Compiling perf on FreeBSD + +To use perf test on FreeBSD over NVMe-oF, explicitly link userspace library of HBA. For example, on a setup with Mellanox HBA, + + LIBS += -lmlx5 diff --git a/src/spdk/examples/nvme/perf/perf.c b/src/spdk/examples/nvme/perf/perf.c new file mode 100644 index 000000000..9e8cf6793 --- /dev/null +++ b/src/spdk/examples/nvme/perf/perf.c @@ -0,0 +1,2308 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "spdk/stdinc.h" + +#include "spdk/env.h" +#include "spdk/fd.h" +#include "spdk/nvme.h" +#include "spdk/vmd.h" +#include "spdk/queue.h" +#include "spdk/string.h" +#include "spdk/nvme_intel.h" +#include "spdk/histogram_data.h" +#include "spdk/endian.h" +#include "spdk/dif.h" +#include "spdk/util.h" +#include "spdk/log.h" +#include "spdk/likely.h" + +#ifdef SPDK_CONFIG_URING +#include <liburing.h> +#endif + +#if HAVE_LIBAIO +#include <libaio.h> +#endif + +struct ctrlr_entry { + struct spdk_nvme_ctrlr *ctrlr; + enum spdk_nvme_transport_type trtype; + struct spdk_nvme_intel_rw_latency_page *latency_page; + + struct spdk_nvme_qpair **unused_qpairs; + + struct ctrlr_entry *next; + char name[1024]; +}; + +enum entry_type { + ENTRY_TYPE_NVME_NS, + ENTRY_TYPE_AIO_FILE, + ENTRY_TYPE_URING_FILE, +}; + +struct ns_fn_table; + +struct ns_entry { + enum entry_type type; + const struct ns_fn_table *fn_table; + + union { + struct { + struct spdk_nvme_ctrlr *ctrlr; + struct spdk_nvme_ns *ns; + } nvme; +#ifdef SPDK_CONFIG_URING + struct { + int fd; + } uring; +#endif +#if HAVE_LIBAIO + struct { + int fd; + } aio; +#endif + } u; + + struct ns_entry *next; + uint32_t io_size_blocks; + uint32_t num_io_requests; + uint64_t size_in_ios; + uint32_t block_size; + uint32_t md_size; + bool md_interleave; + bool pi_loc; + enum spdk_nvme_pi_type pi_type; + uint32_t io_flags; + char name[1024]; +}; + +static const double g_latency_cutoffs[] = { + 0.01, + 0.10, + 0.25, + 0.50, + 0.75, + 0.90, + 0.95, + 0.98, + 0.99, + 0.995, + 0.999, + 0.9999, + 0.99999, + 0.999999, + 0.9999999, + -1, +}; + +struct ns_worker_ctx { + struct ns_entry *entry; + uint64_t io_completed; + uint64_t last_io_completed; + uint64_t total_tsc; + uint64_t min_tsc; + uint64_t max_tsc; + uint64_t current_queue_depth; + uint64_t offset_in_ios; + bool is_draining; + + union { + struct { + int num_active_qpairs; + int num_all_qpairs; + struct spdk_nvme_qpair **qpair; + struct spdk_nvme_poll_group *group; + int last_qpair; + } nvme; + +#ifdef SPDK_CONFIG_URING + struct { + struct io_uring ring; + uint64_t io_inflight; + uint64_t io_pending; + struct io_uring_cqe **cqes; + + } uring; +#endif +#if HAVE_LIBAIO + struct { + struct io_event *events; + io_context_t ctx; + } aio; +#endif + } u; + + struct ns_worker_ctx *next; + + struct spdk_histogram_data *histogram; +}; + +struct perf_task { + struct ns_worker_ctx *ns_ctx; + struct iovec iov; + struct iovec md_iov; + uint64_t submit_tsc; + bool is_read; + struct spdk_dif_ctx dif_ctx; +#if HAVE_LIBAIO + struct iocb iocb; +#endif +}; + +struct worker_thread { + struct ns_worker_ctx *ns_ctx; + struct worker_thread *next; + unsigned lcore; +}; + +struct ns_fn_table { + void (*setup_payload)(struct perf_task *task, uint8_t pattern); + + int (*submit_io)(struct perf_task *task, struct ns_worker_ctx *ns_ctx, + struct ns_entry *entry, uint64_t offset_in_ios); + + void (*check_io)(struct ns_worker_ctx *ns_ctx); + + void (*verify_io)(struct perf_task *task, struct ns_entry *entry); + + int (*init_ns_worker_ctx)(struct ns_worker_ctx *ns_ctx); + + void (*cleanup_ns_worker_ctx)(struct ns_worker_ctx *ns_ctx); +}; + +static int g_outstanding_commands; + +static bool g_latency_ssd_tracking_enable; +static int g_latency_sw_tracking_level; + +static bool g_vmd; +static const char *g_workload_type; +static struct ctrlr_entry *g_controllers; +static struct ns_entry *g_namespaces; +static int g_num_namespaces; +static struct worker_thread *g_workers; +static int g_num_workers; +static uint32_t g_master_core; + +static uint64_t g_tsc_rate; + +static uint32_t g_io_align = 0x200; +static uint32_t g_io_size_bytes; +static uint32_t g_max_io_md_size; +static uint32_t g_max_io_size_blocks; +static uint32_t g_metacfg_pract_flag; +static uint32_t g_metacfg_prchk_flags; +static int g_rw_percentage = -1; +static int g_is_random; +static int g_queue_depth; +static int g_nr_io_queues_per_ns = 1; +static int g_nr_unused_io_queues; +static int g_time_in_sec; +static uint32_t g_max_completions; +static int g_dpdk_mem; +static int g_shm_id = -1; +static uint32_t g_disable_sq_cmb; +static bool g_use_uring; +static bool g_no_pci; +static bool g_warn; +static bool g_header_digest; +static bool g_data_digest; +static bool g_no_shn_notification; +static bool g_mix_specified; +/* Default to 10 seconds for the keep alive value. This value is arbitrary. */ +static uint32_t g_keep_alive_timeout_in_ms = 10000; + +static const char *g_core_mask; + +struct trid_entry { + struct spdk_nvme_transport_id trid; + uint16_t nsid; + TAILQ_ENTRY(trid_entry) tailq; +}; + +static TAILQ_HEAD(, trid_entry) g_trid_list = TAILQ_HEAD_INITIALIZER(g_trid_list); + +static int g_file_optind; /* Index of first filename in argv */ + +static inline void +task_complete(struct perf_task *task); + +#ifdef SPDK_CONFIG_URING + +static void +uring_setup_payload(struct perf_task *task, uint8_t pattern) +{ + task->iov.iov_base = spdk_dma_zmalloc(g_io_size_bytes, g_io_align, NULL); + task->iov.iov_len = g_io_size_bytes; + if (task->iov.iov_base == NULL) { + fprintf(stderr, "spdk_dma_zmalloc() for task->iov.iov_base failed\n"); + exit(1); + } + memset(task->iov.iov_base, pattern, task->iov.iov_len); +} + +static int +uring_submit_io(struct perf_task *task, struct ns_worker_ctx *ns_ctx, + struct ns_entry *entry, uint64_t offset_in_ios) +{ + struct io_uring_sqe *sqe; + + sqe = io_uring_get_sqe(&ns_ctx->u.uring.ring); + if (!sqe) { + fprintf(stderr, "Cannot get sqe\n"); + return -1; + } + + if (task->is_read) { + io_uring_prep_readv(sqe, entry->u.uring.fd, &task->iov, 1, offset_in_ios * task->iov.iov_len); + } else { + io_uring_prep_writev(sqe, entry->u.uring.fd, &task->iov, 1, offset_in_ios * task->iov.iov_len); + } + + io_uring_sqe_set_data(sqe, task); + ns_ctx->u.uring.io_pending++; + + return 0; +} + +static void +uring_check_io(struct ns_worker_ctx *ns_ctx) +{ + int i, count, to_complete, to_submit, ret = 0; + struct perf_task *task; + + to_submit = ns_ctx->u.uring.io_pending; + + if (to_submit > 0) { + /* If there are I/O to submit, use io_uring_submit here. + * It will automatically call spdk_io_uring_enter appropriately. */ + ret = io_uring_submit(&ns_ctx->u.uring.ring); + if (ret < 0) { + return; + } + ns_ctx->u.uring.io_pending = 0; + ns_ctx->u.uring.io_inflight += to_submit; + } + + to_complete = ns_ctx->u.uring.io_inflight; + if (to_complete > 0) { + count = io_uring_peek_batch_cqe(&ns_ctx->u.uring.ring, ns_ctx->u.uring.cqes, to_complete); + ns_ctx->u.uring.io_inflight -= count; + for (i = 0; i < count; i++) { + assert(ns_ctx->u.uring.cqes[i] != NULL); + task = (struct perf_task *)ns_ctx->u.uring.cqes[i]->user_data; + if (ns_ctx->u.uring.cqes[i]->res != (int)task->iov.iov_len) { + fprintf(stderr, "cqe[i]->status=%d\n", ns_ctx->u.uring.cqes[i]->res); + exit(0); + } + io_uring_cqe_seen(&ns_ctx->u.uring.ring, ns_ctx->u.uring.cqes[i]); + task_complete(task); + } + } +} + +static void +uring_verify_io(struct perf_task *task, struct ns_entry *entry) +{ +} + +static int +uring_init_ns_worker_ctx(struct ns_worker_ctx *ns_ctx) +{ + if (io_uring_queue_init(g_queue_depth, &ns_ctx->u.uring.ring, 0) < 0) { + SPDK_ERRLOG("uring I/O context setup failure\n"); + return -1; + } + + ns_ctx->u.uring.cqes = calloc(g_queue_depth, sizeof(struct io_uring_cqe *)); + if (!ns_ctx->u.uring.cqes) { + io_uring_queue_exit(&ns_ctx->u.uring.ring); + return -1; + } + + return 0; +} + +static void +uring_cleanup_ns_worker_ctx(struct ns_worker_ctx *ns_ctx) +{ + io_uring_queue_exit(&ns_ctx->u.uring.ring); + free(ns_ctx->u.uring.cqes); +} + +static const struct ns_fn_table uring_fn_table = { + .setup_payload = uring_setup_payload, + .submit_io = uring_submit_io, + .check_io = uring_check_io, + .verify_io = uring_verify_io, + .init_ns_worker_ctx = uring_init_ns_worker_ctx, + .cleanup_ns_worker_ctx = uring_cleanup_ns_worker_ctx, +}; + +#endif + +#ifdef HAVE_LIBAIO +static void +aio_setup_payload(struct perf_task *task, uint8_t pattern) +{ + task->iov.iov_base = spdk_dma_zmalloc(g_io_size_bytes, g_io_align, NULL); + task->iov.iov_len = g_io_size_bytes; + if (task->iov.iov_base == NULL) { + fprintf(stderr, "spdk_dma_zmalloc() for task->buf failed\n"); + exit(1); + } + memset(task->iov.iov_base, pattern, task->iov.iov_len); +} + +static int +aio_submit(io_context_t aio_ctx, struct iocb *iocb, int fd, enum io_iocb_cmd cmd, + struct iovec *iov, uint64_t offset, void *cb_ctx) +{ + iocb->aio_fildes = fd; + iocb->aio_reqprio = 0; + iocb->aio_lio_opcode = cmd; + iocb->u.c.buf = iov->iov_base; + iocb->u.c.nbytes = iov->iov_len; + iocb->u.c.offset = offset * iov->iov_len; + iocb->data = cb_ctx; + + if (io_submit(aio_ctx, 1, &iocb) < 0) { + printf("io_submit"); + return -1; + } + + return 0; +} + +static int +aio_submit_io(struct perf_task *task, struct ns_worker_ctx *ns_ctx, + struct ns_entry *entry, uint64_t offset_in_ios) +{ + if (task->is_read) { + return aio_submit(ns_ctx->u.aio.ctx, &task->iocb, entry->u.aio.fd, IO_CMD_PREAD, + &task->iov, offset_in_ios, task); + } else { + return aio_submit(ns_ctx->u.aio.ctx, &task->iocb, entry->u.aio.fd, IO_CMD_PWRITE, + &task->iov, offset_in_ios, task); + } +} + +static void +aio_check_io(struct ns_worker_ctx *ns_ctx) +{ + int count, i; + struct timespec timeout; + + timeout.tv_sec = 0; + timeout.tv_nsec = 0; + + count = io_getevents(ns_ctx->u.aio.ctx, 1, g_queue_depth, ns_ctx->u.aio.events, &timeout); + if (count < 0) { + fprintf(stderr, "io_getevents error\n"); + exit(1); + } + + for (i = 0; i < count; i++) { + task_complete(ns_ctx->u.aio.events[i].data); + } +} + +static void +aio_verify_io(struct perf_task *task, struct ns_entry *entry) +{ +} + +static int +aio_init_ns_worker_ctx(struct ns_worker_ctx *ns_ctx) +{ + ns_ctx->u.aio.events = calloc(g_queue_depth, sizeof(struct io_event)); + if (!ns_ctx->u.aio.events) { + return -1; + } + ns_ctx->u.aio.ctx = 0; + if (io_setup(g_queue_depth, &ns_ctx->u.aio.ctx) < 0) { + free(ns_ctx->u.aio.events); + perror("io_setup"); + return -1; + } + return 0; +} + +static void +aio_cleanup_ns_worker_ctx(struct ns_worker_ctx *ns_ctx) +{ + io_destroy(ns_ctx->u.aio.ctx); + free(ns_ctx->u.aio.events); +} + +static const struct ns_fn_table aio_fn_table = { + .setup_payload = aio_setup_payload, + .submit_io = aio_submit_io, + .check_io = aio_check_io, + .verify_io = aio_verify_io, + .init_ns_worker_ctx = aio_init_ns_worker_ctx, + .cleanup_ns_worker_ctx = aio_cleanup_ns_worker_ctx, +}; + +#endif /* HAVE_LIBAIO */ + +#if defined(HAVE_LIBAIO) || defined(SPDK_CONFIG_URING) + +static int +register_file(const char *path) +{ + struct ns_entry *entry; + + int flags, fd; + uint64_t size; + uint32_t blklen; + + if (g_rw_percentage == 100) { + flags = O_RDONLY; + } else if (g_rw_percentage == 0) { + flags = O_WRONLY; + } else { + flags = O_RDWR; + } + + flags |= O_DIRECT; + + fd = open(path, flags); + if (fd < 0) { + fprintf(stderr, "Could not open device %s: %s\n", path, strerror(errno)); + return -1; + } + + size = spdk_fd_get_size(fd); + if (size == 0) { + fprintf(stderr, "Could not determine size of device %s\n", path); + close(fd); + return -1; + } + + blklen = spdk_fd_get_blocklen(fd); + if (blklen == 0) { + fprintf(stderr, "Could not determine block size of device %s\n", path); + close(fd); + return -1; + } + + /* + * TODO: This should really calculate the LCM of the current g_io_align and blklen. + * For now, it's fairly safe to just assume all block sizes are powers of 2. + */ + if (g_io_align < blklen) { + g_io_align = blklen; + } + + entry = malloc(sizeof(struct ns_entry)); + if (entry == NULL) { + close(fd); + perror("ns_entry malloc"); + return -1; + } + + if (g_use_uring) { +#ifdef SPDK_CONFIG_URING + entry->type = ENTRY_TYPE_URING_FILE; + entry->fn_table = &uring_fn_table; + entry->u.uring.fd = fd; +#endif + } else { +#if HAVE_LIBAIO + entry->type = ENTRY_TYPE_AIO_FILE; + entry->fn_table = &aio_fn_table; + entry->u.aio.fd = fd; +#endif + } + entry->size_in_ios = size / g_io_size_bytes; + entry->io_size_blocks = g_io_size_bytes / blklen; + + snprintf(entry->name, sizeof(entry->name), "%s", path); + + g_num_namespaces++; + entry->next = g_namespaces; + g_namespaces = entry; + + return 0; +} + +static int +register_files(int argc, char **argv) +{ + int i; + + /* Treat everything after the options as files for AIO/URING */ + for (i = g_file_optind; i < argc; i++) { + if (register_file(argv[i]) != 0) { + return 1; + } + } + + return 0; +} +#endif + +static void io_complete(void *ctx, const struct spdk_nvme_cpl *cpl); + +static void +nvme_setup_payload(struct perf_task *task, uint8_t pattern) +{ + uint32_t max_io_size_bytes, max_io_md_size; + + /* maximum extended lba format size from all active namespace, + * it's same with g_io_size_bytes for namespace without metadata. + */ + max_io_size_bytes = g_io_size_bytes + g_max_io_md_size * g_max_io_size_blocks; + task->iov.iov_base = spdk_dma_zmalloc(max_io_size_bytes, g_io_align, NULL); + task->iov.iov_len = max_io_size_bytes; + if (task->iov.iov_base == NULL) { + fprintf(stderr, "task->buf spdk_dma_zmalloc failed\n"); + exit(1); + } + memset(task->iov.iov_base, pattern, task->iov.iov_len); + + max_io_md_size = g_max_io_md_size * g_max_io_size_blocks; + if (max_io_md_size != 0) { + task->md_iov.iov_base = spdk_dma_zmalloc(max_io_md_size, g_io_align, NULL); + task->md_iov.iov_len = max_io_md_size; + if (task->md_iov.iov_base == NULL) { + fprintf(stderr, "task->md_buf spdk_dma_zmalloc failed\n"); + spdk_dma_free(task->iov.iov_base); + exit(1); + } + } +} + +static int +nvme_submit_io(struct perf_task *task, struct ns_worker_ctx *ns_ctx, + struct ns_entry *entry, uint64_t offset_in_ios) +{ + uint64_t lba; + int rc; + int qp_num; + + enum dif_mode { + DIF_MODE_NONE = 0, + DIF_MODE_DIF = 1, + DIF_MODE_DIX = 2, + } mode = DIF_MODE_NONE; + + lba = offset_in_ios * entry->io_size_blocks; + + if (entry->md_size != 0 && !(entry->io_flags & SPDK_NVME_IO_FLAGS_PRACT)) { + if (entry->md_interleave) { + mode = DIF_MODE_DIF; + } else { + mode = DIF_MODE_DIX; + } + } + + qp_num = ns_ctx->u.nvme.last_qpair; + ns_ctx->u.nvme.last_qpair++; + if (ns_ctx->u.nvme.last_qpair == ns_ctx->u.nvme.num_active_qpairs) { + ns_ctx->u.nvme.last_qpair = 0; + } + + if (mode != DIF_MODE_NONE) { + rc = spdk_dif_ctx_init(&task->dif_ctx, entry->block_size, entry->md_size, + entry->md_interleave, entry->pi_loc, + (enum spdk_dif_type)entry->pi_type, entry->io_flags, + lba, 0xFFFF, (uint16_t)entry->io_size_blocks, 0, 0); + if (rc != 0) { + fprintf(stderr, "Initialization of DIF context failed\n"); + exit(1); + } + } + + if (task->is_read) { + return spdk_nvme_ns_cmd_read_with_md(entry->u.nvme.ns, ns_ctx->u.nvme.qpair[qp_num], + task->iov.iov_base, task->md_iov.iov_base, + lba, + entry->io_size_blocks, io_complete, + task, entry->io_flags, + task->dif_ctx.apptag_mask, task->dif_ctx.app_tag); + } else { + switch (mode) { + case DIF_MODE_DIF: + rc = spdk_dif_generate(&task->iov, 1, entry->io_size_blocks, &task->dif_ctx); + if (rc != 0) { + fprintf(stderr, "Generation of DIF failed\n"); + return rc; + } + break; + case DIF_MODE_DIX: + rc = spdk_dix_generate(&task->iov, 1, &task->md_iov, entry->io_size_blocks, + &task->dif_ctx); + if (rc != 0) { + fprintf(stderr, "Generation of DIX failed\n"); + return rc; + } + break; + default: + break; + } + + return spdk_nvme_ns_cmd_write_with_md(entry->u.nvme.ns, ns_ctx->u.nvme.qpair[qp_num], + task->iov.iov_base, task->md_iov.iov_base, + lba, + entry->io_size_blocks, io_complete, + task, entry->io_flags, + task->dif_ctx.apptag_mask, task->dif_ctx.app_tag); + } +} + +static void +perf_disconnect_cb(struct spdk_nvme_qpair *qpair, void *ctx) +{ + +} + +static void +nvme_check_io(struct ns_worker_ctx *ns_ctx) +{ + int64_t rc; + + rc = spdk_nvme_poll_group_process_completions(ns_ctx->u.nvme.group, 0, perf_disconnect_cb); + if (rc < 0) { + fprintf(stderr, "NVMe io qpair process completion error\n"); + exit(1); + } +} + +static void +nvme_verify_io(struct perf_task *task, struct ns_entry *entry) +{ + struct spdk_dif_error err_blk = {}; + int rc; + + if (!task->is_read || (entry->io_flags & SPDK_NVME_IO_FLAGS_PRACT)) { + return; + } + + if (entry->md_interleave) { + rc = spdk_dif_verify(&task->iov, 1, entry->io_size_blocks, &task->dif_ctx, + &err_blk); + if (rc != 0) { + fprintf(stderr, "DIF error detected. type=%d, offset=%" PRIu32 "\n", + err_blk.err_type, err_blk.err_offset); + } + } else { + rc = spdk_dix_verify(&task->iov, 1, &task->md_iov, entry->io_size_blocks, + &task->dif_ctx, &err_blk); + if (rc != 0) { + fprintf(stderr, "DIX error detected. type=%d, offset=%" PRIu32 "\n", + err_blk.err_type, err_blk.err_offset); + } + } +} + +/* + * TODO: If a controller has multiple namespaces, they could all use the same queue. + * For now, give each namespace/thread combination its own queue. + */ +static int +nvme_init_ns_worker_ctx(struct ns_worker_ctx *ns_ctx) +{ + struct spdk_nvme_io_qpair_opts opts; + struct ns_entry *entry = ns_ctx->entry; + struct spdk_nvme_poll_group *group; + struct spdk_nvme_qpair *qpair; + int i; + + ns_ctx->u.nvme.num_active_qpairs = g_nr_io_queues_per_ns; + ns_ctx->u.nvme.num_all_qpairs = g_nr_io_queues_per_ns + g_nr_unused_io_queues; + ns_ctx->u.nvme.qpair = calloc(ns_ctx->u.nvme.num_all_qpairs, sizeof(struct spdk_nvme_qpair *)); + if (!ns_ctx->u.nvme.qpair) { + return -1; + } + + spdk_nvme_ctrlr_get_default_io_qpair_opts(entry->u.nvme.ctrlr, &opts, sizeof(opts)); + if (opts.io_queue_requests < entry->num_io_requests) { + opts.io_queue_requests = entry->num_io_requests; + } + opts.delay_cmd_submit = true; + opts.create_only = true; + + ns_ctx->u.nvme.group = spdk_nvme_poll_group_create(NULL); + if (ns_ctx->u.nvme.group == NULL) { + goto poll_group_failed; + } + + group = ns_ctx->u.nvme.group; + for (i = 0; i < ns_ctx->u.nvme.num_all_qpairs; i++) { + ns_ctx->u.nvme.qpair[i] = spdk_nvme_ctrlr_alloc_io_qpair(entry->u.nvme.ctrlr, &opts, + sizeof(opts)); + qpair = ns_ctx->u.nvme.qpair[i]; + if (!qpair) { + printf("ERROR: spdk_nvme_ctrlr_alloc_io_qpair failed\n"); + goto qpair_failed; + } + + if (spdk_nvme_poll_group_add(group, qpair)) { + printf("ERROR: unable to add I/O qpair to poll group.\n"); + spdk_nvme_ctrlr_free_io_qpair(qpair); + goto qpair_failed; + } + + if (spdk_nvme_ctrlr_connect_io_qpair(entry->u.nvme.ctrlr, qpair)) { + printf("ERROR: unable to connect I/O qpair.\n"); + spdk_nvme_poll_group_remove(group, qpair); + spdk_nvme_ctrlr_free_io_qpair(qpair); + goto qpair_failed; + } + } + + return 0; + +qpair_failed: + for (; i > 0; --i) { + spdk_nvme_poll_group_remove(ns_ctx->u.nvme.group, ns_ctx->u.nvme.qpair[i - 1]); + spdk_nvme_ctrlr_free_io_qpair(ns_ctx->u.nvme.qpair[i - 1]); + } + + spdk_nvme_poll_group_destroy(ns_ctx->u.nvme.group); +poll_group_failed: + free(ns_ctx->u.nvme.qpair); + return -1; +} + +static void +nvme_cleanup_ns_worker_ctx(struct ns_worker_ctx *ns_ctx) +{ + int i; + + for (i = 0; i < ns_ctx->u.nvme.num_all_qpairs; i++) { + spdk_nvme_poll_group_remove(ns_ctx->u.nvme.group, ns_ctx->u.nvme.qpair[i]); + spdk_nvme_ctrlr_free_io_qpair(ns_ctx->u.nvme.qpair[i]); + } + + spdk_nvme_poll_group_destroy(ns_ctx->u.nvme.group); + free(ns_ctx->u.nvme.qpair); +} + +static const struct ns_fn_table nvme_fn_table = { + .setup_payload = nvme_setup_payload, + .submit_io = nvme_submit_io, + .check_io = nvme_check_io, + .verify_io = nvme_verify_io, + .init_ns_worker_ctx = nvme_init_ns_worker_ctx, + .cleanup_ns_worker_ctx = nvme_cleanup_ns_worker_ctx, +}; + +static int +build_nvme_name(char *name, size_t length, struct spdk_nvme_ctrlr *ctrlr) +{ + const struct spdk_nvme_transport_id *trid; + int res = 0; + + trid = spdk_nvme_ctrlr_get_transport_id(ctrlr); + + switch (trid->trtype) { + case SPDK_NVME_TRANSPORT_PCIE: + res = snprintf(name, length, "PCIE (%s)", trid->traddr); + break; + case SPDK_NVME_TRANSPORT_RDMA: + res = snprintf(name, length, "RDMA (addr:%s subnqn:%s)", trid->traddr, trid->subnqn); + break; + case SPDK_NVME_TRANSPORT_TCP: + res = snprintf(name, length, "TCP (addr:%s subnqn:%s)", trid->traddr, trid->subnqn); + break; + + default: + fprintf(stderr, "Unknown transport type %d\n", trid->trtype); + break; + } + return res; +} + +static void +build_nvme_ns_name(char *name, size_t length, struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid) +{ + int res = 0; + + res = build_nvme_name(name, length, ctrlr); + if (res > 0) { + snprintf(name + res, length - res, " NSID %u", nsid); + } + +} + +static void +register_ns(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns *ns) +{ + struct ns_entry *entry; + const struct spdk_nvme_ctrlr_data *cdata; + uint32_t max_xfer_size, entries, sector_size; + uint64_t ns_size; + struct spdk_nvme_io_qpair_opts opts; + + cdata = spdk_nvme_ctrlr_get_data(ctrlr); + + if (!spdk_nvme_ns_is_active(ns)) { + printf("Controller %-20.20s (%-20.20s): Skipping inactive NS %u\n", + cdata->mn, cdata->sn, + spdk_nvme_ns_get_id(ns)); + g_warn = true; + return; + } + + ns_size = spdk_nvme_ns_get_size(ns); + sector_size = spdk_nvme_ns_get_sector_size(ns); + + if (ns_size < g_io_size_bytes || sector_size > g_io_size_bytes) { + printf("WARNING: controller %-20.20s (%-20.20s) ns %u has invalid " + "ns size %" PRIu64 " / block size %u for I/O size %u\n", + cdata->mn, cdata->sn, spdk_nvme_ns_get_id(ns), + ns_size, spdk_nvme_ns_get_sector_size(ns), g_io_size_bytes); + g_warn = true; + return; + } + + max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns); + spdk_nvme_ctrlr_get_default_io_qpair_opts(ctrlr, &opts, sizeof(opts)); + /* NVMe driver may add additional entries based on + * stripe size and maximum transfer size, we assume + * 1 more entry be used for stripe. + */ + entries = (g_io_size_bytes - 1) / max_xfer_size + 2; + if ((g_queue_depth * entries) > opts.io_queue_size) { + printf("controller IO queue size %u less than required\n", + opts.io_queue_size); + printf("Consider using lower queue depth or small IO size because " + "IO requests may be queued at the NVMe driver.\n"); + } + /* For requests which have children requests, parent request itself + * will also occupy 1 entry. + */ + entries += 1; + + entry = calloc(1, sizeof(struct ns_entry)); + if (entry == NULL) { + perror("ns_entry malloc"); + exit(1); + } + + entry->type = ENTRY_TYPE_NVME_NS; + entry->fn_table = &nvme_fn_table; + entry->u.nvme.ctrlr = ctrlr; + entry->u.nvme.ns = ns; + entry->num_io_requests = g_queue_depth * entries; + + entry->size_in_ios = ns_size / g_io_size_bytes; + entry->io_size_blocks = g_io_size_bytes / sector_size; + + entry->block_size = spdk_nvme_ns_get_extended_sector_size(ns); + entry->md_size = spdk_nvme_ns_get_md_size(ns); + entry->md_interleave = spdk_nvme_ns_supports_extended_lba(ns); + entry->pi_loc = spdk_nvme_ns_get_data(ns)->dps.md_start; + entry->pi_type = spdk_nvme_ns_get_pi_type(ns); + + if (spdk_nvme_ns_get_flags(ns) & SPDK_NVME_NS_DPS_PI_SUPPORTED) { + entry->io_flags = g_metacfg_pract_flag | g_metacfg_prchk_flags; + } + + /* If metadata size = 8 bytes, PI is stripped (read) or inserted (write), + * and so reduce metadata size from block size. (If metadata size > 8 bytes, + * PI is passed (read) or replaced (write). So block size is not necessary + * to change.) + */ + if ((entry->io_flags & SPDK_NVME_IO_FLAGS_PRACT) && (entry->md_size == 8)) { + entry->block_size = spdk_nvme_ns_get_sector_size(ns); + } + + if (g_max_io_md_size < entry->md_size) { + g_max_io_md_size = entry->md_size; + } + + if (g_max_io_size_blocks < entry->io_size_blocks) { + g_max_io_size_blocks = entry->io_size_blocks; + } + + build_nvme_ns_name(entry->name, sizeof(entry->name), ctrlr, spdk_nvme_ns_get_id(ns)); + + g_num_namespaces++; + entry->next = g_namespaces; + g_namespaces = entry; +} + +static void +unregister_namespaces(void) +{ + struct ns_entry *entry = g_namespaces; + + while (entry) { + struct ns_entry *next = entry->next; + free(entry); + entry = next; + } +} + +static void +enable_latency_tracking_complete(void *cb_arg, const struct spdk_nvme_cpl *cpl) +{ + if (spdk_nvme_cpl_is_error(cpl)) { + printf("enable_latency_tracking_complete failed\n"); + } + g_outstanding_commands--; +} + +static void +set_latency_tracking_feature(struct spdk_nvme_ctrlr *ctrlr, bool enable) +{ + int res; + union spdk_nvme_intel_feat_latency_tracking latency_tracking; + + if (enable) { + latency_tracking.bits.enable = 0x01; + } else { + latency_tracking.bits.enable = 0x00; + } + + res = spdk_nvme_ctrlr_cmd_set_feature(ctrlr, SPDK_NVME_INTEL_FEAT_LATENCY_TRACKING, + latency_tracking.raw, 0, NULL, 0, enable_latency_tracking_complete, NULL); + if (res) { + printf("fail to allocate nvme request.\n"); + return; + } + g_outstanding_commands++; + + while (g_outstanding_commands) { + spdk_nvme_ctrlr_process_admin_completions(ctrlr); + } +} + +static void +register_ctrlr(struct spdk_nvme_ctrlr *ctrlr, struct trid_entry *trid_entry) +{ + struct spdk_nvme_ns *ns; + struct ctrlr_entry *entry = malloc(sizeof(struct ctrlr_entry)); + uint32_t nsid; + + if (entry == NULL) { + perror("ctrlr_entry malloc"); + exit(1); + } + + entry->latency_page = spdk_dma_zmalloc(sizeof(struct spdk_nvme_intel_rw_latency_page), + 4096, NULL); + if (entry->latency_page == NULL) { + printf("Allocation error (latency page)\n"); + exit(1); + } + + build_nvme_name(entry->name, sizeof(entry->name), ctrlr); + + entry->ctrlr = ctrlr; + entry->trtype = trid_entry->trid.trtype; + entry->next = g_controllers; + g_controllers = entry; + + if (g_latency_ssd_tracking_enable && + spdk_nvme_ctrlr_is_feature_supported(ctrlr, SPDK_NVME_INTEL_FEAT_LATENCY_TRACKING)) { + set_latency_tracking_feature(ctrlr, true); + } + + if (trid_entry->nsid == 0) { + for (nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr); + nsid != 0; nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid)) { + ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); + if (ns == NULL) { + continue; + } + register_ns(ctrlr, ns); + } + } else { + ns = spdk_nvme_ctrlr_get_ns(ctrlr, trid_entry->nsid); + if (!ns) { + perror("Namespace does not exist."); + exit(1); + } + + register_ns(ctrlr, ns); + } +} + +static __thread unsigned int seed = 0; + +static inline void +submit_single_io(struct perf_task *task) +{ + uint64_t offset_in_ios; + int rc; + struct ns_worker_ctx *ns_ctx = task->ns_ctx; + struct ns_entry *entry = ns_ctx->entry; + + if (g_is_random) { + offset_in_ios = rand_r(&seed) % entry->size_in_ios; + } else { + offset_in_ios = ns_ctx->offset_in_ios++; + if (ns_ctx->offset_in_ios == entry->size_in_ios) { + ns_ctx->offset_in_ios = 0; + } + } + + task->submit_tsc = spdk_get_ticks(); + + if ((g_rw_percentage == 100) || + (g_rw_percentage != 0 && ((rand_r(&seed) % 100) < g_rw_percentage))) { + task->is_read = true; + } else { + task->is_read = false; + } + + rc = entry->fn_table->submit_io(task, ns_ctx, entry, offset_in_ios); + + if (spdk_unlikely(rc != 0)) { + fprintf(stderr, "starting I/O failed\n"); + } else { + ns_ctx->current_queue_depth++; + } +} + +static inline void +task_complete(struct perf_task *task) +{ + struct ns_worker_ctx *ns_ctx; + uint64_t tsc_diff; + struct ns_entry *entry; + + ns_ctx = task->ns_ctx; + entry = ns_ctx->entry; + ns_ctx->current_queue_depth--; + ns_ctx->io_completed++; + tsc_diff = spdk_get_ticks() - task->submit_tsc; + ns_ctx->total_tsc += tsc_diff; + if (spdk_unlikely(ns_ctx->min_tsc > tsc_diff)) { + ns_ctx->min_tsc = tsc_diff; + } + if (spdk_unlikely(ns_ctx->max_tsc < tsc_diff)) { + ns_ctx->max_tsc = tsc_diff; + } + if (spdk_unlikely(g_latency_sw_tracking_level > 0)) { + spdk_histogram_data_tally(ns_ctx->histogram, tsc_diff); + } + + if (spdk_unlikely(entry->md_size > 0)) { + /* add application level verification for end-to-end data protection */ + entry->fn_table->verify_io(task, entry); + } + + /* + * is_draining indicates when time has expired for the test run + * and we are just waiting for the previously submitted I/O + * to complete. In this case, do not submit a new I/O to replace + * the one just completed. + */ + if (spdk_unlikely(ns_ctx->is_draining)) { + spdk_dma_free(task->iov.iov_base); + spdk_dma_free(task->md_iov.iov_base); + free(task); + } else { + submit_single_io(task); + } +} + +static void +io_complete(void *ctx, const struct spdk_nvme_cpl *cpl) +{ + struct perf_task *task = ctx; + + if (spdk_unlikely(spdk_nvme_cpl_is_error(cpl))) { + fprintf(stderr, "%s completed with error (sct=%d, sc=%d)\n", + task->is_read ? "Read" : "Write", + cpl->status.sct, cpl->status.sc); + } + + task_complete(task); +} + +static struct perf_task * +allocate_task(struct ns_worker_ctx *ns_ctx, int queue_depth) +{ + struct perf_task *task; + + task = calloc(1, sizeof(*task)); + if (task == NULL) { + fprintf(stderr, "Out of memory allocating tasks\n"); + exit(1); + } + + ns_ctx->entry->fn_table->setup_payload(task, queue_depth % 8 + 1); + + task->ns_ctx = ns_ctx; + + return task; +} + +static void +submit_io(struct ns_worker_ctx *ns_ctx, int queue_depth) +{ + struct perf_task *task; + + while (queue_depth-- > 0) { + task = allocate_task(ns_ctx, queue_depth); + submit_single_io(task); + } +} + +static int +init_ns_worker_ctx(struct ns_worker_ctx *ns_ctx) +{ + return ns_ctx->entry->fn_table->init_ns_worker_ctx(ns_ctx); +} + +static void +cleanup_ns_worker_ctx(struct ns_worker_ctx *ns_ctx) +{ + ns_ctx->entry->fn_table->cleanup_ns_worker_ctx(ns_ctx); +} + +static void +print_periodic_performance(void) +{ + uint64_t io_this_second; + double mb_this_second; + struct worker_thread *worker; + struct ns_worker_ctx *ns_ctx; + + if (!isatty(STDOUT_FILENO)) { + /* Don't print periodic stats if output is not going + * to a terminal. + */ + return; + } + + io_this_second = 0; + worker = g_workers; + while (worker) { + ns_ctx = worker->ns_ctx; + while (ns_ctx) { + io_this_second += ns_ctx->io_completed - ns_ctx->last_io_completed; + ns_ctx->last_io_completed = ns_ctx->io_completed; + ns_ctx = ns_ctx->next; + } + worker = worker->next; + } + + mb_this_second = (double)io_this_second * g_io_size_bytes / (1024 * 1024); + printf("%9ju IOPS, %8.2f MiB/s\r", io_this_second, mb_this_second); + fflush(stdout); +} + +static int +work_fn(void *arg) +{ + uint64_t tsc_end, tsc_current, tsc_next_print; + struct worker_thread *worker = (struct worker_thread *)arg; + struct ns_worker_ctx *ns_ctx = NULL; + uint32_t unfinished_ns_ctx; + + /* Allocate queue pairs for each namespace. */ + ns_ctx = worker->ns_ctx; + while (ns_ctx != NULL) { + if (init_ns_worker_ctx(ns_ctx) != 0) { + printf("ERROR: init_ns_worker_ctx() failed\n"); + return 1; + } + ns_ctx = ns_ctx->next; + } + + tsc_current = spdk_get_ticks(); + tsc_end = tsc_current + g_time_in_sec * g_tsc_rate; + tsc_next_print = tsc_current + g_tsc_rate; + + /* Submit initial I/O for each namespace. */ + ns_ctx = worker->ns_ctx; + while (ns_ctx != NULL) { + submit_io(ns_ctx, g_queue_depth); + ns_ctx = ns_ctx->next; + } + + while (1) { + /* + * Check for completed I/O for each controller. A new + * I/O will be submitted in the io_complete callback + * to replace each I/O that is completed. + */ + ns_ctx = worker->ns_ctx; + while (ns_ctx != NULL) { + ns_ctx->entry->fn_table->check_io(ns_ctx); + ns_ctx = ns_ctx->next; + } + + tsc_current = spdk_get_ticks(); + + if (worker->lcore == g_master_core && tsc_current > tsc_next_print) { + tsc_next_print += g_tsc_rate; + print_periodic_performance(); + } + + if (tsc_current > tsc_end) { + break; + } + } + + /* drain the io of each ns_ctx in round robin to make the fairness */ + do { + unfinished_ns_ctx = 0; + ns_ctx = worker->ns_ctx; + while (ns_ctx != NULL) { + /* first time will enter into this if case */ + if (!ns_ctx->is_draining) { + ns_ctx->is_draining = true; + } + + if (ns_ctx->current_queue_depth > 0) { + ns_ctx->entry->fn_table->check_io(ns_ctx); + if (ns_ctx->current_queue_depth == 0) { + cleanup_ns_worker_ctx(ns_ctx); + } else { + unfinished_ns_ctx++; + } + } + ns_ctx = ns_ctx->next; + } + } while (unfinished_ns_ctx > 0); + + return 0; +} + +static void usage(char *program_name) +{ + printf("%s options", program_name); +#if defined(SPDK_CONFIG_URING) || defined(HAVE_LIBAIO) + printf(" [Kernel device(s)]..."); +#endif + printf("\n"); + printf("\t[-q io depth]\n"); + printf("\t[-o io size in bytes]\n"); + printf("\t[-P number of io queues per namespace. default: 1]\n"); + printf("\t[-U number of unused io queues per controller. default: 0]\n"); + printf("\t[-w io pattern type, must be one of\n"); + printf("\t\t(read, write, randread, randwrite, rw, randrw)]\n"); + printf("\t[-M rwmixread (100 for reads, 0 for writes)]\n"); + printf("\t[-L enable latency tracking via sw, default: disabled]\n"); + printf("\t\t-L for latency summary, -LL for detailed histogram\n"); + printf("\t[-l enable latency tracking via ssd (if supported), default: disabled]\n"); + printf("\t[-t time in seconds]\n"); + printf("\t[-c core mask for I/O submission/completion.]\n"); + printf("\t\t(default: 1)\n"); + printf("\t[-D disable submission queue in controller memory buffer, default: enabled]\n"); + printf("\t[-H enable header digest for TCP transport, default: disabled]\n"); + printf("\t[-I enable data digest for TCP transport, default: disabled]\n"); + printf("\t[-N no shutdown notification process for controllers, default: disabled]\n"); + printf("\t[-r Transport ID for local PCIe NVMe or NVMeoF]\n"); + printf("\t Format: 'key:value [key:value] ...'\n"); + printf("\t Keys:\n"); + printf("\t trtype Transport type (e.g. PCIe, RDMA)\n"); + printf("\t adrfam Address family (e.g. IPv4, IPv6)\n"); + printf("\t traddr Transport address (e.g. 0000:04:00.0 for PCIe or 192.168.100.8 for RDMA)\n"); + printf("\t trsvcid Transport service identifier (e.g. 4420)\n"); + printf("\t subnqn Subsystem NQN (default: %s)\n", SPDK_NVMF_DISCOVERY_NQN); + printf("\t Example: -r 'trtype:PCIe traddr:0000:04:00.0' for PCIe or\n"); + printf("\t -r 'trtype:RDMA adrfam:IPv4 traddr:192.168.100.8 trsvcid:4420' for NVMeoF\n"); + printf("\t[-e metadata configuration]\n"); + printf("\t Keys:\n"); + printf("\t PRACT Protection Information Action bit (PRACT=1 or PRACT=0)\n"); + printf("\t PRCHK Control of Protection Information Checking (PRCHK=GUARD|REFTAG|APPTAG)\n"); + printf("\t Example: -e 'PRACT=0,PRCHK=GUARD|REFTAG|APPTAG'\n"); + printf("\t -e 'PRACT=1,PRCHK=GUARD'\n"); + printf("\t[-k keep alive timeout period in millisecond]\n"); + printf("\t[-s DPDK huge memory size in MB.]\n"); + printf("\t[-C max completions per poll]\n"); + printf("\t\t(default: 0 - unlimited)\n"); + printf("\t[-i shared memory group ID]\n"); + printf("\t"); + spdk_log_usage(stdout, "-T"); +#ifdef SPDK_CONFIG_URING + printf("\t[-R enable using liburing to drive kernel devices (Default: libaio)]\n"); +#endif +#ifdef DEBUG + printf("\t[-G enable debug logging]\n"); +#else + printf("\t[-G enable debug logging (flag disabled, must reconfigure with --enable-debug)\n"); +#endif +} + +static void +check_cutoff(void *ctx, uint64_t start, uint64_t end, uint64_t count, + uint64_t total, uint64_t so_far) +{ + double so_far_pct; + double **cutoff = ctx; + + if (count == 0) { + return; + } + + so_far_pct = (double)so_far / total; + while (so_far_pct >= **cutoff && **cutoff > 0) { + printf("%9.5f%% : %9.3fus\n", **cutoff * 100, (double)end * 1000 * 1000 / g_tsc_rate); + (*cutoff)++; + } +} + +static void +print_bucket(void *ctx, uint64_t start, uint64_t end, uint64_t count, + uint64_t total, uint64_t so_far) +{ + double so_far_pct; + + if (count == 0) { + return; + } + + so_far_pct = (double)so_far * 100 / total; + printf("%9.3f - %9.3f: %9.4f%% (%9ju)\n", + (double)start * 1000 * 1000 / g_tsc_rate, + (double)end * 1000 * 1000 / g_tsc_rate, + so_far_pct, count); +} + +static void +print_performance(void) +{ + uint64_t total_io_completed, total_io_tsc; + double io_per_second, mb_per_second, average_latency, min_latency, max_latency; + double sum_ave_latency, min_latency_so_far, max_latency_so_far; + double total_io_per_second, total_mb_per_second; + int ns_count; + struct worker_thread *worker; + struct ns_worker_ctx *ns_ctx; + uint32_t max_strlen; + + total_io_per_second = 0; + total_mb_per_second = 0; + total_io_completed = 0; + total_io_tsc = 0; + min_latency_so_far = (double)UINT64_MAX; + max_latency_so_far = 0; + ns_count = 0; + + max_strlen = 0; + worker = g_workers; + while (worker) { + ns_ctx = worker->ns_ctx; + while (ns_ctx) { + max_strlen = spdk_max(strlen(ns_ctx->entry->name), max_strlen); + ns_ctx = ns_ctx->next; + } + worker = worker->next; + } + + printf("========================================================\n"); + printf("%*s\n", max_strlen + 60, "Latency(us)"); + printf("%-*s: %10s %10s %10s %10s %10s\n", + max_strlen + 13, "Device Information", "IOPS", "MiB/s", "Average", "min", "max"); + + worker = g_workers; + while (worker) { + ns_ctx = worker->ns_ctx; + while (ns_ctx) { + if (ns_ctx->io_completed != 0) { + io_per_second = (double)ns_ctx->io_completed / g_time_in_sec; + mb_per_second = io_per_second * g_io_size_bytes / (1024 * 1024); + average_latency = ((double)ns_ctx->total_tsc / ns_ctx->io_completed) * 1000 * 1000 / g_tsc_rate; + min_latency = (double)ns_ctx->min_tsc * 1000 * 1000 / g_tsc_rate; + if (min_latency < min_latency_so_far) { + min_latency_so_far = min_latency; + } + + max_latency = (double)ns_ctx->max_tsc * 1000 * 1000 / g_tsc_rate; + if (max_latency > max_latency_so_far) { + max_latency_so_far = max_latency; + } + + printf("%-*.*s from core %2u: %10.2f %10.2f %10.2f %10.2f %10.2f\n", + max_strlen, max_strlen, ns_ctx->entry->name, worker->lcore, + io_per_second, mb_per_second, + average_latency, min_latency, max_latency); + total_io_per_second += io_per_second; + total_mb_per_second += mb_per_second; + total_io_completed += ns_ctx->io_completed; + total_io_tsc += ns_ctx->total_tsc; + ns_count++; + } + ns_ctx = ns_ctx->next; + } + worker = worker->next; + } + + if (ns_count != 0 && total_io_completed) { + sum_ave_latency = ((double)total_io_tsc / total_io_completed) * 1000 * 1000 / g_tsc_rate; + printf("========================================================\n"); + printf("%-*s: %10.2f %10.2f %10.2f %10.2f %10.2f\n", + max_strlen + 13, "Total", total_io_per_second, total_mb_per_second, + sum_ave_latency, min_latency_so_far, max_latency_so_far); + printf("\n"); + } + + if (g_latency_sw_tracking_level == 0 || total_io_completed == 0) { + return; + } + + worker = g_workers; + while (worker) { + ns_ctx = worker->ns_ctx; + while (ns_ctx) { + const double *cutoff = g_latency_cutoffs; + + printf("Summary latency data for %-43.43s from core %u:\n", ns_ctx->entry->name, worker->lcore); + printf("=================================================================================\n"); + + spdk_histogram_data_iterate(ns_ctx->histogram, check_cutoff, &cutoff); + + printf("\n"); + ns_ctx = ns_ctx->next; + } + worker = worker->next; + } + + if (g_latency_sw_tracking_level == 1) { + return; + } + + worker = g_workers; + while (worker) { + ns_ctx = worker->ns_ctx; + while (ns_ctx) { + printf("Latency histogram for %-43.43s from core %u:\n", ns_ctx->entry->name, worker->lcore); + printf("==============================================================================\n"); + printf(" Range in us Cumulative IO count\n"); + + spdk_histogram_data_iterate(ns_ctx->histogram, print_bucket, NULL); + printf("\n"); + ns_ctx = ns_ctx->next; + } + worker = worker->next; + } + +} + +static void +print_latency_page(struct ctrlr_entry *entry) +{ + int i; + + printf("\n"); + printf("%s\n", entry->name); + printf("--------------------------------------------------------\n"); + + for (i = 0; i < 32; i++) { + if (entry->latency_page->buckets_32us[i]) { + printf("Bucket %dus - %dus: %d\n", i * 32, (i + 1) * 32, entry->latency_page->buckets_32us[i]); + } + } + for (i = 0; i < 31; i++) { + if (entry->latency_page->buckets_1ms[i]) { + printf("Bucket %dms - %dms: %d\n", i + 1, i + 2, entry->latency_page->buckets_1ms[i]); + } + } + for (i = 0; i < 31; i++) { + if (entry->latency_page->buckets_32ms[i]) + printf("Bucket %dms - %dms: %d\n", (i + 1) * 32, (i + 2) * 32, + entry->latency_page->buckets_32ms[i]); + } +} + +static void +print_latency_statistics(const char *op_name, enum spdk_nvme_intel_log_page log_page) +{ + struct ctrlr_entry *ctrlr; + + printf("%s Latency Statistics:\n", op_name); + printf("========================================================\n"); + ctrlr = g_controllers; + while (ctrlr) { + if (spdk_nvme_ctrlr_is_log_page_supported(ctrlr->ctrlr, log_page)) { + if (spdk_nvme_ctrlr_cmd_get_log_page(ctrlr->ctrlr, log_page, SPDK_NVME_GLOBAL_NS_TAG, + ctrlr->latency_page, sizeof(struct spdk_nvme_intel_rw_latency_page), 0, + enable_latency_tracking_complete, + NULL)) { + printf("nvme_ctrlr_cmd_get_log_page() failed\n"); + exit(1); + } + + g_outstanding_commands++; + } else { + printf("Controller %s: %s latency statistics not supported\n", ctrlr->name, op_name); + } + ctrlr = ctrlr->next; + } + + while (g_outstanding_commands) { + ctrlr = g_controllers; + while (ctrlr) { + spdk_nvme_ctrlr_process_admin_completions(ctrlr->ctrlr); + ctrlr = ctrlr->next; + } + } + + ctrlr = g_controllers; + while (ctrlr) { + if (spdk_nvme_ctrlr_is_log_page_supported(ctrlr->ctrlr, log_page)) { + print_latency_page(ctrlr); + } + ctrlr = ctrlr->next; + } + printf("\n"); +} + +static void +print_stats(void) +{ + print_performance(); + if (g_latency_ssd_tracking_enable) { + if (g_rw_percentage != 0) { + print_latency_statistics("Read", SPDK_NVME_INTEL_LOG_READ_CMD_LATENCY); + } + if (g_rw_percentage != 100) { + print_latency_statistics("Write", SPDK_NVME_INTEL_LOG_WRITE_CMD_LATENCY); + } + } +} + +static void +unregister_trids(void) +{ + struct trid_entry *trid_entry, *tmp; + + TAILQ_FOREACH_SAFE(trid_entry, &g_trid_list, tailq, tmp) { + TAILQ_REMOVE(&g_trid_list, trid_entry, tailq); + free(trid_entry); + } +} + +static int +add_trid(const char *trid_str) +{ + struct trid_entry *trid_entry; + struct spdk_nvme_transport_id *trid; + char *ns; + + trid_entry = calloc(1, sizeof(*trid_entry)); + if (trid_entry == NULL) { + return -1; + } + + trid = &trid_entry->trid; + trid->trtype = SPDK_NVME_TRANSPORT_PCIE; + snprintf(trid->subnqn, sizeof(trid->subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN); + + if (spdk_nvme_transport_id_parse(trid, trid_str) != 0) { + fprintf(stderr, "Invalid transport ID format '%s'\n", trid_str); + free(trid_entry); + return 1; + } + + spdk_nvme_transport_id_populate_trstring(trid, + spdk_nvme_transport_id_trtype_str(trid->trtype)); + + ns = strcasestr(trid_str, "ns:"); + if (ns) { + char nsid_str[6]; /* 5 digits maximum in an nsid */ + int len; + int nsid; + + ns += 3; + + len = strcspn(ns, " \t\n"); + if (len > 5) { + fprintf(stderr, "NVMe namespace IDs must be 5 digits or less\n"); + free(trid_entry); + return 1; + } + + memcpy(nsid_str, ns, len); + nsid_str[len] = '\0'; + + nsid = spdk_strtol(nsid_str, 10); + if (nsid <= 0 || nsid > 65535) { + fprintf(stderr, "NVMe namespace IDs must be less than 65536 and greater than 0\n"); + free(trid_entry); + return 1; + } + + trid_entry->nsid = (uint16_t)nsid; + } + + TAILQ_INSERT_TAIL(&g_trid_list, trid_entry, tailq); + return 0; +} + +static size_t +parse_next_key(const char **str, char *key, char *val, size_t key_buf_size, + size_t val_buf_size) +{ + const char *sep; + const char *separator = ", \t\n"; + size_t key_len, val_len; + + *str += strspn(*str, separator); + + sep = strchr(*str, '='); + if (!sep) { + fprintf(stderr, "Key without '=' separator\n"); + return 0; + } + + key_len = sep - *str; + if (key_len >= key_buf_size) { + fprintf(stderr, "Key length %zu is greater than maximum allowed %zu\n", + key_len, key_buf_size - 1); + return 0; + } + + memcpy(key, *str, key_len); + key[key_len] = '\0'; + + *str += key_len + 1; /* Skip key */ + val_len = strcspn(*str, separator); + if (val_len == 0) { + fprintf(stderr, "Key without value\n"); + return 0; + } + + if (val_len >= val_buf_size) { + fprintf(stderr, "Value length %zu is greater than maximum allowed %zu\n", + val_len, val_buf_size - 1); + return 0; + } + + memcpy(val, *str, val_len); + val[val_len] = '\0'; + + *str += val_len; + + return val_len; +} + +static int +parse_metadata(const char *metacfg_str) +{ + const char *str; + size_t val_len; + char key[32]; + char val[1024]; + + if (metacfg_str == NULL) { + return -EINVAL; + } + + str = metacfg_str; + + while (*str != '\0') { + val_len = parse_next_key(&str, key, val, sizeof(key), sizeof(val)); + if (val_len == 0) { + fprintf(stderr, "Failed to parse metadata\n"); + return -EINVAL; + } + + if (strcmp(key, "PRACT") == 0) { + if (*val == '1') { + g_metacfg_pract_flag = SPDK_NVME_IO_FLAGS_PRACT; + } + } else if (strcmp(key, "PRCHK") == 0) { + if (strstr(val, "GUARD") != NULL) { + g_metacfg_prchk_flags |= SPDK_NVME_IO_FLAGS_PRCHK_GUARD; + } + if (strstr(val, "REFTAG") != NULL) { + g_metacfg_prchk_flags |= SPDK_NVME_IO_FLAGS_PRCHK_REFTAG; + } + if (strstr(val, "APPTAG") != NULL) { + g_metacfg_prchk_flags |= SPDK_NVME_IO_FLAGS_PRCHK_APPTAG; + } + } else { + fprintf(stderr, "Unknown key '%s'\n", key); + } + } + + return 0; +} + +static int +parse_args(int argc, char **argv) +{ + int op; + long int val; + int rc; + + while ((op = getopt(argc, argv, "c:e:i:lo:q:r:k:s:t:w:C:DGHILM:NP:RT:U:V")) != -1) { + switch (op) { + case 'i': + case 'C': + case 'P': + case 'o': + case 'q': + case 'k': + case 's': + case 't': + case 'M': + case 'U': + val = spdk_strtol(optarg, 10); + if (val < 0) { + fprintf(stderr, "Converting a string to integer failed\n"); + return val; + } + switch (op) { + case 'i': + g_shm_id = val; + break; + case 'C': + g_max_completions = val; + break; + case 'P': + g_nr_io_queues_per_ns = val; + break; + case 'o': + g_io_size_bytes = val; + break; + case 'q': + g_queue_depth = val; + break; + case 'k': + g_keep_alive_timeout_in_ms = val; + break; + case 's': + g_dpdk_mem = val; + break; + case 't': + g_time_in_sec = val; + break; + case 'M': + g_rw_percentage = val; + g_mix_specified = true; + break; + case 'U': + g_nr_unused_io_queues = val; + break; + } + break; + case 'c': + g_core_mask = optarg; + break; + case 'e': + if (parse_metadata(optarg)) { + usage(argv[0]); + return 1; + } + break; + case 'l': + g_latency_ssd_tracking_enable = true; + break; + case 'r': + if (add_trid(optarg)) { + usage(argv[0]); + return 1; + } + break; + case 'w': + g_workload_type = optarg; + break; + case 'D': + g_disable_sq_cmb = 1; + break; + case 'G': +#ifndef DEBUG + fprintf(stderr, "%s must be configured with --enable-debug for -G flag\n", + argv[0]); + usage(argv[0]); + return 1; +#else + spdk_log_set_flag("nvme"); + spdk_log_set_print_level(SPDK_LOG_DEBUG); + break; +#endif + case 'H': + g_header_digest = 1; + break; + case 'I': + g_data_digest = 1; + break; + case 'L': + g_latency_sw_tracking_level++; + break; + case 'N': + g_no_shn_notification = true; + break; + case 'R': +#ifndef SPDK_CONFIG_URING + fprintf(stderr, "%s must be rebuilt with CONFIG_URING=y for -R flag.\n", + argv[0]); + usage(argv[0]); + return 0; +#endif + g_use_uring = true; + break; + case 'T': + rc = spdk_log_set_flag(optarg); + if (rc < 0) { + fprintf(stderr, "unknown flag\n"); + usage(argv[0]); + exit(EXIT_FAILURE); + } + spdk_log_set_print_level(SPDK_LOG_DEBUG); +#ifndef DEBUG + fprintf(stderr, "%s must be rebuilt with CONFIG_DEBUG=y for -T flag.\n", + argv[0]); + usage(argv[0]); + return 0; +#endif + break; + case 'V': + g_vmd = true; + break; + default: + usage(argv[0]); + return 1; + } + } + + if (!g_nr_io_queues_per_ns) { + usage(argv[0]); + return 1; + } + + if (!g_queue_depth) { + fprintf(stderr, "missing -q (queue size) operand\n"); + usage(argv[0]); + return 1; + } + if (!g_io_size_bytes) { + fprintf(stderr, "missing -o (block size) operand\n"); + usage(argv[0]); + return 1; + } + if (!g_workload_type) { + fprintf(stderr, "missing -w (io pattern type) operand\n"); + usage(argv[0]); + return 1; + } + if (!g_time_in_sec) { + fprintf(stderr, "missing -t (test time in seconds) operand\n"); + usage(argv[0]); + return 1; + } + + if (strncmp(g_workload_type, "rand", 4) == 0) { + g_is_random = 1; + g_workload_type = &g_workload_type[4]; + } + + if (strcmp(g_workload_type, "read") == 0 || strcmp(g_workload_type, "write") == 0) { + g_rw_percentage = strcmp(g_workload_type, "read") == 0 ? 100 : 0; + if (g_mix_specified) { + fprintf(stderr, "Ignoring -M option... Please use -M option" + " only when using rw or randrw.\n"); + } + } else if (strcmp(g_workload_type, "rw") == 0) { + if (g_rw_percentage < 0 || g_rw_percentage > 100) { + fprintf(stderr, + "-M must be specified to value from 0 to 100 " + "for rw or randrw.\n"); + return 1; + } + } else { + fprintf(stderr, + "io pattern type must be one of\n" + "(read, write, randread, randwrite, rw, randrw)\n"); + return 1; + } + + if (TAILQ_EMPTY(&g_trid_list)) { + /* If no transport IDs specified, default to enumerating all local PCIe devices */ + add_trid("trtype:PCIe"); + } else { + struct trid_entry *trid_entry, *trid_entry_tmp; + + g_no_pci = true; + /* check whether there is local PCIe type */ + TAILQ_FOREACH_SAFE(trid_entry, &g_trid_list, tailq, trid_entry_tmp) { + if (trid_entry->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) { + g_no_pci = false; + break; + } + } + } + + g_file_optind = optind; + + return 0; +} + +static int +register_workers(void) +{ + uint32_t i; + struct worker_thread *worker; + + g_workers = NULL; + g_num_workers = 0; + + SPDK_ENV_FOREACH_CORE(i) { + worker = calloc(1, sizeof(*worker)); + if (worker == NULL) { + fprintf(stderr, "Unable to allocate worker\n"); + return -1; + } + + worker->lcore = i; + worker->next = g_workers; + g_workers = worker; + g_num_workers++; + } + + return 0; +} + +static void +unregister_workers(void) +{ + struct worker_thread *worker = g_workers; + + /* Free namespace context and worker thread */ + while (worker) { + struct worker_thread *next_worker = worker->next; + struct ns_worker_ctx *ns_ctx = worker->ns_ctx; + + while (ns_ctx) { + struct ns_worker_ctx *next_ns_ctx = ns_ctx->next; + spdk_histogram_data_free(ns_ctx->histogram); + free(ns_ctx); + ns_ctx = next_ns_ctx; + } + + free(worker); + worker = next_worker; + } +} + +static bool +probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, + struct spdk_nvme_ctrlr_opts *opts) +{ + if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) { + if (g_disable_sq_cmb) { + opts->use_cmb_sqs = false; + } + if (g_no_shn_notification) { + opts->no_shn_notification = true; + } + } + + /* Set io_queue_size to UINT16_MAX, NVMe driver + * will then reduce this to MQES to maximize + * the io_queue_size as much as possible. + */ + opts->io_queue_size = UINT16_MAX; + + /* Set the header and data_digest */ + opts->header_digest = g_header_digest; + opts->data_digest = g_data_digest; + opts->keep_alive_timeout_ms = g_keep_alive_timeout_in_ms; + + return true; +} + +static void +attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, + struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts) +{ + struct trid_entry *trid_entry = cb_ctx; + struct spdk_pci_addr pci_addr; + struct spdk_pci_device *pci_dev; + struct spdk_pci_id pci_id; + + if (trid->trtype != SPDK_NVME_TRANSPORT_PCIE) { + printf("Attached to NVMe over Fabrics controller at %s:%s: %s\n", + trid->traddr, trid->trsvcid, + trid->subnqn); + } else { + if (spdk_pci_addr_parse(&pci_addr, trid->traddr)) { + return; + } + + pci_dev = spdk_nvme_ctrlr_get_pci_device(ctrlr); + if (!pci_dev) { + return; + } + + pci_id = spdk_pci_device_get_id(pci_dev); + + printf("Attached to NVMe Controller at %s [%04x:%04x]\n", + trid->traddr, + pci_id.vendor_id, pci_id.device_id); + } + + register_ctrlr(ctrlr, trid_entry); +} + +static int +register_controllers(void) +{ + struct trid_entry *trid_entry; + + printf("Initializing NVMe Controllers\n"); + + if (g_vmd && spdk_vmd_init()) { + fprintf(stderr, "Failed to initialize VMD." + " Some NVMe devices can be unavailable.\n"); + } + + TAILQ_FOREACH(trid_entry, &g_trid_list, tailq) { + if (spdk_nvme_probe(&trid_entry->trid, trid_entry, probe_cb, attach_cb, NULL) != 0) { + fprintf(stderr, "spdk_nvme_probe() failed for transport address '%s'\n", + trid_entry->trid.traddr); + return -1; + } + } + + return 0; +} + +static void +unregister_controllers(void) +{ + struct ctrlr_entry *entry = g_controllers; + + while (entry) { + struct ctrlr_entry *next = entry->next; + spdk_dma_free(entry->latency_page); + if (g_latency_ssd_tracking_enable && + spdk_nvme_ctrlr_is_feature_supported(entry->ctrlr, SPDK_NVME_INTEL_FEAT_LATENCY_TRACKING)) { + set_latency_tracking_feature(entry->ctrlr, false); + } + + if (g_nr_unused_io_queues) { + int i; + + for (i = 0; i < g_nr_unused_io_queues; i++) { + spdk_nvme_ctrlr_free_io_qpair(entry->unused_qpairs[i]); + } + + free(entry->unused_qpairs); + } + + spdk_nvme_detach(entry->ctrlr); + free(entry); + entry = next; + } + + if (g_vmd) { + spdk_vmd_fini(); + } +} + +static int +associate_workers_with_ns(void) +{ + struct ns_entry *entry = g_namespaces; + struct worker_thread *worker = g_workers; + struct ns_worker_ctx *ns_ctx; + int i, count; + + count = g_num_namespaces > g_num_workers ? g_num_namespaces : g_num_workers; + + for (i = 0; i < count; i++) { + if (entry == NULL) { + break; + } + + ns_ctx = calloc(1, sizeof(struct ns_worker_ctx)); + if (!ns_ctx) { + return -1; + } + + printf("Associating %s with lcore %d\n", entry->name, worker->lcore); + ns_ctx->min_tsc = UINT64_MAX; + ns_ctx->entry = entry; + ns_ctx->next = worker->ns_ctx; + ns_ctx->histogram = spdk_histogram_data_alloc(); + worker->ns_ctx = ns_ctx; + + worker = worker->next; + if (worker == NULL) { + worker = g_workers; + } + + entry = entry->next; + if (entry == NULL) { + entry = g_namespaces; + } + + } + + return 0; +} + +static void * +nvme_poll_ctrlrs(void *arg) +{ + struct ctrlr_entry *entry; + int oldstate; + + spdk_unaffinitize_thread(); + + while (true) { + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate); + + entry = g_controllers; + while (entry) { + if (entry->trtype != SPDK_NVME_TRANSPORT_PCIE) { + spdk_nvme_ctrlr_process_admin_completions(entry->ctrlr); + } + entry = entry->next; + } + + pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &oldstate); + + /* This is a pthread cancellation point and cannot be removed. */ + sleep(1); + } + + return NULL; +} + +int main(int argc, char **argv) +{ + int rc; + struct worker_thread *worker, *master_worker; + struct spdk_env_opts opts; + pthread_t thread_id = 0; + + rc = parse_args(argc, argv); + if (rc != 0) { + return rc; + } + + spdk_env_opts_init(&opts); + opts.name = "perf"; + opts.shm_id = g_shm_id; + if (g_core_mask) { + opts.core_mask = g_core_mask; + } + + if (g_dpdk_mem) { + opts.mem_size = g_dpdk_mem; + } + if (g_no_pci) { + opts.no_pci = g_no_pci; + } + if (spdk_env_init(&opts) < 0) { + fprintf(stderr, "Unable to initialize SPDK env\n"); + rc = -1; + goto cleanup; + } + + g_tsc_rate = spdk_get_ticks_hz(); + + if (register_workers() != 0) { + rc = -1; + goto cleanup; + } + +#if defined(HAVE_LIBAIO) || defined(SPDK_CONFIG_URING) + if (register_files(argc, argv) != 0) { + rc = -1; + goto cleanup; + } +#endif + + if (register_controllers() != 0) { + rc = -1; + goto cleanup; + } + + if (g_warn) { + printf("WARNING: Some requested NVMe devices were skipped\n"); + } + + if (g_num_namespaces == 0) { + fprintf(stderr, "No valid NVMe controllers or AIO or URING devices found\n"); + goto cleanup; + } + + rc = pthread_create(&thread_id, NULL, &nvme_poll_ctrlrs, NULL); + if (rc != 0) { + fprintf(stderr, "Unable to spawn a thread to poll admin queues.\n"); + goto cleanup; + } + + if (associate_workers_with_ns() != 0) { + rc = -1; + goto cleanup; + } + + printf("Initialization complete. Launching workers.\n"); + + /* Launch all of the slave workers */ + g_master_core = spdk_env_get_current_core(); + master_worker = NULL; + worker = g_workers; + while (worker != NULL) { + if (worker->lcore != g_master_core) { + spdk_env_thread_launch_pinned(worker->lcore, work_fn, worker); + } else { + assert(master_worker == NULL); + master_worker = worker; + } + worker = worker->next; + } + + assert(master_worker != NULL); + rc = work_fn(master_worker); + + spdk_env_thread_wait_all(); + + print_stats(); + +cleanup: + if (thread_id && pthread_cancel(thread_id) == 0) { + pthread_join(thread_id, NULL); + } + unregister_trids(); + unregister_namespaces(); + unregister_controllers(); + unregister_workers(); + + if (rc != 0) { + fprintf(stderr, "%s: errors occured\n", argv[0]); + } + + return rc; +} |