diff options
Diffstat (limited to 'src/spdk/examples/bdev')
| -rw-r--r-- | src/spdk/examples/bdev/Makefile | 48 | ||||
| -rw-r--r-- | src/spdk/examples/bdev/fio_plugin/.gitignore | 1 | ||||
| -rw-r--r-- | src/spdk/examples/bdev/fio_plugin/Makefile | 60 | ||||
| -rw-r--r-- | src/spdk/examples/bdev/fio_plugin/README.md | 69 | ||||
| -rw-r--r-- | src/spdk/examples/bdev/fio_plugin/bdev.conf.in | 3 | ||||
| -rw-r--r-- | src/spdk/examples/bdev/fio_plugin/example_config.fio | 16 | ||||
| -rw-r--r-- | src/spdk/examples/bdev/fio_plugin/fio_plugin.c | 779 | ||||
| -rw-r--r-- | src/spdk/examples/bdev/fio_plugin/full_bench.fio | 20 | ||||
| -rw-r--r-- | src/spdk/examples/bdev/hello_world/.gitignore | 1 | ||||
| -rw-r--r-- | src/spdk/examples/bdev/hello_world/Makefile | 56 | ||||
| -rw-r--r-- | src/spdk/examples/bdev/hello_world/bdev.conf | 5 | ||||
| -rw-r--r-- | src/spdk/examples/bdev/hello_world/hello_bdev.c | 294 |
12 files changed, 1352 insertions, 0 deletions
diff --git a/src/spdk/examples/bdev/Makefile b/src/spdk/examples/bdev/Makefile new file mode 100644 index 00000000..dc1f5221 --- /dev/null +++ b/src/spdk/examples/bdev/Makefile @@ -0,0 +1,48 @@ +# +# BSD LICENSE +# +# Copyright (c) Intel Corporation. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..) +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk + +DIRS-$(CONFIG_FIO_PLUGIN) = fio_plugin +DIRS-y += hello_world + +.PHONY: all clean $(DIRS-y) + +all: $(DIRS-y) + @: + +clean: $(DIRS-y) + @: + +include $(SPDK_ROOT_DIR)/mk/spdk.subdirs.mk diff --git a/src/spdk/examples/bdev/fio_plugin/.gitignore b/src/spdk/examples/bdev/fio_plugin/.gitignore new file mode 100644 index 00000000..1b0b36ac --- /dev/null +++ b/src/spdk/examples/bdev/fio_plugin/.gitignore @@ -0,0 +1 @@ +fio_plugin diff --git a/src/spdk/examples/bdev/fio_plugin/Makefile b/src/spdk/examples/bdev/fio_plugin/Makefile new file mode 100644 index 00000000..d571c203 --- /dev/null +++ b/src/spdk/examples/bdev/fio_plugin/Makefile @@ -0,0 +1,60 @@ +# +# BSD LICENSE +# +# Copyright (c) Intel Corporation. +# Copyright (c) 2015-2016, Micron Technology, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..) +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk +include $(SPDK_ROOT_DIR)/mk/spdk.app.mk +include $(SPDK_ROOT_DIR)/mk/spdk.modules.mk + +APP := fio_plugin + +C_SRCS = fio_plugin.c +CFLAGS += -I$(CONFIG_FIO_SOURCE_DIR) +LDFLAGS += -shared -rdynamic + +SPDK_LIB_LIST += thread util bdev conf copy rpc jsonrpc json log trace + +LIBS += $(BLOCKDEV_MODULES_LINKER_ARGS) +LIBS += $(SPDK_LIB_LINKER_ARGS) $(ENV_LINKER_ARGS) + +all: $(APP) + @: + +$(APP) : $(OBJS) $(SPDK_LIB_FILES) $(ENV_LIBS) $(BLOCKDEV_MODULES_FILES) + $(LINK_C) + +clean: + $(CLEAN_C) $(APP) + +include $(SPDK_ROOT_DIR)/mk/spdk.deps.mk diff --git a/src/spdk/examples/bdev/fio_plugin/README.md b/src/spdk/examples/bdev/fio_plugin/README.md new file mode 100644 index 00000000..65d6ef6d --- /dev/null +++ b/src/spdk/examples/bdev/fio_plugin/README.md @@ -0,0 +1,69 @@ +# Compiling fio + +Clone the fio source repository from https://github.com/axboe/fio + + git clone https://github.com/axboe/fio + +Then check out the fio 3.3: + + cd fio && git checkout fio-3.3 + +Finally, compile the code: + + make + +# Compiling SPDK + +Clone the SPDK source repository from https://github.com/spdk/spdk + + git clone https://github.com/spdk/spdk + git submodule update --init + +Then, run the SPDK configure script to enable fio (point it to the root of the fio repository): + + cd spdk + ./configure --with-fio=/path/to/fio/repo <other configuration options> + +Finally, build SPDK: + + make + +**Note to advanced users**: These steps assume you're using the DPDK submodule. If you are using your +own version of DPDK, the fio plugin requires that DPDK be compiled with -fPIC. You can compile DPDK +with -fPIC by modifying your DPDK configuration file and adding the line: + + EXTRA_CFLAGS=-fPIC + +# Usage + +To use the SPDK fio plugin with fio, specify the plugin binary using LD_PRELOAD when running +fio and set ioengine=spdk_bdev in the fio configuration file (see example_config.fio in the same +directory as this README). + + LD_PRELOAD=<path to spdk repo>/examples/bdev/fio_plugin/fio_plugin fio + +The fio configuration file must contain one new parameter: + + spdk_conf=./examples/bdev/fio_plugin/bdev.conf + +This must point at an SPDK configuration file. There are a number of example configuration +files in the SPDK repository under etc/spdk. + +You can specify which block device to run against by setting the filename parameter +to the block device name: + + filename=Malloc0 + +Or for NVMe devices: + + filename=Nvme0n1 + +Currently the SPDK fio plugin is limited to the thread usage model, so fio jobs must also specify thread=1 +when using the SPDK fio plugin. + +fio also currently has a race condition on shutdown if dynamically loading the ioengine by specifying the +engine's full path via the ioengine parameter - LD_PRELOAD is recommended to avoid this race condition. + +When testing random workloads, it is recommended to set norandommap=1. fio's random map +processing consumes extra CPU cycles which will degrade performance over time with +the fio_plugin since all I/O are submitted and completed on a single CPU core. diff --git a/src/spdk/examples/bdev/fio_plugin/bdev.conf.in b/src/spdk/examples/bdev/fio_plugin/bdev.conf.in new file mode 100644 index 00000000..948cebe3 --- /dev/null +++ b/src/spdk/examples/bdev/fio_plugin/bdev.conf.in @@ -0,0 +1,3 @@ +[Malloc] + NumberOfLuns 1 + LunSizeInMB 128 diff --git a/src/spdk/examples/bdev/fio_plugin/example_config.fio b/src/spdk/examples/bdev/fio_plugin/example_config.fio new file mode 100644 index 00000000..3a35432e --- /dev/null +++ b/src/spdk/examples/bdev/fio_plugin/example_config.fio @@ -0,0 +1,16 @@ +[global] +ioengine=spdk_bdev +spdk_conf=./examples/bdev/fio_plugin/bdev.conf.in +thread=1 +group_reporting=1 +direct=1 +verify=0 +time_based=1 +ramp_time=0 +runtime=2 +iodepth=128 +rw=randrw +bs=4k + +[test] +numjobs=1 diff --git a/src/spdk/examples/bdev/fio_plugin/fio_plugin.c b/src/spdk/examples/bdev/fio_plugin/fio_plugin.c new file mode 100644 index 00000000..4406abd4 --- /dev/null +++ b/src/spdk/examples/bdev/fio_plugin/fio_plugin.c @@ -0,0 +1,779 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "spdk/stdinc.h" + +#include "spdk/bdev.h" +#include "spdk/copy_engine.h" +#include "spdk/conf.h" +#include "spdk/env.h" +#include "spdk/thread.h" +#include "spdk/log.h" +#include "spdk/string.h" +#include "spdk/queue.h" + +#include "config-host.h" +#include "fio.h" +#include "optgroup.h" + +struct spdk_fio_options { + void *pad; + char *conf; + unsigned mem_mb; + bool mem_single_seg; +}; + +/* Used to pass messages between fio threads */ +struct spdk_fio_msg { + spdk_thread_fn cb_fn; + void *cb_arg; +}; + +/* A polling function */ +struct spdk_fio_poller { + spdk_poller_fn cb_fn; + void *cb_arg; + uint64_t period_microseconds; + + TAILQ_ENTRY(spdk_fio_poller) link; +}; + +struct spdk_fio_request { + struct io_u *io; + struct thread_data *td; +}; + +struct spdk_fio_target { + struct spdk_bdev *bdev; + struct spdk_bdev_desc *desc; + struct spdk_io_channel *ch; + + TAILQ_ENTRY(spdk_fio_target) link; +}; + +struct spdk_fio_thread { + struct thread_data *td; /* fio thread context */ + struct spdk_thread *thread; /* spdk thread context */ + struct spdk_ring *ring; /* ring for passing messages to this thread */ + TAILQ_HEAD(, spdk_fio_poller) pollers; /* List of registered pollers on this thread */ + + TAILQ_HEAD(, spdk_fio_target) targets; + + struct io_u **iocq; // io completion queue + unsigned int iocq_count; // number of iocq entries filled by last getevents + unsigned int iocq_size; // number of iocq entries allocated +}; + +static struct spdk_fio_thread *g_init_thread = NULL; +static pthread_t g_init_thread_id = 0; +static bool g_spdk_env_initialized = false; + +static int spdk_fio_init(struct thread_data *td); +static void spdk_fio_cleanup(struct thread_data *td); +static size_t spdk_fio_poll_thread(struct spdk_fio_thread *fio_thread); + +static void +spdk_fio_send_msg(spdk_thread_fn fn, void *ctx, void *thread_ctx) +{ + struct spdk_fio_thread *thread = thread_ctx; + struct spdk_fio_msg *msg; + size_t count; + + msg = calloc(1, sizeof(*msg)); + assert(msg != NULL); + + msg->cb_fn = fn; + msg->cb_arg = ctx; + + count = spdk_ring_enqueue(thread->ring, (void **)&msg, 1); + if (count != 1) { + SPDK_ERRLOG("Unable to send message to thread %p. rc: %lu\n", thread, count); + } +} + +static void +spdk_fio_bdev_init_done(void *cb_arg, int rc) +{ + *(bool *)cb_arg = true; +} + +static struct spdk_poller * +spdk_fio_start_poller(void *thread_ctx, + spdk_poller_fn fn, + void *arg, + uint64_t period_microseconds) +{ + struct spdk_fio_thread *fio_thread = thread_ctx; + struct spdk_fio_poller *fio_poller; + + fio_poller = calloc(1, sizeof(*fio_poller)); + if (!fio_poller) { + SPDK_ERRLOG("Unable to allocate poller\n"); + return NULL; + } + + fio_poller->cb_fn = fn; + fio_poller->cb_arg = arg; + fio_poller->period_microseconds = period_microseconds; + + TAILQ_INSERT_TAIL(&fio_thread->pollers, fio_poller, link); + + return (struct spdk_poller *)fio_poller; +} + +static void +spdk_fio_stop_poller(struct spdk_poller *poller, void *thread_ctx) +{ + struct spdk_fio_poller *fio_poller; + struct spdk_fio_thread *fio_thread = thread_ctx; + + fio_poller = (struct spdk_fio_poller *)poller; + + TAILQ_REMOVE(&fio_thread->pollers, fio_poller, link); + + free(fio_poller); +} + +static int +spdk_fio_init_thread(struct thread_data *td) +{ + struct spdk_fio_thread *fio_thread; + + fio_thread = calloc(1, sizeof(*fio_thread)); + if (!fio_thread) { + SPDK_ERRLOG("failed to allocate thread local context\n"); + return -1; + } + + fio_thread->td = td; + td->io_ops_data = fio_thread; + + fio_thread->ring = spdk_ring_create(SPDK_RING_TYPE_MP_SC, 4096, SPDK_ENV_SOCKET_ID_ANY); + if (!fio_thread->ring) { + SPDK_ERRLOG("failed to allocate ring\n"); + free(fio_thread); + return -1; + } + + fio_thread->thread = spdk_allocate_thread(spdk_fio_send_msg, + spdk_fio_start_poller, + spdk_fio_stop_poller, + fio_thread, + "fio_thread"); + if (!fio_thread->thread) { + spdk_ring_free(fio_thread->ring); + free(fio_thread); + SPDK_ERRLOG("failed to allocate thread\n"); + return -1; + } + + TAILQ_INIT(&fio_thread->pollers); + + fio_thread->iocq_size = td->o.iodepth; + fio_thread->iocq = calloc(fio_thread->iocq_size, sizeof(struct io_u *)); + assert(fio_thread->iocq != NULL); + + TAILQ_INIT(&fio_thread->targets); + + return 0; +} + +static void * +spdk_init_thread_poll(void *arg) +{ + struct spdk_fio_thread *thread = arg; + int oldstate; + int rc; + + /* Loop until the thread is cancelled */ + while (true) { + rc = pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate); + if (rc != 0) { + SPDK_ERRLOG("Unable to set cancel state disabled on g_init_thread (%d): %s\n", + rc, spdk_strerror(rc)); + } + + spdk_fio_poll_thread(thread); + + rc = pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &oldstate); + if (rc != 0) { + SPDK_ERRLOG("Unable to set cancel state enabled on g_init_thread (%d): %s\n", + rc, spdk_strerror(rc)); + } + + /* This is a pthread cancellation point and cannot be removed. */ + sleep(1); + } + + return NULL; +} + +static int +spdk_fio_init_env(struct thread_data *td) +{ + struct spdk_fio_thread *fio_thread; + struct spdk_fio_options *eo; + bool done = false; + int rc; + struct spdk_conf *config; + struct spdk_env_opts opts; + size_t count; + + /* Parse the SPDK configuration file */ + eo = td->eo; + if (!eo->conf || !strlen(eo->conf)) { + SPDK_ERRLOG("No configuration file provided\n"); + return -1; + } + + config = spdk_conf_allocate(); + if (!config) { + SPDK_ERRLOG("Unable to allocate configuration file\n"); + return -1; + } + + rc = spdk_conf_read(config, eo->conf); + if (rc != 0) { + SPDK_ERRLOG("Invalid configuration file format\n"); + spdk_conf_free(config); + return -1; + } + if (spdk_conf_first_section(config) == NULL) { + SPDK_ERRLOG("Invalid configuration file format\n"); + spdk_conf_free(config); + return -1; + } + spdk_conf_set_as_default(config); + + /* Initialize the environment library */ + spdk_env_opts_init(&opts); + opts.name = "fio"; + + if (eo->mem_mb) { + opts.mem_size = eo->mem_mb; + } + opts.hugepage_single_segments = eo->mem_single_seg; + + if (spdk_env_init(&opts) < 0) { + SPDK_ERRLOG("Unable to initialize SPDK env\n"); + spdk_conf_free(config); + return -1; + } + spdk_unaffinitize_thread(); + + /* Create an SPDK thread temporarily */ + rc = spdk_fio_init_thread(td); + if (rc < 0) { + SPDK_ERRLOG("Failed to create initialization thread\n"); + return -1; + } + + g_init_thread = fio_thread = td->io_ops_data; + + /* Initialize the copy engine */ + spdk_copy_engine_initialize(); + + /* Initialize the bdev layer */ + spdk_bdev_initialize(spdk_fio_bdev_init_done, &done); + + /* First, poll until initialization is done. */ + do { + spdk_fio_poll_thread(fio_thread); + } while (!done); + + /* + * Continue polling until there are no more events. + * This handles any final events posted by pollers. + */ + do { + count = spdk_fio_poll_thread(fio_thread); + } while (count > 0); + + /* + * Spawn a thread to continue polling this thread + * occasionally. + */ + + rc = pthread_create(&g_init_thread_id, NULL, &spdk_init_thread_poll, fio_thread); + if (rc != 0) { + SPDK_ERRLOG("Unable to spawn thread to poll admin queue. It won't be polled.\n"); + } + + return 0; +} + +/* Called for each thread to fill in the 'real_file_size' member for + * each file associated with this thread. This is called prior to + * the init operation (spdk_fio_init()) below. This call will occur + * on the initial start up thread if 'create_serialize' is true, or + * on the thread actually associated with 'thread_data' if 'create_serialize' + * is false. + */ +static int +spdk_fio_setup(struct thread_data *td) +{ + unsigned int i; + struct fio_file *f; + + if (!td->o.use_thread) { + SPDK_ERRLOG("must set thread=1 when using spdk plugin\n"); + return -1; + } + + if (!g_spdk_env_initialized) { + if (spdk_fio_init_env(td)) { + SPDK_ERRLOG("failed to initialize\n"); + return -1; + } + + g_spdk_env_initialized = true; + } + + for_each_file(td, f, i) { + struct spdk_bdev *bdev; + + bdev = spdk_bdev_get_by_name(f->file_name); + if (!bdev) { + SPDK_ERRLOG("Unable to find bdev with name %s\n", f->file_name); + return -1; + } + + f->real_file_size = spdk_bdev_get_num_blocks(bdev) * + spdk_bdev_get_block_size(bdev); + + } + + return 0; +} + +/* Called for each thread, on that thread, shortly after the thread + * starts. + */ +static int +spdk_fio_init(struct thread_data *td) +{ + struct spdk_fio_thread *fio_thread; + unsigned int i; + struct fio_file *f; + int rc; + + spdk_fio_init_thread(td); + + fio_thread = td->io_ops_data; + + for_each_file(td, f, i) { + struct spdk_fio_target *target; + + target = calloc(1, sizeof(*target)); + if (!target) { + SPDK_ERRLOG("Unable to allocate memory for I/O target.\n"); + return -1; + } + + target->bdev = spdk_bdev_get_by_name(f->file_name); + if (!target->bdev) { + SPDK_ERRLOG("Unable to find bdev with name %s\n", f->file_name); + free(target); + return -1; + } + + rc = spdk_bdev_open(target->bdev, true, NULL, NULL, &target->desc); + if (rc) { + SPDK_ERRLOG("Unable to open bdev %s\n", f->file_name); + free(target); + return -1; + } + + target->ch = spdk_bdev_get_io_channel(target->desc); + if (!target->ch) { + SPDK_ERRLOG("Unable to get I/O channel for bdev.\n"); + spdk_bdev_close(target->desc); + free(target); + return -1; + } + + f->engine_data = target; + + TAILQ_INSERT_TAIL(&fio_thread->targets, target, link); + } + + return 0; +} + +static void +spdk_fio_cleanup_thread(struct spdk_fio_thread *fio_thread) +{ + struct spdk_fio_target *target, *tmp; + + TAILQ_FOREACH_SAFE(target, &fio_thread->targets, link, tmp) { + TAILQ_REMOVE(&fio_thread->targets, target, link); + spdk_put_io_channel(target->ch); + spdk_bdev_close(target->desc); + free(target); + } + + while (spdk_fio_poll_thread(fio_thread) > 0) {} + + spdk_free_thread(); + spdk_ring_free(fio_thread->ring); + free(fio_thread->iocq); + free(fio_thread); +} + +static void +spdk_fio_cleanup(struct thread_data *td) +{ + struct spdk_fio_thread *fio_thread = td->io_ops_data; + + spdk_fio_cleanup_thread(fio_thread); + td->io_ops_data = NULL; +} + +static int +spdk_fio_open(struct thread_data *td, struct fio_file *f) +{ + + return 0; +} + +static int +spdk_fio_close(struct thread_data *td, struct fio_file *f) +{ + return 0; +} + +static int +spdk_fio_iomem_alloc(struct thread_data *td, size_t total_mem) +{ + td->orig_buffer = spdk_dma_zmalloc(total_mem, 0x1000, NULL); + return td->orig_buffer == NULL; +} + +static void +spdk_fio_iomem_free(struct thread_data *td) +{ + spdk_dma_free(td->orig_buffer); +} + +static int +spdk_fio_io_u_init(struct thread_data *td, struct io_u *io_u) +{ + struct spdk_fio_request *fio_req; + + fio_req = calloc(1, sizeof(*fio_req)); + if (fio_req == NULL) { + return 1; + } + fio_req->io = io_u; + fio_req->td = td; + + io_u->engine_data = fio_req; + + return 0; +} + +static void +spdk_fio_io_u_free(struct thread_data *td, struct io_u *io_u) +{ + struct spdk_fio_request *fio_req = io_u->engine_data; + + if (fio_req) { + assert(fio_req->io == io_u); + free(fio_req); + io_u->engine_data = NULL; + } +} + +static void +spdk_fio_completion_cb(struct spdk_bdev_io *bdev_io, + bool success, + void *cb_arg) +{ + struct spdk_fio_request *fio_req = cb_arg; + struct thread_data *td = fio_req->td; + struct spdk_fio_thread *fio_thread = td->io_ops_data; + + assert(fio_thread->iocq_count < fio_thread->iocq_size); + fio_req->io->error = success ? 0 : EIO; + fio_thread->iocq[fio_thread->iocq_count++] = fio_req->io; + + spdk_bdev_free_io(bdev_io); +} + +#if FIO_IOOPS_VERSION >= 24 +typedef enum fio_q_status fio_q_status_t; +#else +typedef int fio_q_status_t; +#endif + +static fio_q_status_t +spdk_fio_queue(struct thread_data *td, struct io_u *io_u) +{ + int rc = 1; + struct spdk_fio_request *fio_req = io_u->engine_data; + struct spdk_fio_target *target = io_u->file->engine_data; + + assert(fio_req->td == td); + + if (!target) { + SPDK_ERRLOG("Unable to look up correct I/O target.\n"); + fio_req->io->error = ENODEV; + return FIO_Q_COMPLETED; + } + + switch (io_u->ddir) { + case DDIR_READ: + rc = spdk_bdev_read(target->desc, target->ch, + io_u->buf, io_u->offset, io_u->xfer_buflen, + spdk_fio_completion_cb, fio_req); + break; + case DDIR_WRITE: + rc = spdk_bdev_write(target->desc, target->ch, + io_u->buf, io_u->offset, io_u->xfer_buflen, + spdk_fio_completion_cb, fio_req); + break; + case DDIR_TRIM: + rc = spdk_bdev_unmap(target->desc, target->ch, + io_u->offset, io_u->xfer_buflen, + spdk_fio_completion_cb, fio_req); + break; + default: + assert(false); + break; + } + + if (rc == -ENOMEM) { + return FIO_Q_BUSY; + } + + if (rc != 0) { + fio_req->io->error = abs(rc); + return FIO_Q_COMPLETED; + } + + return FIO_Q_QUEUED; +} + +static struct io_u * +spdk_fio_event(struct thread_data *td, int event) +{ + struct spdk_fio_thread *fio_thread = td->io_ops_data; + + assert(event >= 0); + assert((unsigned)event < fio_thread->iocq_count); + return fio_thread->iocq[event]; +} + +static size_t +spdk_fio_poll_thread(struct spdk_fio_thread *fio_thread) +{ + struct spdk_fio_msg *msg; + struct spdk_fio_poller *p, *tmp; + size_t count; + + /* Process new events */ + count = spdk_ring_dequeue(fio_thread->ring, (void **)&msg, 1); + if (count > 0) { + msg->cb_fn(msg->cb_arg); + free(msg); + } + + /* Call all pollers */ + TAILQ_FOREACH_SAFE(p, &fio_thread->pollers, link, tmp) { + p->cb_fn(p->cb_arg); + } + + return count; +} + +static int +spdk_fio_getevents(struct thread_data *td, unsigned int min, + unsigned int max, const struct timespec *t) +{ + struct spdk_fio_thread *fio_thread = td->io_ops_data; + struct timespec t0, t1; + uint64_t timeout = 0; + + if (t) { + timeout = t->tv_sec * 1000000000L + t->tv_nsec; + clock_gettime(CLOCK_MONOTONIC_RAW, &t0); + } + + fio_thread->iocq_count = 0; + + for (;;) { + spdk_fio_poll_thread(fio_thread); + + if (fio_thread->iocq_count >= min) { + return fio_thread->iocq_count; + } + + if (t) { + clock_gettime(CLOCK_MONOTONIC_RAW, &t1); + uint64_t elapse = ((t1.tv_sec - t0.tv_sec) * 1000000000L) + + t1.tv_nsec - t0.tv_nsec; + if (elapse > timeout) { + break; + } + } + } + + return fio_thread->iocq_count; +} + +static int +spdk_fio_invalidate(struct thread_data *td, struct fio_file *f) +{ + /* TODO: This should probably send a flush to the device, but for now just return successful. */ + return 0; +} + +static struct fio_option options[] = { + { + .name = "spdk_conf", + .lname = "SPDK configuration file", + .type = FIO_OPT_STR_STORE, + .off1 = offsetof(struct spdk_fio_options, conf), + .help = "A SPDK configuration file", + .category = FIO_OPT_C_ENGINE, + .group = FIO_OPT_G_INVALID, + }, + { + .name = "spdk_mem", + .lname = "SPDK memory in MB", + .type = FIO_OPT_INT, + .off1 = offsetof(struct spdk_fio_options, mem_mb), + .help = "Amount of memory in MB to allocate for SPDK", + .category = FIO_OPT_C_ENGINE, + .group = FIO_OPT_G_INVALID, + }, + { + .name = "spdk_single_seg", + .lname = "SPDK switch to create just a single hugetlbfs file", + .type = FIO_OPT_BOOL, + .off1 = offsetof(struct spdk_fio_options, mem_single_seg), + .help = "If set to 1, SPDK will use just a single hugetlbfs file", + .category = FIO_OPT_C_ENGINE, + .group = FIO_OPT_G_INVALID, + }, + { + .name = NULL, + }, +}; + +/* FIO imports this structure using dlsym */ +struct ioengine_ops ioengine = { + .name = "spdk_bdev", + .version = FIO_IOOPS_VERSION, + .flags = FIO_RAWIO | FIO_NOEXTEND | FIO_NODISKUTIL | FIO_MEMALIGN, + .setup = spdk_fio_setup, + .init = spdk_fio_init, + //.prep = unused, + .queue = spdk_fio_queue, + //.commit = unused, + .getevents = spdk_fio_getevents, + .event = spdk_fio_event, + //.errdetails = unused, + //.cancel = unused, + .cleanup = spdk_fio_cleanup, + .open_file = spdk_fio_open, + .close_file = spdk_fio_close, + .invalidate = spdk_fio_invalidate, + //.unlink_file = unused, + //.get_file_size = unused, + //.terminate = unused, + .iomem_alloc = spdk_fio_iomem_alloc, + .iomem_free = spdk_fio_iomem_free, + .io_u_init = spdk_fio_io_u_init, + .io_u_free = spdk_fio_io_u_free, + .option_struct_size = sizeof(struct spdk_fio_options), + .options = options, +}; + +static void fio_init spdk_fio_register(void) +{ + register_ioengine(&ioengine); +} + +static void +spdk_fio_module_finish_done(void *cb_arg) +{ + *(bool *)cb_arg = true; +} + +static void +spdk_fio_finish_env(void) +{ + struct spdk_fio_thread *fio_thread; + bool done = false; + size_t count; + + /* the same thread that called spdk_fio_init_env */ + fio_thread = g_init_thread; + + if (pthread_cancel(g_init_thread_id) == 0) { + pthread_join(g_init_thread_id, NULL); + } + + spdk_bdev_finish(spdk_fio_module_finish_done, &done); + + do { + spdk_fio_poll_thread(fio_thread); + } while (!done); + + do { + count = spdk_fio_poll_thread(fio_thread); + } while (count > 0); + + done = false; + spdk_copy_engine_finish(spdk_fio_module_finish_done, &done); + + do { + spdk_fio_poll_thread(fio_thread); + } while (!done); + + do { + count = spdk_fio_poll_thread(fio_thread); + } while (count > 0); + + spdk_fio_cleanup_thread(fio_thread); +} + +static void fio_exit spdk_fio_unregister(void) +{ + if (g_spdk_env_initialized) { + spdk_fio_finish_env(); + g_spdk_env_initialized = false; + } + unregister_ioengine(&ioengine); +} diff --git a/src/spdk/examples/bdev/fio_plugin/full_bench.fio b/src/spdk/examples/bdev/fio_plugin/full_bench.fio new file mode 100644 index 00000000..f76da18d --- /dev/null +++ b/src/spdk/examples/bdev/fio_plugin/full_bench.fio @@ -0,0 +1,20 @@ +[global] +ioengine=spdk_bdev +spdk_conf=./examples/bdev/fio_plugin/bdev.conf.in +thread=1 +group_reporting=1 +direct=1 +verify=0 +norandommap=1 +cpumask=1 +percentile_list=50:99:99.9:99.99:99.999 + +[4k_randread_qd1] +filename=Malloc0 +description="4KiB Random Read QD=1" +bs=4k +rw=randread +iodepth=1 +time_based=1 +ramp_time=0 +runtime=10 diff --git a/src/spdk/examples/bdev/hello_world/.gitignore b/src/spdk/examples/bdev/hello_world/.gitignore new file mode 100644 index 00000000..7bdf9393 --- /dev/null +++ b/src/spdk/examples/bdev/hello_world/.gitignore @@ -0,0 +1 @@ +hello_bdev diff --git a/src/spdk/examples/bdev/hello_world/Makefile b/src/spdk/examples/bdev/hello_world/Makefile new file mode 100644 index 00000000..dacc1b81 --- /dev/null +++ b/src/spdk/examples/bdev/hello_world/Makefile @@ -0,0 +1,56 @@ +# +# Copyright (c) Intel Corporation. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..) +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk +include $(SPDK_ROOT_DIR)/mk/spdk.app.mk +include $(SPDK_ROOT_DIR)/mk/spdk.modules.mk + +APP = hello_bdev + +C_SRCS := hello_bdev.c + +SPDK_LIB_LIST = event_bdev event_copy +SPDK_LIB_LIST += bdev copy event thread util conf trace log jsonrpc json rpc + +LIBS += $(COPY_MODULES_LINKER_ARGS) $(BLOCKDEV_MODULES_LINKER_ARGS) $(SOCK_MODULES_LINKER_ARGS) +LIBS += $(SPDK_LIB_LINKER_ARGS) $(ENV_LINKER_ARGS) + +all : $(APP) + @: + +$(APP) : $(OBJS) $(SPDK_LIB_FILES) $(BLOCKDEV_MODULES_FILES) $(COPY_MODULES_FILES) $(SOCK_MODULES_FILES) $(LINKER_MODULES) $(ENV_LIBS) + $(LINK_C) + +clean : + $(CLEAN_C) $(APP) + +include $(SPDK_ROOT_DIR)/mk/spdk.deps.mk diff --git a/src/spdk/examples/bdev/hello_world/bdev.conf b/src/spdk/examples/bdev/hello_world/bdev.conf new file mode 100644 index 00000000..c8504b01 --- /dev/null +++ b/src/spdk/examples/bdev/hello_world/bdev.conf @@ -0,0 +1,5 @@ +[Passthru] + PT Malloc1 PT0 +[Malloc] + NumberOfLuns 2 + LunSizeInMB 16 diff --git a/src/spdk/examples/bdev/hello_world/hello_bdev.c b/src/spdk/examples/bdev/hello_world/hello_bdev.c new file mode 100644 index 00000000..cc65b051 --- /dev/null +++ b/src/spdk/examples/bdev/hello_world/hello_bdev.c @@ -0,0 +1,294 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "spdk/stdinc.h" +#include "spdk/thread.h" +#include "spdk/bdev.h" +#include "spdk/env.h" +#include "spdk/event.h" +#include "spdk/log.h" +#include "spdk/string.h" +#include "spdk/bdev_module.h" + +static char *g_bdev_name = "Malloc0"; + +/* + * We'll use this struct to gather housekeeping hello_context to pass between + * our events and callbacks. + */ +struct hello_context_t { + struct spdk_bdev *bdev; + struct spdk_bdev_desc *bdev_desc; + struct spdk_io_channel *bdev_io_channel; + char *buff; + char *bdev_name; + struct spdk_bdev_io_wait_entry bdev_io_wait; +}; + +/* + * Usage function for printing parameters that are specific to this application + */ +static void +hello_bdev_usage(void) +{ + printf(" -b <bdev> name of the bdev to use\n"); +} + +/* + * This function is called to parse the parameters that are specific to this application + */ +static void hello_bdev_parse_arg(int ch, char *arg) +{ + switch (ch) { + case 'b': + g_bdev_name = arg; + break; + } +} + +/* + * Callback function for read io completion. + */ +static void +read_complete(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) +{ + struct hello_context_t *hello_context = cb_arg; + + if (success) { + SPDK_NOTICELOG("Read string from bdev : %s\n", hello_context->buff); + } else { + SPDK_ERRLOG("bdev io read error\n"); + } + + /* Complete the bdev io and close the channel */ + spdk_bdev_free_io(bdev_io); + spdk_put_io_channel(hello_context->bdev_io_channel); + spdk_bdev_close(hello_context->bdev_desc); + SPDK_NOTICELOG("Stopping app\n"); + spdk_app_stop(success ? 0 : -1); +} + +static void +hello_read(void *arg) +{ + struct hello_context_t *hello_context = arg; + int rc = 0; + uint32_t length = spdk_bdev_get_block_size(hello_context->bdev); + + SPDK_NOTICELOG("Reading io\n"); + rc = spdk_bdev_read(hello_context->bdev_desc, hello_context->bdev_io_channel, + hello_context->buff, 0, length, read_complete, hello_context); + + if (rc == -ENOMEM) { + SPDK_NOTICELOG("Queueing io\n"); + /* In case we cannot perform I/O now, queue I/O */ + hello_context->bdev_io_wait.bdev = hello_context->bdev; + hello_context->bdev_io_wait.cb_fn = hello_read; + hello_context->bdev_io_wait.cb_arg = hello_context; + spdk_bdev_queue_io_wait(hello_context->bdev, hello_context->bdev_io_channel, + &hello_context->bdev_io_wait); + } else if (rc) { + SPDK_ERRLOG("%s error while reading from bdev: %d\n", spdk_strerror(-rc), rc); + spdk_put_io_channel(hello_context->bdev_io_channel); + spdk_bdev_close(hello_context->bdev_desc); + spdk_app_stop(-1); + } +} + +/* + * Callback function for write io completion. + */ +static void +write_complete(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) +{ + struct hello_context_t *hello_context = cb_arg; + uint32_t length; + + /* Complete the I/O */ + spdk_bdev_free_io(bdev_io); + + if (success) { + SPDK_NOTICELOG("bdev io write completed successfully\n"); + } else { + SPDK_ERRLOG("bdev io write error: %d\n", EIO); + spdk_put_io_channel(hello_context->bdev_io_channel); + spdk_bdev_close(hello_context->bdev_desc); + spdk_app_stop(-1); + return; + } + + /* Zero the buffer so that we can use it for reading */ + length = spdk_bdev_get_block_size(hello_context->bdev); + memset(hello_context->buff, 0, length); + + hello_read(hello_context); +} + +static void +hello_write(void *arg) +{ + struct hello_context_t *hello_context = arg; + int rc = 0; + uint32_t length = spdk_bdev_get_block_size(hello_context->bdev); + + SPDK_NOTICELOG("Writing to the bdev\n"); + rc = spdk_bdev_write(hello_context->bdev_desc, hello_context->bdev_io_channel, + hello_context->buff, 0, length, write_complete, hello_context); + + if (rc == -ENOMEM) { + SPDK_NOTICELOG("Queueing io\n"); + /* In case we cannot perform I/O now, queue I/O */ + hello_context->bdev_io_wait.bdev = hello_context->bdev; + hello_context->bdev_io_wait.cb_fn = hello_write; + hello_context->bdev_io_wait.cb_arg = hello_context; + spdk_bdev_queue_io_wait(hello_context->bdev, hello_context->bdev_io_channel, + &hello_context->bdev_io_wait); + } else if (rc) { + SPDK_ERRLOG("%s error while writing to bdev: %d\n", spdk_strerror(-rc), rc); + spdk_put_io_channel(hello_context->bdev_io_channel); + spdk_bdev_close(hello_context->bdev_desc); + spdk_app_stop(-1); + } +} + +/* + * Our initial event that kicks off everything from main(). + */ +static void +hello_start(void *arg1, void *arg2) +{ + struct hello_context_t *hello_context = arg1; + uint32_t blk_size, buf_align; + int rc = 0; + hello_context->bdev = NULL; + hello_context->bdev_desc = NULL; + + SPDK_NOTICELOG("Successfully started the application\n"); + + /* + * Get the bdev. There can be many bdevs configured in + * in the configuration file but this application will only + * use the one input by the user at runtime so we get it via its name. + */ + hello_context->bdev = spdk_bdev_get_by_name(hello_context->bdev_name); + if (hello_context->bdev == NULL) { + SPDK_ERRLOG("Could not find the bdev: %s\n", hello_context->bdev_name); + spdk_app_stop(-1); + return; + } + + /* + * Open the bdev by calling spdk_bdev_open() + * The function will return a descriptor + */ + SPDK_NOTICELOG("Opening the bdev %s\n", hello_context->bdev_name); + rc = spdk_bdev_open(hello_context->bdev, true, NULL, NULL, &hello_context->bdev_desc); + if (rc) { + SPDK_ERRLOG("Could not open bdev: %s\n", hello_context->bdev_name); + spdk_app_stop(-1); + return; + } + + SPDK_NOTICELOG("Opening io channel\n"); + /* Open I/O channel */ + hello_context->bdev_io_channel = spdk_bdev_get_io_channel(hello_context->bdev_desc); + if (hello_context->bdev_io_channel == NULL) { + SPDK_ERRLOG("Could not create bdev I/O channel!!\n"); + spdk_bdev_close(hello_context->bdev_desc); + spdk_app_stop(-1); + return; + } + + /* Allocate memory for the write buffer. + * Initialize the write buffer with the string "Hello World!" + */ + blk_size = spdk_bdev_get_block_size(hello_context->bdev); + buf_align = spdk_bdev_get_buf_align(hello_context->bdev); + hello_context->buff = spdk_dma_zmalloc(blk_size, buf_align, NULL); + if (!hello_context->buff) { + SPDK_ERRLOG("Failed to allocate buffer\n"); + spdk_put_io_channel(hello_context->bdev_io_channel); + spdk_bdev_close(hello_context->bdev_desc); + spdk_app_stop(-1); + return; + } + snprintf(hello_context->buff, blk_size, "%s", "Hello World!\n"); + + hello_write(hello_context); +} + +int +main(int argc, char **argv) +{ + struct spdk_app_opts opts = {}; + int rc = 0; + struct hello_context_t hello_context = {}; + + /* Set default values in opts structure. */ + spdk_app_opts_init(&opts); + opts.name = "hello_bdev"; + opts.config_file = "bdev.conf"; + + /* + * The user can provide the config file and bdev name at run time. + * For example, to use Malloc0 in file bdev.conf run with params + * ./hello_bdev -c bdev.conf -b Malloc0 + * To use passthru bdev PT0 run with params + * ./hello_bdev -c bdev.conf -b PT0 + * If none of the parameters are provide the application will use the + * default parameters(-c bdev.conf -b Malloc0). + */ + if ((rc = spdk_app_parse_args(argc, argv, &opts, "b:", NULL, hello_bdev_parse_arg, + hello_bdev_usage)) != SPDK_APP_PARSE_ARGS_SUCCESS) { + exit(rc); + } + hello_context.bdev_name = g_bdev_name; + + /* + * spdk_app_start() will block running hello_start() until + * spdk_app_stop() is called by someone (not simply when + * hello_start() returns), or if an error occurs during + * spdk_app_start() before hello_start() runs. + */ + rc = spdk_app_start(&opts, hello_start, &hello_context, NULL); + if (rc) { + SPDK_ERRLOG("ERROR starting application\n"); + } + + /* When the app stops, free up memory that we allocated. */ + spdk_dma_free(hello_context.buff); + + /* Gracefully close out all of the SPDK subsystems. */ + spdk_app_fini(); + return rc; +} |