Adding upstream version 18.2.2.upstream/18.2.2

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

26 files changed, 23768 insertions, 0 deletions

diff --git a/src/spdk/lib/nvme/Makefile b/src/spdk/lib/nvme/Makefile
new file mode 100644
index 000000000..1c02965f5
--- /dev/null
+++ b/src/spdk/lib/nvme/Makefile
@@ -0,0 +1,73 @@
+#
+#  BSD LICENSE
+#
+#  Copyright (c) Intel Corporation.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+SO_VER := 4
+SO_MINOR := 0
+
+C_SRCS = nvme_ctrlr_cmd.c nvme_ctrlr.c nvme_fabric.c nvme_ns_cmd.c nvme_ns.c nvme_pcie.c nvme_qpair.c nvme.c nvme_quirks.c nvme_transport.c nvme_uevent.c nvme_ctrlr_ocssd_cmd.c \
+	nvme_ns_ocssd_cmd.c nvme_tcp.c nvme_opal.c nvme_io_msg.c nvme_poll_group.c
+C_SRCS-$(CONFIG_RDMA) += nvme_rdma.c
+C_SRCS-$(CONFIG_NVME_CUSE) += nvme_cuse.c
+
+LIBNAME = nvme
+LOCAL_SYS_LIBS = -luuid
+ifeq ($(CONFIG_RDMA),y)
+LOCAL_SYS_LIBS += -libverbs -lrdmacm
+#Attach only if FreeBSD and RDMA is specified with configure
+ifeq ($(OS),FreeBSD)
+# Mellanox - MLX4 HBA Userspace Library
+ifneq ("$(wildcard /usr/lib/libmlx4.*)","")
+LOCAL_SYS_LIBS += -lmlx4
+endif
+# Mellanox - MLX5 HBA Userspace Library
+ifneq ("$(wildcard /usr/lib/libmlx5.*)","")
+LOCAL_SYS_LIBS += -lmlx5
+endif
+# Chelsio HBA Userspace Library
+ifneq ("$(wildcard /usr/lib/libcxgb4.*)","")
+LOCAL_SYS_LIBS += -lcxgb4
+endif
+endif
+endif
+
+ifeq ($(CONFIG_NVME_CUSE),y)
+# fuse requires to set _FILE_OFFSET_BITS to 64 bits even for 64 bit machines
+CFLAGS += -D_FILE_OFFSET_BITS=64
+endif
+
+SPDK_MAP_FILE = $(abspath $(CURDIR)/spdk_nvme.map)
+
+include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk
diff --git a/src/spdk/lib/nvme/nvme.c b/src/spdk/lib/nvme/nvme.c
new file mode 100644
index 000000000..9393810a6
--- /dev/null
+++ b/src/spdk/lib/nvme/nvme.c
@@ -0,0 +1,1423 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation. All rights reserved.
+ *   Copyright (c) 2020 Mellanox Technologies LTD. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/nvmf_spec.h"
+#include "spdk/string.h"
+#include "nvme_internal.h"
+#include "nvme_io_msg.h"
+#include "nvme_uevent.h"
+
+#define SPDK_NVME_DRIVER_NAME "spdk_nvme_driver"
+
+struct nvme_driver	*g_spdk_nvme_driver;
+pid_t			g_spdk_nvme_pid;
+
+/* gross timeout of 180 seconds in milliseconds */
+static int g_nvme_driver_timeout_ms = 3 * 60 * 1000;
+
+/* Per-process attached controller list */
+static TAILQ_HEAD(, spdk_nvme_ctrlr) g_nvme_attached_ctrlrs =
+	TAILQ_HEAD_INITIALIZER(g_nvme_attached_ctrlrs);
+
+/* Returns true if ctrlr should be stored on the multi-process shared_attached_ctrlrs list */
+static bool
+nvme_ctrlr_shared(const struct spdk_nvme_ctrlr *ctrlr)
+{
+	return ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_PCIE;
+}
+
+void
+nvme_ctrlr_connected(struct spdk_nvme_probe_ctx *probe_ctx,
+		     struct spdk_nvme_ctrlr *ctrlr)
+{
+	TAILQ_INSERT_TAIL(&probe_ctx->init_ctrlrs, ctrlr, tailq);
+}
+
+int
+spdk_nvme_detach(struct spdk_nvme_ctrlr *ctrlr)
+{
+	nvme_robust_mutex_lock(&g_spdk_nvme_driver->lock);
+
+	nvme_ctrlr_proc_put_ref(ctrlr);
+
+	if (nvme_ctrlr_get_ref_count(ctrlr) == 0) {
+		nvme_io_msg_ctrlr_detach(ctrlr);
+		if (nvme_ctrlr_shared(ctrlr)) {
+			TAILQ_REMOVE(&g_spdk_nvme_driver->shared_attached_ctrlrs, ctrlr, tailq);
+		} else {
+			TAILQ_REMOVE(&g_nvme_attached_ctrlrs, ctrlr, tailq);
+		}
+		nvme_ctrlr_destruct(ctrlr);
+	}
+
+	nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock);
+	return 0;
+}
+
+void
+nvme_completion_poll_cb(void *arg, const struct spdk_nvme_cpl *cpl)
+{
+	struct nvme_completion_poll_status	*status = arg;
+
+	if (status->timed_out) {
+		/* There is no routine waiting for the completion of this request, free allocated memory */
+		free(status);
+		return;
+	}
+
+	/*
+	 * Copy status into the argument passed by the caller, so that
+	 *  the caller can check the status to determine if the
+	 *  the request passed or failed.
+	 */
+	memcpy(&status->cpl, cpl, sizeof(*cpl));
+	status->done = true;
+}
+
+/**
+ * Poll qpair for completions until a command completes.
+ *
+ * \param qpair queue to poll
+ * \param status completion status. The user must fill this structure with zeroes before calling
+ * this function
+ * \param robust_mutex optional robust mutex to lock while polling qpair
+ *
+ * \return 0 if command completed without error,
+ * -EIO if command completed with error,
+ * -ECANCELED if command is not completed due to transport/device error
+ *
+ * The command to wait upon must be submitted with nvme_completion_poll_cb as the callback
+ * and status as the callback argument.
+ */
+int
+nvme_wait_for_completion_robust_lock(
+	struct spdk_nvme_qpair *qpair,
+	struct nvme_completion_poll_status *status,
+	pthread_mutex_t *robust_mutex)
+{
+	int rc;
+
+	while (status->done == false) {
+		if (robust_mutex) {
+			nvme_robust_mutex_lock(robust_mutex);
+		}
+
+		rc = spdk_nvme_qpair_process_completions(qpair, 0);
+
+		if (robust_mutex) {
+			nvme_robust_mutex_unlock(robust_mutex);
+		}
+
+		if (rc < 0) {
+			status->cpl.status.sct = SPDK_NVME_SCT_GENERIC;
+			status->cpl.status.sc = SPDK_NVME_SC_ABORTED_SQ_DELETION;
+			if (status->done == false) {
+				status->timed_out = true;
+			}
+			return -ECANCELED;
+		}
+	}
+
+	return spdk_nvme_cpl_is_error(&status->cpl) ? -EIO : 0;
+}
+
+int
+nvme_wait_for_completion(struct spdk_nvme_qpair *qpair,
+			 struct nvme_completion_poll_status *status)
+{
+	return nvme_wait_for_completion_robust_lock(qpair, status, NULL);
+}
+
+/**
+ * Poll qpair for completions until a command completes.
+ *
+ * \param qpair queue to poll
+ * \param status completion status. The user must fill this structure with zeroes before calling
+ * this function
+ * \param timeout_in_secs optional timeout
+ *
+ * \return 0 if command completed without error,
+ * -EIO if command completed with error,
+ * -ECANCELED if command is not completed due to transport/device error or time expired
+ *
+ * The command to wait upon must be submitted with nvme_completion_poll_cb as the callback
+ * and status as the callback argument.
+ */
+int
+nvme_wait_for_completion_timeout(struct spdk_nvme_qpair *qpair,
+				 struct nvme_completion_poll_status *status,
+				 uint64_t timeout_in_secs)
+{
+	uint64_t timeout_tsc = 0;
+	int rc = 0;
+
+	if (timeout_in_secs) {
+		timeout_tsc = spdk_get_ticks() + timeout_in_secs * spdk_get_ticks_hz();
+	}
+
+	while (status->done == false) {
+		rc = spdk_nvme_qpair_process_completions(qpair, 0);
+
+		if (rc < 0) {
+			status->cpl.status.sct = SPDK_NVME_SCT_GENERIC;
+			status->cpl.status.sc = SPDK_NVME_SC_ABORTED_SQ_DELETION;
+			break;
+		}
+		if (timeout_tsc && spdk_get_ticks() > timeout_tsc) {
+			break;
+		}
+	}
+
+	if (status->done == false || rc < 0) {
+		if (status->done == false) {
+			status->timed_out = true;
+		}
+		return -ECANCELED;
+	}
+
+	return spdk_nvme_cpl_is_error(&status->cpl) ? -EIO : 0;
+}
+
+static void
+nvme_user_copy_cmd_complete(void *arg, const struct spdk_nvme_cpl *cpl)
+{
+	struct nvme_request *req = arg;
+	enum spdk_nvme_data_transfer xfer;
+
+	if (req->user_buffer && req->payload_size) {
+		/* Copy back to the user buffer and free the contig buffer */
+		assert(nvme_payload_type(&req->payload) == NVME_PAYLOAD_TYPE_CONTIG);
+		xfer = spdk_nvme_opc_get_data_transfer(req->cmd.opc);
+		if (xfer == SPDK_NVME_DATA_CONTROLLER_TO_HOST ||
+		    xfer == SPDK_NVME_DATA_BIDIRECTIONAL) {
+			assert(req->pid == getpid());
+			memcpy(req->user_buffer, req->payload.contig_or_cb_arg, req->payload_size);
+		}
+
+		spdk_free(req->payload.contig_or_cb_arg);
+	}
+
+	/* Call the user's original callback now that the buffer has been copied */
+	req->user_cb_fn(req->user_cb_arg, cpl);
+}
+
+/**
+ * Allocate a request as well as a DMA-capable buffer to copy to/from the user's buffer.
+ *
+ * This is intended for use in non-fast-path functions (admin commands, reservations, etc.)
+ * where the overhead of a copy is not a problem.
+ */
+struct nvme_request *
+nvme_allocate_request_user_copy(struct spdk_nvme_qpair *qpair,
+				void *buffer, uint32_t payload_size, spdk_nvme_cmd_cb cb_fn,
+				void *cb_arg, bool host_to_controller)
+{
+	struct nvme_request *req;
+	void *dma_buffer = NULL;
+
+	if (buffer && payload_size) {
+		dma_buffer = spdk_zmalloc(payload_size, 4096, NULL,
+					  SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA);
+		if (!dma_buffer) {
+			return NULL;
+		}
+
+		if (host_to_controller) {
+			memcpy(dma_buffer, buffer, payload_size);
+		}
+	}
+
+	req = nvme_allocate_request_contig(qpair, dma_buffer, payload_size, nvme_user_copy_cmd_complete,
+					   NULL);
+	if (!req) {
+		spdk_free(dma_buffer);
+		return NULL;
+	}
+
+	req->user_cb_fn = cb_fn;
+	req->user_cb_arg = cb_arg;
+	req->user_buffer = buffer;
+	req->cb_arg = req;
+
+	return req;
+}
+
+/**
+ * Check if a request has exceeded the controller timeout.
+ *
+ * \param req request to check for timeout.
+ * \param cid command ID for command submitted by req (will be passed to timeout_cb_fn)
+ * \param active_proc per-process data for the controller associated with req
+ * \param now_tick current time from spdk_get_ticks()
+ * \return 0 if requests submitted more recently than req should still be checked for timeouts, or
+ * 1 if requests newer than req need not be checked.
+ *
+ * The request's timeout callback will be called if needed; the caller is only responsible for
+ * calling this function on each outstanding request.
+ */
+int
+nvme_request_check_timeout(struct nvme_request *req, uint16_t cid,
+			   struct spdk_nvme_ctrlr_process *active_proc,
+			   uint64_t now_tick)
+{
+	struct spdk_nvme_qpair *qpair = req->qpair;
+	struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
+
+	assert(active_proc->timeout_cb_fn != NULL);
+
+	if (req->timed_out || req->submit_tick == 0) {
+		return 0;
+	}
+
+	if (req->pid != g_spdk_nvme_pid) {
+		return 0;
+	}
+
+	if (nvme_qpair_is_admin_queue(qpair) &&
+	    req->cmd.opc == SPDK_NVME_OPC_ASYNC_EVENT_REQUEST) {
+		return 0;
+	}
+
+	if (req->submit_tick + active_proc->timeout_ticks > now_tick) {
+		return 1;
+	}
+
+	req->timed_out = true;
+
+	/*
+	 * We don't want to expose the admin queue to the user,
+	 * so when we're timing out admin commands set the
+	 * qpair to NULL.
+	 */
+	active_proc->timeout_cb_fn(active_proc->timeout_cb_arg, ctrlr,
+				   nvme_qpair_is_admin_queue(qpair) ? NULL : qpair,
+				   cid);
+	return 0;
+}
+
+int
+nvme_robust_mutex_init_shared(pthread_mutex_t *mtx)
+{
+	int rc = 0;
+
+#ifdef __FreeBSD__
+	pthread_mutex_init(mtx, NULL);
+#else
+	pthread_mutexattr_t attr;
+
+	if (pthread_mutexattr_init(&attr)) {
+		return -1;
+	}
+	if (pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED) ||
+	    pthread_mutexattr_setrobust(&attr, PTHREAD_MUTEX_ROBUST) ||
+	    pthread_mutex_init(mtx, &attr)) {
+		rc = -1;
+	}
+	pthread_mutexattr_destroy(&attr);
+#endif
+
+	return rc;
+}
+
+int
+nvme_driver_init(void)
+{
+	static pthread_mutex_t g_init_mutex = PTHREAD_MUTEX_INITIALIZER;
+	int ret = 0;
+	/* Any socket ID */
+	int socket_id = -1;
+
+	/* Use a special process-private mutex to ensure the global
+	 * nvme driver object (g_spdk_nvme_driver) gets initialized by
+	 * only one thread.  Once that object is established and its
+	 * mutex is initialized, we can unlock this mutex and use that
+	 * one instead.
+	 */
+	pthread_mutex_lock(&g_init_mutex);
+
+	/* Each process needs its own pid. */
+	g_spdk_nvme_pid = getpid();
+
+	/*
+	 * Only one thread from one process will do this driver init work.
+	 * The primary process will reserve the shared memory and do the
+	 *  initialization.
+	 * The secondary process will lookup the existing reserved memory.
+	 */
+	if (spdk_process_is_primary()) {
+		/* The unique named memzone already reserved. */
+		if (g_spdk_nvme_driver != NULL) {
+			pthread_mutex_unlock(&g_init_mutex);
+			return 0;
+		} else {
+			g_spdk_nvme_driver = spdk_memzone_reserve(SPDK_NVME_DRIVER_NAME,
+					     sizeof(struct nvme_driver), socket_id,
+					     SPDK_MEMZONE_NO_IOVA_CONTIG);
+		}
+
+		if (g_spdk_nvme_driver == NULL) {
+			SPDK_ERRLOG("primary process failed to reserve memory\n");
+			pthread_mutex_unlock(&g_init_mutex);
+			return -1;
+		}
+	} else {
+		g_spdk_nvme_driver = spdk_memzone_lookup(SPDK_NVME_DRIVER_NAME);
+
+		/* The unique named memzone already reserved by the primary process. */
+		if (g_spdk_nvme_driver != NULL) {
+			int ms_waited = 0;
+
+			/* Wait the nvme driver to get initialized. */
+			while ((g_spdk_nvme_driver->initialized == false) &&
+			       (ms_waited < g_nvme_driver_timeout_ms)) {
+				ms_waited++;
+				nvme_delay(1000); /* delay 1ms */
+			}
+			if (g_spdk_nvme_driver->initialized == false) {
+				SPDK_ERRLOG("timeout waiting for primary process to init\n");
+				pthread_mutex_unlock(&g_init_mutex);
+				return -1;
+			}
+		} else {
+			SPDK_ERRLOG("primary process is not started yet\n");
+			pthread_mutex_unlock(&g_init_mutex);
+			return -1;
+		}
+
+		pthread_mutex_unlock(&g_init_mutex);
+		return 0;
+	}
+
+	/*
+	 * At this moment, only one thread from the primary process will do
+	 * the g_spdk_nvme_driver initialization
+	 */
+	assert(spdk_process_is_primary());
+
+	ret = nvme_robust_mutex_init_shared(&g_spdk_nvme_driver->lock);
+	if (ret != 0) {
+		SPDK_ERRLOG("failed to initialize mutex\n");
+		spdk_memzone_free(SPDK_NVME_DRIVER_NAME);
+		pthread_mutex_unlock(&g_init_mutex);
+		return ret;
+	}
+
+	/* The lock in the shared g_spdk_nvme_driver object is now ready to
+	 * be used - so we can unlock the g_init_mutex here.
+	 */
+	pthread_mutex_unlock(&g_init_mutex);
+	nvme_robust_mutex_lock(&g_spdk_nvme_driver->lock);
+
+	g_spdk_nvme_driver->initialized = false;
+	g_spdk_nvme_driver->hotplug_fd = nvme_uevent_connect();
+	if (g_spdk_nvme_driver->hotplug_fd < 0) {
+		SPDK_DEBUGLOG(SPDK_LOG_NVME, "Failed to open uevent netlink socket\n");
+	}
+
+	TAILQ_INIT(&g_spdk_nvme_driver->shared_attached_ctrlrs);
+
+	spdk_uuid_generate(&g_spdk_nvme_driver->default_extended_host_id);
+
+	nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock);
+
+	return ret;
+}
+
+/* This function must only be called while holding g_spdk_nvme_driver->lock */
+int
+nvme_ctrlr_probe(const struct spdk_nvme_transport_id *trid,
+		 struct spdk_nvme_probe_ctx *probe_ctx, void *devhandle)
+{
+	struct spdk_nvme_ctrlr *ctrlr;
+	struct spdk_nvme_ctrlr_opts opts;
+
+	assert(trid != NULL);
+
+	spdk_nvme_ctrlr_get_default_ctrlr_opts(&opts, sizeof(opts));
+
+	if (!probe_ctx->probe_cb || probe_ctx->probe_cb(probe_ctx->cb_ctx, trid, &opts)) {
+		ctrlr = nvme_get_ctrlr_by_trid_unsafe(trid);
+		if (ctrlr) {
+			/* This ctrlr already exists.
+			* Increase the ref count before calling attach_cb() as the user may
+			* call nvme_detach() immediately. */
+			nvme_ctrlr_proc_get_ref(ctrlr);
+
+			if (probe_ctx->attach_cb) {
+				nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock);
+				probe_ctx->attach_cb(probe_ctx->cb_ctx, &ctrlr->trid, ctrlr, &ctrlr->opts);
+				nvme_robust_mutex_lock(&g_spdk_nvme_driver->lock);
+			}
+			return 0;
+		}
+
+		ctrlr = nvme_transport_ctrlr_construct(trid, &opts, devhandle);
+		if (ctrlr == NULL) {
+			SPDK_ERRLOG("Failed to construct NVMe controller for SSD: %s\n", trid->traddr);
+			return -1;
+		}
+		ctrlr->remove_cb = probe_ctx->remove_cb;
+		ctrlr->cb_ctx = probe_ctx->cb_ctx;
+
+		if (ctrlr->quirks & NVME_QUIRK_MINIMUM_IO_QUEUE_SIZE &&
+		    ctrlr->opts.io_queue_size == DEFAULT_IO_QUEUE_SIZE) {
+			/* If the user specifically set an IO queue size different than the
+			 * default, use that value.  Otherwise overwrite with the quirked value.
+			 * This allows this quirk to be overridden when necessary.
+			 * However, cap.mqes still needs to be respected.
+			 */
+			ctrlr->opts.io_queue_size = spdk_min(DEFAULT_IO_QUEUE_SIZE_FOR_QUIRK, ctrlr->cap.bits.mqes + 1u);
+		}
+
+		nvme_qpair_set_state(ctrlr->adminq, NVME_QPAIR_ENABLED);
+		TAILQ_INSERT_TAIL(&probe_ctx->init_ctrlrs, ctrlr, tailq);
+		return 0;
+	}
+
+	return 1;
+}
+
+static int
+nvme_ctrlr_poll_internal(struct spdk_nvme_ctrlr *ctrlr,
+			 struct spdk_nvme_probe_ctx *probe_ctx)
+{
+	int	rc = 0;
+
+	rc = nvme_ctrlr_process_init(ctrlr);
+
+	if (rc) {
+		/* Controller failed to initialize. */
+		TAILQ_REMOVE(&probe_ctx->init_ctrlrs, ctrlr, tailq);
+		SPDK_ERRLOG("Failed to initialize SSD: %s\n", ctrlr->trid.traddr);
+		nvme_ctrlr_fail(ctrlr, false);
+		nvme_ctrlr_destruct(ctrlr);
+		return rc;
+	}
+
+	if (ctrlr->state != NVME_CTRLR_STATE_READY) {
+		return 0;
+	}
+
+	STAILQ_INIT(&ctrlr->io_producers);
+
+	/*
+	 * Controller has been initialized.
+	 *  Move it to the attached_ctrlrs list.
+	 */
+	TAILQ_REMOVE(&probe_ctx->init_ctrlrs, ctrlr, tailq);
+
+	nvme_robust_mutex_lock(&g_spdk_nvme_driver->lock);
+	if (nvme_ctrlr_shared(ctrlr)) {
+		TAILQ_INSERT_TAIL(&g_spdk_nvme_driver->shared_attached_ctrlrs, ctrlr, tailq);
+	} else {
+		TAILQ_INSERT_TAIL(&g_nvme_attached_ctrlrs, ctrlr, tailq);
+	}
+
+	/*
+	 * Increase the ref count before calling attach_cb() as the user may
+	 * call nvme_detach() immediately.
+	 */
+	nvme_ctrlr_proc_get_ref(ctrlr);
+	nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock);
+
+	if (probe_ctx->attach_cb) {
+		probe_ctx->attach_cb(probe_ctx->cb_ctx, &ctrlr->trid, ctrlr, &ctrlr->opts);
+		return 0;
+	}
+
+	return 0;
+}
+
+static int
+nvme_init_controllers(struct spdk_nvme_probe_ctx *probe_ctx)
+{
+	int rc = 0;
+
+	while (true) {
+		rc = spdk_nvme_probe_poll_async(probe_ctx);
+		if (rc != -EAGAIN) {
+			return rc;
+		}
+	}
+
+	return rc;
+}
+
+/* This function must not be called while holding g_spdk_nvme_driver->lock */
+static struct spdk_nvme_ctrlr *
+nvme_get_ctrlr_by_trid(const struct spdk_nvme_transport_id *trid)
+{
+	struct spdk_nvme_ctrlr *ctrlr;
+
+	nvme_robust_mutex_lock(&g_spdk_nvme_driver->lock);
+	ctrlr = nvme_get_ctrlr_by_trid_unsafe(trid);
+	nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock);
+
+	return ctrlr;
+}
+
+/* This function must be called while holding g_spdk_nvme_driver->lock */
+struct spdk_nvme_ctrlr *
+nvme_get_ctrlr_by_trid_unsafe(const struct spdk_nvme_transport_id *trid)
+{
+	struct spdk_nvme_ctrlr *ctrlr;
+
+	/* Search per-process list */
+	TAILQ_FOREACH(ctrlr, &g_nvme_attached_ctrlrs, tailq) {
+		if (spdk_nvme_transport_id_compare(&ctrlr->trid, trid) == 0) {
+			return ctrlr;
+		}
+	}
+
+	/* Search multi-process shared list */
+	TAILQ_FOREACH(ctrlr, &g_spdk_nvme_driver->shared_attached_ctrlrs, tailq) {
+		if (spdk_nvme_transport_id_compare(&ctrlr->trid, trid) == 0) {
+			return ctrlr;
+		}
+	}
+
+	return NULL;
+}
+
+/* This function must only be called while holding g_spdk_nvme_driver->lock */
+static int
+nvme_probe_internal(struct spdk_nvme_probe_ctx *probe_ctx,
+		    bool direct_connect)
+{
+	int rc;
+	struct spdk_nvme_ctrlr *ctrlr, *ctrlr_tmp;
+
+	spdk_nvme_trid_populate_transport(&probe_ctx->trid, probe_ctx->trid.trtype);
+	if (!spdk_nvme_transport_available_by_name(probe_ctx->trid.trstring)) {
+		SPDK_ERRLOG("NVMe trtype %u not available\n", probe_ctx->trid.trtype);
+		return -1;
+	}
+
+	nvme_robust_mutex_lock(&g_spdk_nvme_driver->lock);
+
+	rc = nvme_transport_ctrlr_scan(probe_ctx, direct_connect);
+	if (rc != 0) {
+		SPDK_ERRLOG("NVMe ctrlr scan failed\n");
+		TAILQ_FOREACH_SAFE(ctrlr, &probe_ctx->init_ctrlrs, tailq, ctrlr_tmp) {
+			TAILQ_REMOVE(&probe_ctx->init_ctrlrs, ctrlr, tailq);
+			nvme_transport_ctrlr_destruct(ctrlr);
+		}
+		nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock);
+		return -1;
+	}
+
+	/*
+	 * Probe controllers on the shared_attached_ctrlrs list
+	 */
+	if (!spdk_process_is_primary() && (probe_ctx->trid.trtype == SPDK_NVME_TRANSPORT_PCIE)) {
+		TAILQ_FOREACH(ctrlr, &g_spdk_nvme_driver->shared_attached_ctrlrs, tailq) {
+			/* Do not attach other ctrlrs if user specify a valid trid */
+			if ((strlen(probe_ctx->trid.traddr) != 0) &&
+			    (spdk_nvme_transport_id_compare(&probe_ctx->trid, &ctrlr->trid))) {
+				continue;
+			}
+
+			/* Do not attach if we failed to initialize it in this process */
+			if (nvme_ctrlr_get_current_process(ctrlr) == NULL) {
+				continue;
+			}
+
+			nvme_ctrlr_proc_get_ref(ctrlr);
+
+			/*
+			 * Unlock while calling attach_cb() so the user can call other functions
+			 *  that may take the driver lock, like nvme_detach().
+			 */
+			if (probe_ctx->attach_cb) {
+				nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock);
+				probe_ctx->attach_cb(probe_ctx->cb_ctx, &ctrlr->trid, ctrlr, &ctrlr->opts);
+				nvme_robust_mutex_lock(&g_spdk_nvme_driver->lock);
+			}
+		}
+	}
+
+	nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock);
+
+	return 0;
+}
+
+static void
+nvme_probe_ctx_init(struct spdk_nvme_probe_ctx *probe_ctx,
+		    const struct spdk_nvme_transport_id *trid,
+		    void *cb_ctx,
+		    spdk_nvme_probe_cb probe_cb,
+		    spdk_nvme_attach_cb attach_cb,
+		    spdk_nvme_remove_cb remove_cb)
+{
+	probe_ctx->trid = *trid;
+	probe_ctx->cb_ctx = cb_ctx;
+	probe_ctx->probe_cb = probe_cb;
+	probe_ctx->attach_cb = attach_cb;
+	probe_ctx->remove_cb = remove_cb;
+	TAILQ_INIT(&probe_ctx->init_ctrlrs);
+}
+
+int
+spdk_nvme_probe(const struct spdk_nvme_transport_id *trid, void *cb_ctx,
+		spdk_nvme_probe_cb probe_cb, spdk_nvme_attach_cb attach_cb,
+		spdk_nvme_remove_cb remove_cb)
+{
+	struct spdk_nvme_transport_id trid_pcie;
+	struct spdk_nvme_probe_ctx *probe_ctx;
+
+	if (trid == NULL) {
+		memset(&trid_pcie, 0, sizeof(trid_pcie));
+		spdk_nvme_trid_populate_transport(&trid_pcie, SPDK_NVME_TRANSPORT_PCIE);
+		trid = &trid_pcie;
+	}
+
+	probe_ctx = spdk_nvme_probe_async(trid, cb_ctx, probe_cb,
+					  attach_cb, remove_cb);
+	if (!probe_ctx) {
+		SPDK_ERRLOG("Create probe context failed\n");
+		return -1;
+	}
+
+	/*
+	 * Keep going even if one or more nvme_attach() calls failed,
+	 *  but maintain the value of rc to signal errors when we return.
+	 */
+	return nvme_init_controllers(probe_ctx);
+}
+
+static bool
+nvme_connect_probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
+		      struct spdk_nvme_ctrlr_opts *opts)
+{
+	struct spdk_nvme_ctrlr_opts *requested_opts = cb_ctx;
+
+	assert(requested_opts);
+	memcpy(opts, requested_opts, sizeof(*opts));
+
+	return true;
+}
+
+static void
+nvme_ctrlr_opts_init(struct spdk_nvme_ctrlr_opts *opts,
+		     const struct spdk_nvme_ctrlr_opts *opts_user,
+		     size_t opts_size_user)
+{
+	assert(opts);
+	assert(opts_user);
+
+	spdk_nvme_ctrlr_get_default_ctrlr_opts(opts, opts_size_user);
+
+#define FIELD_OK(field) \
+        offsetof(struct spdk_nvme_ctrlr_opts, field) + sizeof(opts->field) <= (opts->opts_size)
+
+	if (FIELD_OK(num_io_queues)) {
+		opts->num_io_queues = opts_user->num_io_queues;
+	}
+
+	if (FIELD_OK(use_cmb_sqs)) {
+		opts->use_cmb_sqs = opts_user->use_cmb_sqs;
+	}
+
+	if (FIELD_OK(no_shn_notification)) {
+		opts->no_shn_notification = opts_user->no_shn_notification;
+	}
+
+	if (FIELD_OK(arb_mechanism)) {
+		opts->arb_mechanism = opts_user->arb_mechanism;
+	}
+
+	if (FIELD_OK(arbitration_burst)) {
+		opts->arbitration_burst = opts_user->arbitration_burst;
+	}
+
+	if (FIELD_OK(low_priority_weight)) {
+		opts->low_priority_weight = opts_user->low_priority_weight;
+	}
+
+	if (FIELD_OK(medium_priority_weight)) {
+		opts->medium_priority_weight = opts_user->medium_priority_weight;
+	}
+
+	if (FIELD_OK(high_priority_weight)) {
+		opts->high_priority_weight = opts_user->high_priority_weight;
+	}
+
+	if (FIELD_OK(keep_alive_timeout_ms)) {
+		opts->keep_alive_timeout_ms =  opts_user->keep_alive_timeout_ms;
+	}
+
+	if (FIELD_OK(transport_retry_count)) {
+		opts->transport_retry_count = opts_user->transport_retry_count;
+	}
+
+	if (FIELD_OK(io_queue_size)) {
+		opts->io_queue_size =  opts_user->io_queue_size;
+	}
+
+	if (FIELD_OK(hostnqn)) {
+		memcpy(opts->hostnqn, opts_user->hostnqn, sizeof(opts_user->hostnqn));
+	}
+
+	if (FIELD_OK(io_queue_requests)) {
+		opts->io_queue_requests =  opts_user->io_queue_requests;
+	}
+
+	if (FIELD_OK(src_addr)) {
+		memcpy(opts->src_addr, opts_user->src_addr, sizeof(opts_user->src_addr));
+	}
+
+	if (FIELD_OK(src_svcid)) {
+		memcpy(opts->src_svcid, opts_user->src_svcid, sizeof(opts_user->src_svcid));
+	}
+
+	if (FIELD_OK(host_id)) {
+		memcpy(opts->host_id, opts_user->host_id, sizeof(opts_user->host_id));
+	}
+	if (FIELD_OK(extended_host_id)) {
+		memcpy(opts->extended_host_id, opts_user->extended_host_id,
+		       sizeof(opts_user->extended_host_id));
+	}
+
+	if (FIELD_OK(command_set)) {
+		opts->command_set = opts_user->command_set;
+	}
+
+	if (FIELD_OK(admin_timeout_ms)) {
+		opts->admin_timeout_ms = opts_user->admin_timeout_ms;
+	}
+
+	if (FIELD_OK(header_digest)) {
+		opts->header_digest = opts_user->header_digest;
+	}
+
+	if (FIELD_OK(data_digest)) {
+		opts->data_digest = opts_user->data_digest;
+	}
+
+	if (FIELD_OK(disable_error_logging)) {
+		opts->disable_error_logging = opts_user->disable_error_logging;
+	}
+
+	if (FIELD_OK(transport_ack_timeout)) {
+		opts->transport_ack_timeout = opts_user->transport_ack_timeout;
+	}
+
+	if (FIELD_OK(admin_queue_size)) {
+		opts->admin_queue_size = opts_user->admin_queue_size;
+	}
+#undef FIELD_OK
+}
+
+struct spdk_nvme_ctrlr *
+spdk_nvme_connect(const struct spdk_nvme_transport_id *trid,
+		  const struct spdk_nvme_ctrlr_opts *opts, size_t opts_size)
+{
+	int rc;
+	struct spdk_nvme_ctrlr *ctrlr = NULL;
+	struct spdk_nvme_probe_ctx *probe_ctx;
+	struct spdk_nvme_ctrlr_opts *opts_local_p = NULL;
+	struct spdk_nvme_ctrlr_opts opts_local;
+
+	if (trid == NULL) {
+		SPDK_ERRLOG("No transport ID specified\n");
+		return NULL;
+	}
+
+	if (opts) {
+		opts_local_p = &opts_local;
+		nvme_ctrlr_opts_init(opts_local_p, opts, opts_size);
+	}
+
+	probe_ctx = spdk_nvme_connect_async(trid, opts_local_p, NULL);
+	if (!probe_ctx) {
+		SPDK_ERRLOG("Create probe context failed\n");
+		return NULL;
+	}
+
+	rc = nvme_init_controllers(probe_ctx);
+	if (rc != 0) {
+		return NULL;
+	}
+
+	ctrlr = nvme_get_ctrlr_by_trid(trid);
+
+	return ctrlr;
+}
+
+void
+spdk_nvme_trid_populate_transport(struct spdk_nvme_transport_id *trid,
+				  enum spdk_nvme_transport_type trtype)
+{
+	const char *trstring = "";
+
+	trid->trtype = trtype;
+	switch (trtype) {
+	case SPDK_NVME_TRANSPORT_FC:
+		trstring = SPDK_NVME_TRANSPORT_NAME_FC;
+		break;
+	case SPDK_NVME_TRANSPORT_PCIE:
+		trstring = SPDK_NVME_TRANSPORT_NAME_PCIE;
+		break;
+	case SPDK_NVME_TRANSPORT_RDMA:
+		trstring = SPDK_NVME_TRANSPORT_NAME_RDMA;
+		break;
+	case SPDK_NVME_TRANSPORT_TCP:
+		trstring = SPDK_NVME_TRANSPORT_NAME_TCP;
+		break;
+	case SPDK_NVME_TRANSPORT_CUSTOM:
+	default:
+		SPDK_ERRLOG("don't use this for custom transports\n");
+		assert(0);
+		return;
+	}
+	snprintf(trid->trstring, SPDK_NVMF_TRSTRING_MAX_LEN, "%s", trstring);
+}
+
+int
+spdk_nvme_transport_id_populate_trstring(struct spdk_nvme_transport_id *trid, const char *trstring)
+{
+	int len, i, rc;
+
+	if (trstring == NULL) {
+		return -EINVAL;
+	}
+
+	len = strnlen(trstring, SPDK_NVMF_TRSTRING_MAX_LEN);
+	if (len == SPDK_NVMF_TRSTRING_MAX_LEN) {
+		return -EINVAL;
+	}
+
+	rc = snprintf(trid->trstring, SPDK_NVMF_TRSTRING_MAX_LEN, "%s", trstring);
+	if (rc < 0) {
+		return rc;
+	}
+
+	/* cast official trstring to uppercase version of input. */
+	for (i = 0; i < len; i++) {
+		trid->trstring[i] = toupper(trid->trstring[i]);
+	}
+	return 0;
+}
+
+int
+spdk_nvme_transport_id_parse_trtype(enum spdk_nvme_transport_type *trtype, const char *str)
+{
+	if (trtype == NULL || str == NULL) {
+		return -EINVAL;
+	}
+
+	if (strcasecmp(str, "PCIe") == 0) {
+		*trtype = SPDK_NVME_TRANSPORT_PCIE;
+	} else if (strcasecmp(str, "RDMA") == 0) {
+		*trtype = SPDK_NVME_TRANSPORT_RDMA;
+	} else if (strcasecmp(str, "FC") == 0) {
+		*trtype = SPDK_NVME_TRANSPORT_FC;
+	} else if (strcasecmp(str, "TCP") == 0) {
+		*trtype = SPDK_NVME_TRANSPORT_TCP;
+	} else {
+		*trtype = SPDK_NVME_TRANSPORT_CUSTOM;
+	}
+	return 0;
+}
+
+const char *
+spdk_nvme_transport_id_trtype_str(enum spdk_nvme_transport_type trtype)
+{
+	switch (trtype) {
+	case SPDK_NVME_TRANSPORT_PCIE:
+		return "PCIe";
+	case SPDK_NVME_TRANSPORT_RDMA:
+		return "RDMA";
+	case SPDK_NVME_TRANSPORT_FC:
+		return "FC";
+	case SPDK_NVME_TRANSPORT_TCP:
+		return "TCP";
+	case SPDK_NVME_TRANSPORT_CUSTOM:
+		return "CUSTOM";
+	default:
+		return NULL;
+	}
+}
+
+int
+spdk_nvme_transport_id_parse_adrfam(enum spdk_nvmf_adrfam *adrfam, const char *str)
+{
+	if (adrfam == NULL || str == NULL) {
+		return -EINVAL;
+	}
+
+	if (strcasecmp(str, "IPv4") == 0) {
+		*adrfam = SPDK_NVMF_ADRFAM_IPV4;
+	} else if (strcasecmp(str, "IPv6") == 0) {
+		*adrfam = SPDK_NVMF_ADRFAM_IPV6;
+	} else if (strcasecmp(str, "IB") == 0) {
+		*adrfam = SPDK_NVMF_ADRFAM_IB;
+	} else if (strcasecmp(str, "FC") == 0) {
+		*adrfam = SPDK_NVMF_ADRFAM_FC;
+	} else {
+		return -ENOENT;
+	}
+	return 0;
+}
+
+const char *
+spdk_nvme_transport_id_adrfam_str(enum spdk_nvmf_adrfam adrfam)
+{
+	switch (adrfam) {
+	case SPDK_NVMF_ADRFAM_IPV4:
+		return "IPv4";
+	case SPDK_NVMF_ADRFAM_IPV6:
+		return "IPv6";
+	case SPDK_NVMF_ADRFAM_IB:
+		return "IB";
+	case SPDK_NVMF_ADRFAM_FC:
+		return "FC";
+	default:
+		return NULL;
+	}
+}
+
+static size_t
+parse_next_key(const char **str, char *key, char *val, size_t key_buf_size, size_t val_buf_size)
+{
+
+	const char *sep, *sep1;
+	const char *whitespace = " \t\n";
+	size_t key_len, val_len;
+
+	*str += strspn(*str, whitespace);
+
+	sep = strchr(*str, ':');
+	if (!sep) {
+		sep = strchr(*str, '=');
+		if (!sep) {
+			SPDK_ERRLOG("Key without ':' or '=' separator\n");
+			return 0;
+		}
+	} else {
+		sep1 = strchr(*str, '=');
+		if ((sep1 != NULL) && (sep1 < sep)) {
+			sep = sep1;
+		}
+	}
+
+	key_len = sep - *str;
+	if (key_len >= key_buf_size) {
+		SPDK_ERRLOG("Key length %zu greater than maximum allowed %zu\n",
+			    key_len, key_buf_size - 1);
+		return 0;
+	}
+
+	memcpy(key, *str, key_len);
+	key[key_len] = '\0';
+
+	*str += key_len + 1; /* Skip key: */
+	val_len = strcspn(*str, whitespace);
+	if (val_len == 0) {
+		SPDK_ERRLOG("Key without value\n");
+		return 0;
+	}
+
+	if (val_len >= val_buf_size) {
+		SPDK_ERRLOG("Value length %zu greater than maximum allowed %zu\n",
+			    val_len, val_buf_size - 1);
+		return 0;
+	}
+
+	memcpy(val, *str, val_len);
+	val[val_len] = '\0';
+
+	*str += val_len;
+
+	return val_len;
+}
+
+int
+spdk_nvme_transport_id_parse(struct spdk_nvme_transport_id *trid, const char *str)
+{
+	size_t val_len;
+	char key[32];
+	char val[1024];
+
+	if (trid == NULL || str == NULL) {
+		return -EINVAL;
+	}
+
+	while (*str != '\0') {
+
+		val_len = parse_next_key(&str, key, val, sizeof(key), sizeof(val));
+
+		if (val_len == 0) {
+			SPDK_ERRLOG("Failed to parse transport ID\n");
+			return -EINVAL;
+		}
+
+		if (strcasecmp(key, "trtype") == 0) {
+			if (spdk_nvme_transport_id_populate_trstring(trid, val) != 0) {
+				SPDK_ERRLOG("invalid transport '%s'\n", val);
+				return -EINVAL;
+			}
+			if (spdk_nvme_transport_id_parse_trtype(&trid->trtype, val) != 0) {
+				SPDK_ERRLOG("Unknown trtype '%s'\n", val);
+				return -EINVAL;
+			}
+		} else if (strcasecmp(key, "adrfam") == 0) {
+			if (spdk_nvme_transport_id_parse_adrfam(&trid->adrfam, val) != 0) {
+				SPDK_ERRLOG("Unknown adrfam '%s'\n", val);
+				return -EINVAL;
+			}
+		} else if (strcasecmp(key, "traddr") == 0) {
+			if (val_len > SPDK_NVMF_TRADDR_MAX_LEN) {
+				SPDK_ERRLOG("traddr length %zu greater than maximum allowed %u\n",
+					    val_len, SPDK_NVMF_TRADDR_MAX_LEN);
+				return -EINVAL;
+			}
+			memcpy(trid->traddr, val, val_len + 1);
+		} else if (strcasecmp(key, "trsvcid") == 0) {
+			if (val_len > SPDK_NVMF_TRSVCID_MAX_LEN) {
+				SPDK_ERRLOG("trsvcid length %zu greater than maximum allowed %u\n",
+					    val_len, SPDK_NVMF_TRSVCID_MAX_LEN);
+				return -EINVAL;
+			}
+			memcpy(trid->trsvcid, val, val_len + 1);
+		} else if (strcasecmp(key, "priority") == 0) {
+			if (val_len > SPDK_NVMF_PRIORITY_MAX_LEN) {
+				SPDK_ERRLOG("priority length %zu greater than maximum allowed %u\n",
+					    val_len, SPDK_NVMF_PRIORITY_MAX_LEN);
+				return -EINVAL;
+			}
+			trid->priority = spdk_strtol(val, 10);
+		} else if (strcasecmp(key, "subnqn") == 0) {
+			if (val_len > SPDK_NVMF_NQN_MAX_LEN) {
+				SPDK_ERRLOG("subnqn length %zu greater than maximum allowed %u\n",
+					    val_len, SPDK_NVMF_NQN_MAX_LEN);
+				return -EINVAL;
+			}
+			memcpy(trid->subnqn, val, val_len + 1);
+		} else if (strcasecmp(key, "hostaddr") == 0) {
+			continue;
+		} else if (strcasecmp(key, "hostsvcid") == 0) {
+			continue;
+		} else if (strcasecmp(key, "ns") == 0) {
+			/*
+			 * Special case.  The namespace id parameter may
+			 * optionally be passed in the transport id string
+			 * for an SPDK application (e.g. nvme/perf)
+			 * and additionally parsed therein to limit
+			 * targeting a specific namespace.  For this
+			 * scenario, just silently ignore this key
+			 * rather than letting it default to logging
+			 * it as an invalid key.
+			 */
+			continue;
+		} else if (strcasecmp(key, "alt_traddr") == 0) {
+			/*
+			 * Used by applications for enabling transport ID failover.
+			 * Please see the case above for more information on custom parameters.
+			 */
+			continue;
+		} else {
+			SPDK_ERRLOG("Unknown transport ID key '%s'\n", key);
+		}
+	}
+
+	return 0;
+}
+
+int
+spdk_nvme_host_id_parse(struct spdk_nvme_host_id *hostid, const char *str)
+{
+
+	size_t key_size = 32;
+	size_t val_size = 1024;
+	size_t val_len;
+	char key[key_size];
+	char val[val_size];
+
+	if (hostid == NULL || str == NULL) {
+		return -EINVAL;
+	}
+
+	while (*str != '\0') {
+
+		val_len = parse_next_key(&str, key, val, key_size, val_size);
+
+		if (val_len == 0) {
+			SPDK_ERRLOG("Failed to parse host ID\n");
+			return val_len;
+		}
+
+		/* Ignore the rest of the options from the transport ID. */
+		if (strcasecmp(key, "trtype") == 0) {
+			continue;
+		} else if (strcasecmp(key, "adrfam") == 0) {
+			continue;
+		} else if (strcasecmp(key, "traddr") == 0) {
+			continue;
+		} else if (strcasecmp(key, "trsvcid") == 0) {
+			continue;
+		} else if (strcasecmp(key, "subnqn") == 0) {
+			continue;
+		} else if (strcasecmp(key, "priority") == 0) {
+			continue;
+		} else if (strcasecmp(key, "ns") == 0) {
+			continue;
+		} else if (strcasecmp(key, "hostaddr") == 0) {
+			if (val_len > SPDK_NVMF_TRADDR_MAX_LEN) {
+				SPDK_ERRLOG("hostaddr length %zu greater than maximum allowed %u\n",
+					    val_len, SPDK_NVMF_TRADDR_MAX_LEN);
+				return -EINVAL;
+			}
+			memcpy(hostid->hostaddr, val, val_len + 1);
+
+		} else if (strcasecmp(key, "hostsvcid") == 0) {
+			if (val_len > SPDK_NVMF_TRSVCID_MAX_LEN) {
+				SPDK_ERRLOG("trsvcid length %zu greater than maximum allowed %u\n",
+					    val_len, SPDK_NVMF_TRSVCID_MAX_LEN);
+				return -EINVAL;
+			}
+			memcpy(hostid->hostsvcid, val, val_len + 1);
+		} else {
+			SPDK_ERRLOG("Unknown transport ID key '%s'\n", key);
+		}
+	}
+
+	return 0;
+}
+
+static int
+cmp_int(int a, int b)
+{
+	return a - b;
+}
+
+int
+spdk_nvme_transport_id_compare(const struct spdk_nvme_transport_id *trid1,
+			       const struct spdk_nvme_transport_id *trid2)
+{
+	int cmp;
+
+	if (trid1->trtype == SPDK_NVME_TRANSPORT_CUSTOM) {
+		cmp = strcasecmp(trid1->trstring, trid2->trstring);
+	} else {
+		cmp = cmp_int(trid1->trtype, trid2->trtype);
+	}
+
+	if (cmp) {
+		return cmp;
+	}
+
+	if (trid1->trtype == SPDK_NVME_TRANSPORT_PCIE) {
+		struct spdk_pci_addr pci_addr1 = {};
+		struct spdk_pci_addr pci_addr2 = {};
+
+		/* Normalize PCI addresses before comparing */
+		if (spdk_pci_addr_parse(&pci_addr1, trid1->traddr) < 0 ||
+		    spdk_pci_addr_parse(&pci_addr2, trid2->traddr) < 0) {
+			return -1;
+		}
+
+		/* PCIe transport ID only uses trtype and traddr */
+		return spdk_pci_addr_compare(&pci_addr1, &pci_addr2);
+	}
+
+	cmp = strcasecmp(trid1->traddr, trid2->traddr);
+	if (cmp) {
+		return cmp;
+	}
+
+	cmp = cmp_int(trid1->adrfam, trid2->adrfam);
+	if (cmp) {
+		return cmp;
+	}
+
+	cmp = strcasecmp(trid1->trsvcid, trid2->trsvcid);
+	if (cmp) {
+		return cmp;
+	}
+
+	cmp = strcmp(trid1->subnqn, trid2->subnqn);
+	if (cmp) {
+		return cmp;
+	}
+
+	return 0;
+}
+
+int
+spdk_nvme_prchk_flags_parse(uint32_t *prchk_flags, const char *str)
+{
+	size_t val_len;
+	char key[32];
+	char val[1024];
+
+	if (prchk_flags == NULL || str == NULL) {
+		return -EINVAL;
+	}
+
+	while (*str != '\0') {
+		val_len = parse_next_key(&str, key, val, sizeof(key), sizeof(val));
+
+		if (val_len == 0) {
+			SPDK_ERRLOG("Failed to parse prchk\n");
+			return -EINVAL;
+		}
+
+		if (strcasecmp(key, "prchk") == 0) {
+			if (strcasestr(val, "reftag") != NULL) {
+				*prchk_flags |= SPDK_NVME_IO_FLAGS_PRCHK_REFTAG;
+			}
+			if (strcasestr(val, "guard") != NULL) {
+				*prchk_flags |= SPDK_NVME_IO_FLAGS_PRCHK_GUARD;
+			}
+		} else {
+			SPDK_ERRLOG("Unknown key '%s'\n", key);
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+const char *
+spdk_nvme_prchk_flags_str(uint32_t prchk_flags)
+{
+	if (prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_REFTAG) {
+		if (prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_GUARD) {
+			return "prchk:reftag|guard";
+		} else {
+			return "prchk:reftag";
+		}
+	} else {
+		if (prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_GUARD) {
+			return "prchk:guard";
+		} else {
+			return NULL;
+		}
+	}
+}
+
+struct spdk_nvme_probe_ctx *
+spdk_nvme_probe_async(const struct spdk_nvme_transport_id *trid,
+		      void *cb_ctx,
+		      spdk_nvme_probe_cb probe_cb,
+		      spdk_nvme_attach_cb attach_cb,
+		      spdk_nvme_remove_cb remove_cb)
+{
+	int rc;
+	struct spdk_nvme_probe_ctx *probe_ctx;
+
+	rc = nvme_driver_init();
+	if (rc != 0) {
+		return NULL;
+	}
+
+	probe_ctx = calloc(1, sizeof(*probe_ctx));
+	if (!probe_ctx) {
+		return NULL;
+	}
+
+	nvme_probe_ctx_init(probe_ctx, trid, cb_ctx, probe_cb, attach_cb, remove_cb);
+	rc = nvme_probe_internal(probe_ctx, false);
+	if (rc != 0) {
+		free(probe_ctx);
+		return NULL;
+	}
+
+	return probe_ctx;
+}
+
+int
+spdk_nvme_probe_poll_async(struct spdk_nvme_probe_ctx *probe_ctx)
+{
+	int rc = 0;
+	struct spdk_nvme_ctrlr *ctrlr, *ctrlr_tmp;
+
+	if (!spdk_process_is_primary() && probe_ctx->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) {
+		free(probe_ctx);
+		return 0;
+	}
+
+	TAILQ_FOREACH_SAFE(ctrlr, &probe_ctx->init_ctrlrs, tailq, ctrlr_tmp) {
+		rc = nvme_ctrlr_poll_internal(ctrlr, probe_ctx);
+		if (rc != 0) {
+			rc = -EIO;
+			break;
+		}
+	}
+
+	if (rc != 0 || TAILQ_EMPTY(&probe_ctx->init_ctrlrs)) {
+		nvme_robust_mutex_lock(&g_spdk_nvme_driver->lock);
+		g_spdk_nvme_driver->initialized = true;
+		nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock);
+		free(probe_ctx);
+		return rc;
+	}
+
+	return -EAGAIN;
+}
+
+struct spdk_nvme_probe_ctx *
+spdk_nvme_connect_async(const struct spdk_nvme_transport_id *trid,
+			const struct spdk_nvme_ctrlr_opts *opts,
+			spdk_nvme_attach_cb attach_cb)
+{
+	int rc;
+	spdk_nvme_probe_cb probe_cb = NULL;
+	struct spdk_nvme_probe_ctx *probe_ctx;
+
+	rc = nvme_driver_init();
+	if (rc != 0) {
+		return NULL;
+	}
+
+	probe_ctx = calloc(1, sizeof(*probe_ctx));
+	if (!probe_ctx) {
+		return NULL;
+	}
+
+	if (opts) {
+		probe_cb = nvme_connect_probe_cb;
+	}
+
+	nvme_probe_ctx_init(probe_ctx, trid, (void *)opts, probe_cb, attach_cb, NULL);
+	rc = nvme_probe_internal(probe_ctx, true);
+	if (rc != 0) {
+		free(probe_ctx);
+		return NULL;
+	}
+
+	return probe_ctx;
+}
+
+SPDK_LOG_REGISTER_COMPONENT("nvme", SPDK_LOG_NVME)
diff --git a/src/spdk/lib/nvme/nvme_ctrlr.c b/src/spdk/lib/nvme/nvme_ctrlr.c
new file mode 100644
index 000000000..ced02e9bb
--- /dev/null
+++ b/src/spdk/lib/nvme/nvme_ctrlr.c
@@ -0,0 +1,3639 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation. All rights reserved.
+ *   Copyright (c) 2019, 2020 Mellanox Technologies LTD. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+
+#include "nvme_internal.h"
+#include "nvme_io_msg.h"
+
+#include "spdk/env.h"
+#include "spdk/string.h"
+
+struct nvme_active_ns_ctx;
+
+static void nvme_ctrlr_destruct_namespaces(struct spdk_nvme_ctrlr *ctrlr);
+static int nvme_ctrlr_construct_and_submit_aer(struct spdk_nvme_ctrlr *ctrlr,
+		struct nvme_async_event_request *aer);
+static void nvme_ctrlr_identify_active_ns_async(struct nvme_active_ns_ctx *ctx);
+static int nvme_ctrlr_identify_ns_async(struct spdk_nvme_ns *ns);
+static int nvme_ctrlr_identify_id_desc_async(struct spdk_nvme_ns *ns);
+
+static int
+nvme_ctrlr_get_cc(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_cc_register *cc)
+{
+	return nvme_transport_ctrlr_get_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, cc.raw),
+					      &cc->raw);
+}
+
+static int
+nvme_ctrlr_get_csts(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_csts_register *csts)
+{
+	return nvme_transport_ctrlr_get_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, csts.raw),
+					      &csts->raw);
+}
+
+int
+nvme_ctrlr_get_cap(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_cap_register *cap)
+{
+	return nvme_transport_ctrlr_get_reg_8(ctrlr, offsetof(struct spdk_nvme_registers, cap.raw),
+					      &cap->raw);
+}
+
+int
+nvme_ctrlr_get_vs(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_vs_register *vs)
+{
+	return nvme_transport_ctrlr_get_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, vs.raw),
+					      &vs->raw);
+}
+
+static int
+nvme_ctrlr_set_cc(struct spdk_nvme_ctrlr *ctrlr, const union spdk_nvme_cc_register *cc)
+{
+	return nvme_transport_ctrlr_set_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, cc.raw),
+					      cc->raw);
+}
+
+int
+nvme_ctrlr_get_cmbsz(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_cmbsz_register *cmbsz)
+{
+	return nvme_transport_ctrlr_get_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, cmbsz.raw),
+					      &cmbsz->raw);
+}
+
+/* When the field in spdk_nvme_ctrlr_opts are changed and you change this function, please
+ * also update the nvme_ctrl_opts_init function in nvme_ctrlr.c
+ */
+void
+spdk_nvme_ctrlr_get_default_ctrlr_opts(struct spdk_nvme_ctrlr_opts *opts, size_t opts_size)
+{
+	char host_id_str[SPDK_UUID_STRING_LEN];
+
+	assert(opts);
+
+	opts->opts_size = opts_size;
+
+#define FIELD_OK(field) \
+	offsetof(struct spdk_nvme_ctrlr_opts, field) + sizeof(opts->field) <= opts_size
+
+	if (FIELD_OK(num_io_queues)) {
+		opts->num_io_queues = DEFAULT_MAX_IO_QUEUES;
+	}
+
+	if (FIELD_OK(use_cmb_sqs)) {
+		opts->use_cmb_sqs = true;
+	}
+
+	if (FIELD_OK(no_shn_notification)) {
+		opts->no_shn_notification = false;
+	}
+
+	if (FIELD_OK(arb_mechanism)) {
+		opts->arb_mechanism = SPDK_NVME_CC_AMS_RR;
+	}
+
+	if (FIELD_OK(arbitration_burst)) {
+		opts->arbitration_burst = 0;
+	}
+
+	if (FIELD_OK(low_priority_weight)) {
+		opts->low_priority_weight = 0;
+	}
+
+	if (FIELD_OK(medium_priority_weight)) {
+		opts->medium_priority_weight = 0;
+	}
+
+	if (FIELD_OK(high_priority_weight)) {
+		opts->high_priority_weight = 0;
+	}
+
+	if (FIELD_OK(keep_alive_timeout_ms)) {
+		opts->keep_alive_timeout_ms = MIN_KEEP_ALIVE_TIMEOUT_IN_MS;
+	}
+
+	if (FIELD_OK(transport_retry_count)) {
+		opts->transport_retry_count = SPDK_NVME_DEFAULT_RETRY_COUNT;
+	}
+
+	if (FIELD_OK(io_queue_size)) {
+		opts->io_queue_size = DEFAULT_IO_QUEUE_SIZE;
+	}
+
+	if (nvme_driver_init() == 0) {
+		if (FIELD_OK(hostnqn)) {
+			spdk_uuid_fmt_lower(host_id_str, sizeof(host_id_str),
+					    &g_spdk_nvme_driver->default_extended_host_id);
+			snprintf(opts->hostnqn, sizeof(opts->hostnqn), "2014-08.org.nvmexpress:uuid:%s", host_id_str);
+		}
+
+		if (FIELD_OK(extended_host_id)) {
+			memcpy(opts->extended_host_id, &g_spdk_nvme_driver->default_extended_host_id,
+			       sizeof(opts->extended_host_id));
+		}
+
+	}
+
+	if (FIELD_OK(io_queue_requests)) {
+		opts->io_queue_requests = DEFAULT_IO_QUEUE_REQUESTS;
+	}
+
+	if (FIELD_OK(src_addr)) {
+		memset(opts->src_addr, 0, sizeof(opts->src_addr));
+	}
+
+	if (FIELD_OK(src_svcid)) {
+		memset(opts->src_svcid, 0, sizeof(opts->src_svcid));
+	}
+
+	if (FIELD_OK(host_id)) {
+		memset(opts->host_id, 0, sizeof(opts->host_id));
+	}
+
+	if (FIELD_OK(command_set)) {
+		opts->command_set = SPDK_NVME_CC_CSS_NVM;
+	}
+
+	if (FIELD_OK(admin_timeout_ms)) {
+		opts->admin_timeout_ms = NVME_MAX_ADMIN_TIMEOUT_IN_SECS * 1000;
+	}
+
+	if (FIELD_OK(header_digest)) {
+		opts->header_digest = false;
+	}
+
+	if (FIELD_OK(data_digest)) {
+		opts->data_digest = false;
+	}
+
+	if (FIELD_OK(disable_error_logging)) {
+		opts->disable_error_logging = false;
+	}
+
+	if (FIELD_OK(transport_ack_timeout)) {
+		opts->transport_ack_timeout = SPDK_NVME_DEFAULT_TRANSPORT_ACK_TIMEOUT;
+	}
+
+	if (FIELD_OK(admin_queue_size)) {
+		opts->admin_queue_size = DEFAULT_ADMIN_QUEUE_SIZE;
+	}
+#undef FIELD_OK
+}
+
+/**
+ * This function will be called when the process allocates the IO qpair.
+ * Note: the ctrlr_lock must be held when calling this function.
+ */
+static void
+nvme_ctrlr_proc_add_io_qpair(struct spdk_nvme_qpair *qpair)
+{
+	struct spdk_nvme_ctrlr_process	*active_proc;
+	struct spdk_nvme_ctrlr		*ctrlr = qpair->ctrlr;
+
+	active_proc = nvme_ctrlr_get_current_process(ctrlr);
+	if (active_proc) {
+		TAILQ_INSERT_TAIL(&active_proc->allocated_io_qpairs, qpair, per_process_tailq);
+		qpair->active_proc = active_proc;
+	}
+}
+
+/**
+ * This function will be called when the process frees the IO qpair.
+ * Note: the ctrlr_lock must be held when calling this function.
+ */
+static void
+nvme_ctrlr_proc_remove_io_qpair(struct spdk_nvme_qpair *qpair)
+{
+	struct spdk_nvme_ctrlr_process	*active_proc;
+	struct spdk_nvme_ctrlr		*ctrlr = qpair->ctrlr;
+	struct spdk_nvme_qpair          *active_qpair, *tmp_qpair;
+
+	active_proc = nvme_ctrlr_get_current_process(ctrlr);
+	if (!active_proc) {
+		return;
+	}
+
+	TAILQ_FOREACH_SAFE(active_qpair, &active_proc->allocated_io_qpairs,
+			   per_process_tailq, tmp_qpair) {
+		if (active_qpair == qpair) {
+			TAILQ_REMOVE(&active_proc->allocated_io_qpairs,
+				     active_qpair, per_process_tailq);
+
+			break;
+		}
+	}
+}
+
+void
+spdk_nvme_ctrlr_get_default_io_qpair_opts(struct spdk_nvme_ctrlr *ctrlr,
+		struct spdk_nvme_io_qpair_opts *opts,
+		size_t opts_size)
+{
+	assert(ctrlr);
+
+	assert(opts);
+
+	memset(opts, 0, opts_size);
+
+#define FIELD_OK(field) \
+	offsetof(struct spdk_nvme_io_qpair_opts, field) + sizeof(opts->field) <= opts_size
+
+	if (FIELD_OK(qprio)) {
+		opts->qprio = SPDK_NVME_QPRIO_URGENT;
+	}
+
+	if (FIELD_OK(io_queue_size)) {
+		opts->io_queue_size = ctrlr->opts.io_queue_size;
+	}
+
+	if (FIELD_OK(io_queue_requests)) {
+		opts->io_queue_requests = ctrlr->opts.io_queue_requests;
+	}
+
+	if (FIELD_OK(delay_cmd_submit)) {
+		opts->delay_cmd_submit = false;
+	}
+
+	if (FIELD_OK(sq.vaddr)) {
+		opts->sq.vaddr = NULL;
+	}
+
+	if (FIELD_OK(sq.paddr)) {
+		opts->sq.paddr = 0;
+	}
+
+	if (FIELD_OK(sq.buffer_size)) {
+		opts->sq.buffer_size = 0;
+	}
+
+	if (FIELD_OK(cq.vaddr)) {
+		opts->cq.vaddr = NULL;
+	}
+
+	if (FIELD_OK(cq.paddr)) {
+		opts->cq.paddr = 0;
+	}
+
+	if (FIELD_OK(cq.buffer_size)) {
+		opts->cq.buffer_size = 0;
+	}
+
+	if (FIELD_OK(create_only)) {
+		opts->create_only = false;
+	}
+
+#undef FIELD_OK
+}
+
+static struct spdk_nvme_qpair *
+nvme_ctrlr_create_io_qpair(struct spdk_nvme_ctrlr *ctrlr,
+			   const struct spdk_nvme_io_qpair_opts *opts)
+{
+	uint32_t				qid;
+	struct spdk_nvme_qpair			*qpair;
+	union spdk_nvme_cc_register		cc;
+
+	if (!ctrlr) {
+		return NULL;
+	}
+
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+	if (nvme_ctrlr_get_cc(ctrlr, &cc)) {
+		SPDK_ERRLOG("get_cc failed\n");
+		nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+		return NULL;
+	}
+
+	if (opts->qprio & ~SPDK_NVME_CREATE_IO_SQ_QPRIO_MASK) {
+		nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+		return NULL;
+	}
+
+	/*
+	 * Only value SPDK_NVME_QPRIO_URGENT(0) is valid for the
+	 * default round robin arbitration method.
+	 */
+	if ((cc.bits.ams == SPDK_NVME_CC_AMS_RR) && (opts->qprio != SPDK_NVME_QPRIO_URGENT)) {
+		SPDK_ERRLOG("invalid queue priority for default round robin arbitration method\n");
+		nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+		return NULL;
+	}
+
+	/*
+	 * Get the first available I/O queue ID.
+	 */
+	qid = spdk_bit_array_find_first_set(ctrlr->free_io_qids, 1);
+	if (qid > ctrlr->opts.num_io_queues) {
+		SPDK_ERRLOG("No free I/O queue IDs\n");
+		nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+		return NULL;
+	}
+
+	qpair = nvme_transport_ctrlr_create_io_qpair(ctrlr, qid, opts);
+	if (qpair == NULL) {
+		SPDK_ERRLOG("nvme_transport_ctrlr_create_io_qpair() failed\n");
+		nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+		return NULL;
+	}
+
+	spdk_bit_array_clear(ctrlr->free_io_qids, qid);
+	TAILQ_INSERT_TAIL(&ctrlr->active_io_qpairs, qpair, tailq);
+
+	nvme_ctrlr_proc_add_io_qpair(qpair);
+
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+
+	return qpair;
+}
+
+int
+spdk_nvme_ctrlr_connect_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)
+{
+	int rc;
+
+	if (nvme_qpair_get_state(qpair) != NVME_QPAIR_DISCONNECTED) {
+		return -EISCONN;
+	}
+
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+	rc = nvme_transport_ctrlr_connect_qpair(ctrlr, qpair);
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+
+	if (ctrlr->quirks & NVME_QUIRK_DELAY_AFTER_QUEUE_ALLOC) {
+		spdk_delay_us(100);
+	}
+
+	return rc;
+}
+
+void
+spdk_nvme_ctrlr_disconnect_io_qpair(struct spdk_nvme_qpair *qpair)
+{
+	struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
+
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+	nvme_transport_ctrlr_disconnect_qpair(ctrlr, qpair);
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+}
+
+struct spdk_nvme_qpair *
+spdk_nvme_ctrlr_alloc_io_qpair(struct spdk_nvme_ctrlr *ctrlr,
+			       const struct spdk_nvme_io_qpair_opts *user_opts,
+			       size_t opts_size)
+{
+
+	struct spdk_nvme_qpair		*qpair;
+	struct spdk_nvme_io_qpair_opts	opts;
+	int				rc;
+
+	/*
+	 * Get the default options, then overwrite them with the user-provided options
+	 * up to opts_size.
+	 *
+	 * This allows for extensions of the opts structure without breaking
+	 * ABI compatibility.
+	 */
+	spdk_nvme_ctrlr_get_default_io_qpair_opts(ctrlr, &opts, sizeof(opts));
+	if (user_opts) {
+		memcpy(&opts, user_opts, spdk_min(sizeof(opts), opts_size));
+
+		/* If user passes buffers, make sure they're big enough for the requested queue size */
+		if (opts.sq.vaddr) {
+			if (opts.sq.buffer_size < (opts.io_queue_size * sizeof(struct spdk_nvme_cmd))) {
+				SPDK_ERRLOG("sq buffer size %lx is too small for sq size %lx\n",
+					    opts.sq.buffer_size, (opts.io_queue_size * sizeof(struct spdk_nvme_cmd)));
+				return NULL;
+			}
+		}
+		if (opts.cq.vaddr) {
+			if (opts.cq.buffer_size < (opts.io_queue_size * sizeof(struct spdk_nvme_cpl))) {
+				SPDK_ERRLOG("cq buffer size %lx is too small for cq size %lx\n",
+					    opts.cq.buffer_size, (opts.io_queue_size * sizeof(struct spdk_nvme_cpl)));
+				return NULL;
+			}
+		}
+	}
+
+	qpair = nvme_ctrlr_create_io_qpair(ctrlr, &opts);
+
+	if (qpair == NULL || opts.create_only == true) {
+		return qpair;
+	}
+
+	rc = spdk_nvme_ctrlr_connect_io_qpair(ctrlr, qpair);
+	if (rc != 0) {
+		SPDK_ERRLOG("nvme_transport_ctrlr_connect_io_qpair() failed\n");
+		nvme_transport_ctrlr_delete_io_qpair(ctrlr, qpair);
+		return NULL;
+	}
+
+	return qpair;
+}
+
+int
+spdk_nvme_ctrlr_reconnect_io_qpair(struct spdk_nvme_qpair *qpair)
+{
+	struct spdk_nvme_ctrlr *ctrlr;
+	enum nvme_qpair_state qpair_state;
+	int rc;
+
+	assert(qpair != NULL);
+	assert(nvme_qpair_is_admin_queue(qpair) == false);
+	assert(qpair->ctrlr != NULL);
+
+	ctrlr = qpair->ctrlr;
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+	qpair_state = nvme_qpair_get_state(qpair);
+
+	if (ctrlr->is_removed) {
+		rc = -ENODEV;
+		goto out;
+	}
+
+	if (ctrlr->is_resetting || qpair_state == NVME_QPAIR_DISCONNECTING) {
+		rc = -EAGAIN;
+		goto out;
+	}
+
+	if (ctrlr->is_failed || qpair_state == NVME_QPAIR_DESTROYING) {
+		rc = -ENXIO;
+		goto out;
+	}
+
+	if (qpair_state != NVME_QPAIR_DISCONNECTED) {
+		rc = 0;
+		goto out;
+	}
+
+	rc = nvme_transport_ctrlr_connect_qpair(ctrlr, qpair);
+	if (rc) {
+		rc = -EAGAIN;
+		goto out;
+	}
+
+out:
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+	return rc;
+}
+
+spdk_nvme_qp_failure_reason
+spdk_nvme_ctrlr_get_admin_qp_failure_reason(struct spdk_nvme_ctrlr *ctrlr)
+{
+	return ctrlr->adminq->transport_failure_reason;
+}
+
+/*
+ * This internal function will attempt to take the controller
+ * lock before calling disconnect on a controller qpair.
+ * Functions already holding the controller lock should
+ * call nvme_transport_ctrlr_disconnect_qpair directly.
+ */
+void
+nvme_ctrlr_disconnect_qpair(struct spdk_nvme_qpair *qpair)
+{
+	struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
+
+	assert(ctrlr != NULL);
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+	nvme_transport_ctrlr_disconnect_qpair(ctrlr, qpair);
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+}
+
+int
+spdk_nvme_ctrlr_free_io_qpair(struct spdk_nvme_qpair *qpair)
+{
+	struct spdk_nvme_ctrlr *ctrlr;
+
+	if (qpair == NULL) {
+		return 0;
+	}
+
+	ctrlr = qpair->ctrlr;
+
+	if (qpair->in_completion_context) {
+		/*
+		 * There are many cases where it is convenient to delete an io qpair in the context
+		 *  of that qpair's completion routine.  To handle this properly, set a flag here
+		 *  so that the completion routine will perform an actual delete after the context
+		 *  unwinds.
+		 */
+		qpair->delete_after_completion_context = 1;
+		return 0;
+	}
+
+	if (qpair->poll_group && qpair->poll_group->in_completion_context) {
+		/* Same as above, but in a poll group. */
+		qpair->poll_group->num_qpairs_to_delete++;
+		qpair->delete_after_completion_context = 1;
+		return 0;
+	}
+
+	if (qpair->poll_group) {
+		spdk_nvme_poll_group_remove(qpair->poll_group->group, qpair);
+	}
+
+	/* Do not retry. */
+	nvme_qpair_set_state(qpair, NVME_QPAIR_DESTROYING);
+
+	/* In the multi-process case, a process may call this function on a foreign
+	 * I/O qpair (i.e. one that this process did not create) when that qpairs process
+	 * exits unexpectedly.  In that case, we must not try to abort any reqs associated
+	 * with that qpair, since the callbacks will also be foreign to this process.
+	 */
+	if (qpair->active_proc == nvme_ctrlr_get_current_process(ctrlr)) {
+		nvme_qpair_abort_reqs(qpair, 1);
+	}
+
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+
+	nvme_ctrlr_proc_remove_io_qpair(qpair);
+
+	TAILQ_REMOVE(&ctrlr->active_io_qpairs, qpair, tailq);
+	spdk_bit_array_set(ctrlr->free_io_qids, qpair->id);
+
+	if (nvme_transport_ctrlr_delete_io_qpair(ctrlr, qpair)) {
+		nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+		return -1;
+	}
+
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+	return 0;
+}
+
+static void
+nvme_ctrlr_construct_intel_support_log_page_list(struct spdk_nvme_ctrlr *ctrlr,
+		struct spdk_nvme_intel_log_page_directory *log_page_directory)
+{
+	if (log_page_directory == NULL) {
+		return;
+	}
+
+	if (ctrlr->cdata.vid != SPDK_PCI_VID_INTEL) {
+		return;
+	}
+
+	ctrlr->log_page_supported[SPDK_NVME_INTEL_LOG_PAGE_DIRECTORY] = true;
+
+	if (log_page_directory->read_latency_log_len ||
+	    (ctrlr->quirks & NVME_INTEL_QUIRK_READ_LATENCY)) {
+		ctrlr->log_page_supported[SPDK_NVME_INTEL_LOG_READ_CMD_LATENCY] = true;
+	}
+	if (log_page_directory->write_latency_log_len ||
+	    (ctrlr->quirks & NVME_INTEL_QUIRK_WRITE_LATENCY)) {
+		ctrlr->log_page_supported[SPDK_NVME_INTEL_LOG_WRITE_CMD_LATENCY] = true;
+	}
+	if (log_page_directory->temperature_statistics_log_len) {
+		ctrlr->log_page_supported[SPDK_NVME_INTEL_LOG_TEMPERATURE] = true;
+	}
+	if (log_page_directory->smart_log_len) {
+		ctrlr->log_page_supported[SPDK_NVME_INTEL_LOG_SMART] = true;
+	}
+	if (log_page_directory->marketing_description_log_len) {
+		ctrlr->log_page_supported[SPDK_NVME_INTEL_MARKETING_DESCRIPTION] = true;
+	}
+}
+
+static int nvme_ctrlr_set_intel_support_log_pages(struct spdk_nvme_ctrlr *ctrlr)
+{
+	int rc = 0;
+	struct nvme_completion_poll_status	*status;
+	struct spdk_nvme_intel_log_page_directory *log_page_directory;
+
+	log_page_directory = spdk_zmalloc(sizeof(struct spdk_nvme_intel_log_page_directory),
+					  64, NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA);
+	if (log_page_directory == NULL) {
+		SPDK_ERRLOG("could not allocate log_page_directory\n");
+		return -ENXIO;
+	}
+
+	status = calloc(1, sizeof(*status));
+	if (!status) {
+		SPDK_ERRLOG("Failed to allocate status tracker\n");
+		spdk_free(log_page_directory);
+		return -ENOMEM;
+	}
+
+	rc = spdk_nvme_ctrlr_cmd_get_log_page(ctrlr, SPDK_NVME_INTEL_LOG_PAGE_DIRECTORY,
+					      SPDK_NVME_GLOBAL_NS_TAG, log_page_directory,
+					      sizeof(struct spdk_nvme_intel_log_page_directory),
+					      0, nvme_completion_poll_cb, status);
+	if (rc != 0) {
+		spdk_free(log_page_directory);
+		free(status);
+		return rc;
+	}
+
+	if (nvme_wait_for_completion_timeout(ctrlr->adminq, status,
+					     ctrlr->opts.admin_timeout_ms / 1000)) {
+		spdk_free(log_page_directory);
+		SPDK_WARNLOG("Intel log pages not supported on Intel drive!\n");
+		if (!status->timed_out) {
+			free(status);
+		}
+		return 0;
+	}
+
+	nvme_ctrlr_construct_intel_support_log_page_list(ctrlr, log_page_directory);
+	spdk_free(log_page_directory);
+	free(status);
+	return 0;
+}
+
+static int
+nvme_ctrlr_set_supported_log_pages(struct spdk_nvme_ctrlr *ctrlr)
+{
+	int	rc = 0;
+
+	memset(ctrlr->log_page_supported, 0, sizeof(ctrlr->log_page_supported));
+	/* Mandatory pages */
+	ctrlr->log_page_supported[SPDK_NVME_LOG_ERROR] = true;
+	ctrlr->log_page_supported[SPDK_NVME_LOG_HEALTH_INFORMATION] = true;
+	ctrlr->log_page_supported[SPDK_NVME_LOG_FIRMWARE_SLOT] = true;
+	if (ctrlr->cdata.lpa.celp) {
+		ctrlr->log_page_supported[SPDK_NVME_LOG_COMMAND_EFFECTS_LOG] = true;
+	}
+	if (ctrlr->cdata.vid == SPDK_PCI_VID_INTEL && !(ctrlr->quirks & NVME_INTEL_QUIRK_NO_LOG_PAGES)) {
+		rc = nvme_ctrlr_set_intel_support_log_pages(ctrlr);
+	}
+
+	return rc;
+}
+
+static void
+nvme_ctrlr_set_intel_supported_features(struct spdk_nvme_ctrlr *ctrlr)
+{
+	ctrlr->feature_supported[SPDK_NVME_INTEL_FEAT_MAX_LBA] = true;
+	ctrlr->feature_supported[SPDK_NVME_INTEL_FEAT_NATIVE_MAX_LBA] = true;
+	ctrlr->feature_supported[SPDK_NVME_INTEL_FEAT_POWER_GOVERNOR_SETTING] = true;
+	ctrlr->feature_supported[SPDK_NVME_INTEL_FEAT_SMBUS_ADDRESS] = true;
+	ctrlr->feature_supported[SPDK_NVME_INTEL_FEAT_LED_PATTERN] = true;
+	ctrlr->feature_supported[SPDK_NVME_INTEL_FEAT_RESET_TIMED_WORKLOAD_COUNTERS] = true;
+	ctrlr->feature_supported[SPDK_NVME_INTEL_FEAT_LATENCY_TRACKING] = true;
+}
+
+static void
+nvme_ctrlr_set_arbitration_feature(struct spdk_nvme_ctrlr *ctrlr)
+{
+	uint32_t cdw11;
+	struct nvme_completion_poll_status *status;
+
+	if (ctrlr->opts.arbitration_burst == 0) {
+		return;
+	}
+
+	if (ctrlr->opts.arbitration_burst > 7) {
+		SPDK_WARNLOG("Valid arbitration burst values is from 0-7\n");
+		return;
+	}
+
+	status = calloc(1, sizeof(*status));
+	if (!status) {
+		SPDK_ERRLOG("Failed to allocate status tracker\n");
+		return;
+	}
+
+	cdw11 = ctrlr->opts.arbitration_burst;
+
+	if (spdk_nvme_ctrlr_get_flags(ctrlr) & SPDK_NVME_CTRLR_WRR_SUPPORTED) {
+		cdw11 |= (uint32_t)ctrlr->opts.low_priority_weight << 8;
+		cdw11 |= (uint32_t)ctrlr->opts.medium_priority_weight << 16;
+		cdw11 |= (uint32_t)ctrlr->opts.high_priority_weight << 24;
+	}
+
+	if (spdk_nvme_ctrlr_cmd_set_feature(ctrlr, SPDK_NVME_FEAT_ARBITRATION,
+					    cdw11, 0, NULL, 0,
+					    nvme_completion_poll_cb, status) < 0) {
+		SPDK_ERRLOG("Set arbitration feature failed\n");
+		free(status);
+		return;
+	}
+
+	if (nvme_wait_for_completion_timeout(ctrlr->adminq, status,
+					     ctrlr->opts.admin_timeout_ms / 1000)) {
+		SPDK_ERRLOG("Timeout to set arbitration feature\n");
+	}
+
+	if (!status->timed_out) {
+		free(status);
+	}
+}
+
+static void
+nvme_ctrlr_set_supported_features(struct spdk_nvme_ctrlr *ctrlr)
+{
+	memset(ctrlr->feature_supported, 0, sizeof(ctrlr->feature_supported));
+	/* Mandatory features */
+	ctrlr->feature_supported[SPDK_NVME_FEAT_ARBITRATION] = true;
+	ctrlr->feature_supported[SPDK_NVME_FEAT_POWER_MANAGEMENT] = true;
+	ctrlr->feature_supported[SPDK_NVME_FEAT_TEMPERATURE_THRESHOLD] = true;
+	ctrlr->feature_supported[SPDK_NVME_FEAT_ERROR_RECOVERY] = true;
+	ctrlr->feature_supported[SPDK_NVME_FEAT_NUMBER_OF_QUEUES] = true;
+	ctrlr->feature_supported[SPDK_NVME_FEAT_INTERRUPT_COALESCING] = true;
+	ctrlr->feature_supported[SPDK_NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION] = true;
+	ctrlr->feature_supported[SPDK_NVME_FEAT_WRITE_ATOMICITY] = true;
+	ctrlr->feature_supported[SPDK_NVME_FEAT_ASYNC_EVENT_CONFIGURATION] = true;
+	/* Optional features */
+	if (ctrlr->cdata.vwc.present) {
+		ctrlr->feature_supported[SPDK_NVME_FEAT_VOLATILE_WRITE_CACHE] = true;
+	}
+	if (ctrlr->cdata.apsta.supported) {
+		ctrlr->feature_supported[SPDK_NVME_FEAT_AUTONOMOUS_POWER_STATE_TRANSITION] = true;
+	}
+	if (ctrlr->cdata.hmpre) {
+		ctrlr->feature_supported[SPDK_NVME_FEAT_HOST_MEM_BUFFER] = true;
+	}
+	if (ctrlr->cdata.vid == SPDK_PCI_VID_INTEL) {
+		nvme_ctrlr_set_intel_supported_features(ctrlr);
+	}
+
+	nvme_ctrlr_set_arbitration_feature(ctrlr);
+}
+
+bool
+spdk_nvme_ctrlr_is_failed(struct spdk_nvme_ctrlr *ctrlr)
+{
+	return ctrlr->is_failed;
+}
+
+void
+nvme_ctrlr_fail(struct spdk_nvme_ctrlr *ctrlr, bool hot_remove)
+{
+	/*
+	 * Set the flag here and leave the work failure of qpairs to
+	 * spdk_nvme_qpair_process_completions().
+	 */
+	if (hot_remove) {
+		ctrlr->is_removed = true;
+	}
+	ctrlr->is_failed = true;
+	nvme_transport_ctrlr_disconnect_qpair(ctrlr, ctrlr->adminq);
+	SPDK_ERRLOG("ctrlr %s in failed state.\n", ctrlr->trid.traddr);
+}
+
+/**
+ * This public API function will try to take the controller lock.
+ * Any private functions being called from a thread already holding
+ * the ctrlr lock should call nvme_ctrlr_fail directly.
+ */
+void
+spdk_nvme_ctrlr_fail(struct spdk_nvme_ctrlr *ctrlr)
+{
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+	nvme_ctrlr_fail(ctrlr, false);
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+}
+
+static void
+nvme_ctrlr_shutdown(struct spdk_nvme_ctrlr *ctrlr)
+{
+	union spdk_nvme_cc_register	cc;
+	union spdk_nvme_csts_register	csts;
+	uint32_t			ms_waited = 0;
+	uint32_t			shutdown_timeout_ms;
+
+	if (ctrlr->is_removed) {
+		return;
+	}
+
+	if (nvme_ctrlr_get_cc(ctrlr, &cc)) {
+		SPDK_ERRLOG("ctrlr %s get_cc() failed\n", ctrlr->trid.traddr);
+		return;
+	}
+
+	cc.bits.shn = SPDK_NVME_SHN_NORMAL;
+
+	if (nvme_ctrlr_set_cc(ctrlr, &cc)) {
+		SPDK_ERRLOG("ctrlr %s set_cc() failed\n", ctrlr->trid.traddr);
+		return;
+	}
+
+	/*
+	 * The NVMe specification defines RTD3E to be the time between
+	 *  setting SHN = 1 until the controller will set SHST = 10b.
+	 * If the device doesn't report RTD3 entry latency, or if it
+	 *  reports RTD3 entry latency less than 10 seconds, pick
+	 *  10 seconds as a reasonable amount of time to
+	 *  wait before proceeding.
+	 */
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "RTD3E = %" PRIu32 " us\n", ctrlr->cdata.rtd3e);
+	shutdown_timeout_ms = (ctrlr->cdata.rtd3e + 999) / 1000;
+	shutdown_timeout_ms = spdk_max(shutdown_timeout_ms, 10000);
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "shutdown timeout = %" PRIu32 " ms\n", shutdown_timeout_ms);
+
+	do {
+		if (nvme_ctrlr_get_csts(ctrlr, &csts)) {
+			SPDK_ERRLOG("ctrlr %s get_csts() failed\n", ctrlr->trid.traddr);
+			return;
+		}
+
+		if (csts.bits.shst == SPDK_NVME_SHST_COMPLETE) {
+			SPDK_DEBUGLOG(SPDK_LOG_NVME, "ctrlr %s shutdown complete in %u milliseconds\n",
+				      ctrlr->trid.traddr, ms_waited);
+			return;
+		}
+
+		nvme_delay(1000);
+		ms_waited++;
+	} while (ms_waited < shutdown_timeout_ms);
+
+	SPDK_ERRLOG("ctrlr %s did not shutdown within %u milliseconds\n",
+		    ctrlr->trid.traddr, shutdown_timeout_ms);
+	if (ctrlr->quirks & NVME_QUIRK_SHST_COMPLETE) {
+		SPDK_ERRLOG("likely due to shutdown handling in the VMWare emulated NVMe SSD\n");
+	}
+}
+
+static int
+nvme_ctrlr_enable(struct spdk_nvme_ctrlr *ctrlr)
+{
+	union spdk_nvme_cc_register	cc;
+	int				rc;
+
+	rc = nvme_transport_ctrlr_enable(ctrlr);
+	if (rc != 0) {
+		SPDK_ERRLOG("transport ctrlr_enable failed\n");
+		return rc;
+	}
+
+	if (nvme_ctrlr_get_cc(ctrlr, &cc)) {
+		SPDK_ERRLOG("get_cc() failed\n");
+		return -EIO;
+	}
+
+	if (cc.bits.en != 0) {
+		SPDK_ERRLOG("called with CC.EN = 1\n");
+		return -EINVAL;
+	}
+
+	cc.bits.en = 1;
+	cc.bits.css = 0;
+	cc.bits.shn = 0;
+	cc.bits.iosqes = 6; /* SQ entry size == 64 == 2^6 */
+	cc.bits.iocqes = 4; /* CQ entry size == 16 == 2^4 */
+
+	/* Page size is 2 ^ (12 + mps). */
+	cc.bits.mps = spdk_u32log2(ctrlr->page_size) - 12;
+
+	if (ctrlr->cap.bits.css == 0) {
+		SPDK_INFOLOG(SPDK_LOG_NVME,
+			     "Drive reports no command sets supported. Assuming NVM is supported.\n");
+		ctrlr->cap.bits.css = SPDK_NVME_CAP_CSS_NVM;
+	}
+
+	if (!(ctrlr->cap.bits.css & (1u << ctrlr->opts.command_set))) {
+		SPDK_DEBUGLOG(SPDK_LOG_NVME, "Requested I/O command set %u but supported mask is 0x%x\n",
+			      ctrlr->opts.command_set, ctrlr->cap.bits.css);
+		SPDK_DEBUGLOG(SPDK_LOG_NVME, "Falling back to NVM. Assuming NVM is supported.\n");
+		ctrlr->opts.command_set = SPDK_NVME_CC_CSS_NVM;
+	}
+
+	cc.bits.css = ctrlr->opts.command_set;
+
+	switch (ctrlr->opts.arb_mechanism) {
+	case SPDK_NVME_CC_AMS_RR:
+		break;
+	case SPDK_NVME_CC_AMS_WRR:
+		if (SPDK_NVME_CAP_AMS_WRR & ctrlr->cap.bits.ams) {
+			break;
+		}
+		return -EINVAL;
+	case SPDK_NVME_CC_AMS_VS:
+		if (SPDK_NVME_CAP_AMS_VS & ctrlr->cap.bits.ams) {
+			break;
+		}
+		return -EINVAL;
+	default:
+		return -EINVAL;
+	}
+
+	cc.bits.ams = ctrlr->opts.arb_mechanism;
+
+	if (nvme_ctrlr_set_cc(ctrlr, &cc)) {
+		SPDK_ERRLOG("set_cc() failed\n");
+		return -EIO;
+	}
+
+	return 0;
+}
+
+static int
+nvme_ctrlr_disable(struct spdk_nvme_ctrlr *ctrlr)
+{
+	union spdk_nvme_cc_register	cc;
+
+	if (nvme_ctrlr_get_cc(ctrlr, &cc)) {
+		SPDK_ERRLOG("get_cc() failed\n");
+		return -EIO;
+	}
+
+	if (cc.bits.en == 0) {
+		return 0;
+	}
+
+	cc.bits.en = 0;
+
+	if (nvme_ctrlr_set_cc(ctrlr, &cc)) {
+		SPDK_ERRLOG("set_cc() failed\n");
+		return -EIO;
+	}
+
+	return 0;
+}
+
+#ifdef DEBUG
+static const char *
+nvme_ctrlr_state_string(enum nvme_ctrlr_state state)
+{
+	switch (state) {
+	case NVME_CTRLR_STATE_INIT_DELAY:
+		return "delay init";
+	case NVME_CTRLR_STATE_INIT:
+		return "init";
+	case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1:
+		return "disable and wait for CSTS.RDY = 1";
+	case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0:
+		return "disable and wait for CSTS.RDY = 0";
+	case NVME_CTRLR_STATE_ENABLE:
+		return "enable controller by writing CC.EN = 1";
+	case NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1:
+		return "wait for CSTS.RDY = 1";
+	case NVME_CTRLR_STATE_RESET_ADMIN_QUEUE:
+		return "reset admin queue";
+	case NVME_CTRLR_STATE_IDENTIFY:
+		return "identify controller";
+	case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY:
+		return "wait for identify controller";
+	case NVME_CTRLR_STATE_SET_NUM_QUEUES:
+		return "set number of queues";
+	case NVME_CTRLR_STATE_WAIT_FOR_SET_NUM_QUEUES:
+		return "wait for set number of queues";
+	case NVME_CTRLR_STATE_CONSTRUCT_NS:
+		return "construct namespaces";
+	case NVME_CTRLR_STATE_IDENTIFY_ACTIVE_NS:
+		return "identify active ns";
+	case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ACTIVE_NS:
+		return "wait for identify active ns";
+	case NVME_CTRLR_STATE_IDENTIFY_NS:
+		return "identify ns";
+	case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS:
+		return "wait for identify ns";
+	case NVME_CTRLR_STATE_IDENTIFY_ID_DESCS:
+		return "identify namespace id descriptors";
+	case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ID_DESCS:
+		return "wait for identify namespace id descriptors";
+	case NVME_CTRLR_STATE_CONFIGURE_AER:
+		return "configure AER";
+	case NVME_CTRLR_STATE_WAIT_FOR_CONFIGURE_AER:
+		return "wait for configure aer";
+	case NVME_CTRLR_STATE_SET_SUPPORTED_LOG_PAGES:
+		return "set supported log pages";
+	case NVME_CTRLR_STATE_SET_SUPPORTED_FEATURES:
+		return "set supported features";
+	case NVME_CTRLR_STATE_SET_DB_BUF_CFG:
+		return "set doorbell buffer config";
+	case NVME_CTRLR_STATE_WAIT_FOR_DB_BUF_CFG:
+		return "wait for doorbell buffer config";
+	case NVME_CTRLR_STATE_SET_KEEP_ALIVE_TIMEOUT:
+		return "set keep alive timeout";
+	case NVME_CTRLR_STATE_WAIT_FOR_KEEP_ALIVE_TIMEOUT:
+		return "wait for set keep alive timeout";
+	case NVME_CTRLR_STATE_SET_HOST_ID:
+		return "set host ID";
+	case NVME_CTRLR_STATE_WAIT_FOR_HOST_ID:
+		return "wait for set host ID";
+	case NVME_CTRLR_STATE_READY:
+		return "ready";
+	case NVME_CTRLR_STATE_ERROR:
+		return "error";
+	}
+	return "unknown";
+};
+#endif /* DEBUG */
+
+static void
+nvme_ctrlr_set_state(struct spdk_nvme_ctrlr *ctrlr, enum nvme_ctrlr_state state,
+		     uint64_t timeout_in_ms)
+{
+	uint64_t ticks_per_ms, timeout_in_ticks, now_ticks;
+
+	ctrlr->state = state;
+	if (timeout_in_ms == NVME_TIMEOUT_INFINITE) {
+		goto inf;
+	}
+
+	ticks_per_ms = spdk_get_ticks_hz() / 1000;
+	if (timeout_in_ms > UINT64_MAX / ticks_per_ms) {
+		SPDK_ERRLOG("Specified timeout would cause integer overflow. Defaulting to no timeout.\n");
+		goto inf;
+	}
+
+	now_ticks = spdk_get_ticks();
+	timeout_in_ticks = timeout_in_ms * ticks_per_ms;
+	if (timeout_in_ticks > UINT64_MAX - now_ticks) {
+		SPDK_ERRLOG("Specified timeout would cause integer overflow. Defaulting to no timeout.\n");
+		goto inf;
+	}
+
+	ctrlr->state_timeout_tsc = timeout_in_ticks + now_ticks;
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "setting state to %s (timeout %" PRIu64 " ms)\n",
+		      nvme_ctrlr_state_string(ctrlr->state), timeout_in_ms);
+	return;
+inf:
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "setting state to %s (no timeout)\n",
+		      nvme_ctrlr_state_string(ctrlr->state));
+	ctrlr->state_timeout_tsc = NVME_TIMEOUT_INFINITE;
+}
+
+static void
+nvme_ctrlr_free_doorbell_buffer(struct spdk_nvme_ctrlr *ctrlr)
+{
+	if (ctrlr->shadow_doorbell) {
+		spdk_free(ctrlr->shadow_doorbell);
+		ctrlr->shadow_doorbell = NULL;
+	}
+
+	if (ctrlr->eventidx) {
+		spdk_free(ctrlr->eventidx);
+		ctrlr->eventidx = NULL;
+	}
+}
+
+static void
+nvme_ctrlr_set_doorbell_buffer_config_done(void *arg, const struct spdk_nvme_cpl *cpl)
+{
+	struct spdk_nvme_ctrlr *ctrlr = (struct spdk_nvme_ctrlr *)arg;
+
+	if (spdk_nvme_cpl_is_error(cpl)) {
+		SPDK_WARNLOG("Doorbell buffer config failed\n");
+	} else {
+		SPDK_INFOLOG(SPDK_LOG_NVME, "NVMe controller: %s doorbell buffer config enabled\n",
+			     ctrlr->trid.traddr);
+	}
+	nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_KEEP_ALIVE_TIMEOUT,
+			     ctrlr->opts.admin_timeout_ms);
+}
+
+static int
+nvme_ctrlr_set_doorbell_buffer_config(struct spdk_nvme_ctrlr *ctrlr)
+{
+	int rc = 0;
+	uint64_t prp1, prp2, len;
+
+	if (!ctrlr->cdata.oacs.doorbell_buffer_config) {
+		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_KEEP_ALIVE_TIMEOUT,
+				     ctrlr->opts.admin_timeout_ms);
+		return 0;
+	}
+
+	if (ctrlr->trid.trtype != SPDK_NVME_TRANSPORT_PCIE) {
+		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_KEEP_ALIVE_TIMEOUT,
+				     ctrlr->opts.admin_timeout_ms);
+		return 0;
+	}
+
+	/* only 1 page size for doorbell buffer */
+	ctrlr->shadow_doorbell = spdk_zmalloc(ctrlr->page_size, ctrlr->page_size,
+					      NULL, SPDK_ENV_LCORE_ID_ANY,
+					      SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE);
+	if (ctrlr->shadow_doorbell == NULL) {
+		rc = -ENOMEM;
+		goto error;
+	}
+
+	len = ctrlr->page_size;
+	prp1 = spdk_vtophys(ctrlr->shadow_doorbell, &len);
+	if (prp1 == SPDK_VTOPHYS_ERROR || len != ctrlr->page_size) {
+		rc = -EFAULT;
+		goto error;
+	}
+
+	ctrlr->eventidx = spdk_zmalloc(ctrlr->page_size, ctrlr->page_size,
+				       NULL, SPDK_ENV_LCORE_ID_ANY,
+				       SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE);
+	if (ctrlr->eventidx == NULL) {
+		rc = -ENOMEM;
+		goto error;
+	}
+
+	len = ctrlr->page_size;
+	prp2 = spdk_vtophys(ctrlr->eventidx, &len);
+	if (prp2 == SPDK_VTOPHYS_ERROR || len != ctrlr->page_size) {
+		rc = -EFAULT;
+		goto error;
+	}
+
+	nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_DB_BUF_CFG,
+			     ctrlr->opts.admin_timeout_ms);
+
+	rc = nvme_ctrlr_cmd_doorbell_buffer_config(ctrlr, prp1, prp2,
+			nvme_ctrlr_set_doorbell_buffer_config_done, ctrlr);
+	if (rc != 0) {
+		goto error;
+	}
+
+	return 0;
+
+error:
+	nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
+	nvme_ctrlr_free_doorbell_buffer(ctrlr);
+	return rc;
+}
+
+static void
+nvme_ctrlr_abort_queued_aborts(struct spdk_nvme_ctrlr *ctrlr)
+{
+	struct nvme_request	*req, *tmp;
+	struct spdk_nvme_cpl	cpl = {};
+
+	cpl.status.sc = SPDK_NVME_SC_ABORTED_SQ_DELETION;
+	cpl.status.sct = SPDK_NVME_SCT_GENERIC;
+
+	STAILQ_FOREACH_SAFE(req, &ctrlr->queued_aborts, stailq, tmp) {
+		STAILQ_REMOVE_HEAD(&ctrlr->queued_aborts, stailq);
+
+		nvme_complete_request(req->cb_fn, req->cb_arg, req->qpair, req, &cpl);
+		nvme_free_request(req);
+	}
+}
+
+int
+spdk_nvme_ctrlr_reset(struct spdk_nvme_ctrlr *ctrlr)
+{
+	int rc = 0;
+	struct spdk_nvme_qpair	*qpair;
+
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+
+	if (ctrlr->is_resetting || ctrlr->is_removed) {
+		/*
+		 * Controller is already resetting or has been removed. Return
+		 *  immediately since there is no need to kick off another
+		 *  reset in these cases.
+		 */
+		nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+		return ctrlr->is_resetting ? 0 : -ENXIO;
+	}
+
+	ctrlr->is_resetting = true;
+	ctrlr->is_failed = false;
+
+	SPDK_NOTICELOG("resetting controller\n");
+
+	/* Abort all of the queued abort requests */
+	nvme_ctrlr_abort_queued_aborts(ctrlr);
+
+	nvme_transport_admin_qpair_abort_aers(ctrlr->adminq);
+
+	/* Disable all queues before disabling the controller hardware. */
+	TAILQ_FOREACH(qpair, &ctrlr->active_io_qpairs, tailq) {
+		qpair->transport_failure_reason = SPDK_NVME_QPAIR_FAILURE_LOCAL;
+	}
+
+	ctrlr->adminq->transport_failure_reason = SPDK_NVME_QPAIR_FAILURE_LOCAL;
+	nvme_transport_ctrlr_disconnect_qpair(ctrlr, ctrlr->adminq);
+	if (nvme_transport_ctrlr_connect_qpair(ctrlr, ctrlr->adminq) != 0) {
+		SPDK_ERRLOG("Controller reinitialization failed.\n");
+		rc = -1;
+		goto out;
+	}
+
+	/* Doorbell buffer config is invalid during reset */
+	nvme_ctrlr_free_doorbell_buffer(ctrlr);
+
+	/* Set the state back to INIT to cause a full hardware reset. */
+	nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_INIT, NVME_TIMEOUT_INFINITE);
+
+	nvme_qpair_set_state(ctrlr->adminq, NVME_QPAIR_ENABLED);
+	while (ctrlr->state != NVME_CTRLR_STATE_READY) {
+		if (nvme_ctrlr_process_init(ctrlr) != 0) {
+			SPDK_ERRLOG("controller reinitialization failed\n");
+			rc = -1;
+			break;
+		}
+	}
+
+	/*
+	 * For PCIe controllers, the memory locations of the tranpsort qpair
+	 * don't change when the controller is reset. They simply need to be
+	 * re-enabled with admin commands to the controller. For fabric
+	 * controllers we need to disconnect and reconnect the qpair on its
+	 * own thread outside of the context of the reset.
+	 */
+	if (rc == 0 && ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) {
+		/* Reinitialize qpairs */
+		TAILQ_FOREACH(qpair, &ctrlr->active_io_qpairs, tailq) {
+			if (nvme_transport_ctrlr_connect_qpair(ctrlr, qpair) != 0) {
+				qpair->transport_failure_reason = SPDK_NVME_QPAIR_FAILURE_LOCAL;
+				rc = -1;
+				continue;
+			}
+		}
+	}
+
+out:
+	if (rc) {
+		nvme_ctrlr_fail(ctrlr, false);
+	}
+	ctrlr->is_resetting = false;
+
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+
+	if (!ctrlr->cdata.oaes.ns_attribute_notices) {
+		/*
+		 * If controller doesn't support ns_attribute_notices and
+		 * namespace attributes change (e.g. number of namespaces)
+		 * we need to update system handling device reset.
+		 */
+		nvme_io_msg_ctrlr_update(ctrlr);
+	}
+
+	return rc;
+}
+
+int
+spdk_nvme_ctrlr_set_trid(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_transport_id *trid)
+{
+	int rc = 0;
+
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+
+	if (ctrlr->is_failed == false) {
+		rc = -EPERM;
+		goto out;
+	}
+
+	if (trid->trtype != ctrlr->trid.trtype) {
+		rc = -EINVAL;
+		goto out;
+	}
+
+	if (strncmp(trid->subnqn, ctrlr->trid.subnqn, SPDK_NVMF_NQN_MAX_LEN)) {
+		rc = -EINVAL;
+		goto out;
+	}
+
+	ctrlr->trid = *trid;
+
+out:
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+	return rc;
+}
+
+static void
+nvme_ctrlr_identify_done(void *arg, const struct spdk_nvme_cpl *cpl)
+{
+	struct spdk_nvme_ctrlr *ctrlr = (struct spdk_nvme_ctrlr *)arg;
+
+	if (spdk_nvme_cpl_is_error(cpl)) {
+		SPDK_ERRLOG("nvme_identify_controller failed!\n");
+		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
+		return;
+	}
+
+	/*
+	 * Use MDTS to ensure our default max_xfer_size doesn't exceed what the
+	 *  controller supports.
+	 */
+	ctrlr->max_xfer_size = nvme_transport_ctrlr_get_max_xfer_size(ctrlr);
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "transport max_xfer_size %u\n", ctrlr->max_xfer_size);
+	if (ctrlr->cdata.mdts > 0) {
+		ctrlr->max_xfer_size = spdk_min(ctrlr->max_xfer_size,
+						ctrlr->min_page_size * (1 << (ctrlr->cdata.mdts)));
+		SPDK_DEBUGLOG(SPDK_LOG_NVME, "MDTS max_xfer_size %u\n", ctrlr->max_xfer_size);
+	}
+
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "CNTLID 0x%04" PRIx16 "\n", ctrlr->cdata.cntlid);
+	if (ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) {
+		ctrlr->cntlid = ctrlr->cdata.cntlid;
+	} else {
+		/*
+		 * Fabrics controllers should already have CNTLID from the Connect command.
+		 *
+		 * If CNTLID from Connect doesn't match CNTLID in the Identify Controller data,
+		 * trust the one from Connect.
+		 */
+		if (ctrlr->cntlid != ctrlr->cdata.cntlid) {
+			SPDK_DEBUGLOG(SPDK_LOG_NVME,
+				      "Identify CNTLID 0x%04" PRIx16 " != Connect CNTLID 0x%04" PRIx16 "\n",
+				      ctrlr->cdata.cntlid, ctrlr->cntlid);
+		}
+	}
+
+	if (ctrlr->cdata.sgls.supported) {
+		assert(ctrlr->cdata.sgls.supported != 0x3);
+		ctrlr->flags |= SPDK_NVME_CTRLR_SGL_SUPPORTED;
+		if (ctrlr->cdata.sgls.supported == 0x2) {
+			ctrlr->flags |= SPDK_NVME_CTRLR_SGL_REQUIRES_DWORD_ALIGNMENT;
+		}
+		/*
+		 * Use MSDBD to ensure our max_sges doesn't exceed what the
+		 *  controller supports.
+		 */
+		ctrlr->max_sges = nvme_transport_ctrlr_get_max_sges(ctrlr);
+		if (ctrlr->cdata.nvmf_specific.msdbd != 0) {
+			ctrlr->max_sges = spdk_min(ctrlr->cdata.nvmf_specific.msdbd, ctrlr->max_sges);
+		} else {
+			/* A value 0 indicates no limit. */
+		}
+		SPDK_DEBUGLOG(SPDK_LOG_NVME, "transport max_sges %u\n", ctrlr->max_sges);
+	}
+
+	if (ctrlr->cdata.oacs.security && !(ctrlr->quirks & NVME_QUIRK_OACS_SECURITY)) {
+		ctrlr->flags |= SPDK_NVME_CTRLR_SECURITY_SEND_RECV_SUPPORTED;
+	}
+
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "fuses compare and write: %d\n", ctrlr->cdata.fuses.compare_and_write);
+	if (ctrlr->cdata.fuses.compare_and_write) {
+		ctrlr->flags |= SPDK_NVME_CTRLR_COMPARE_AND_WRITE_SUPPORTED;
+	}
+
+	nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_NUM_QUEUES,
+			     ctrlr->opts.admin_timeout_ms);
+}
+
+static int
+nvme_ctrlr_identify(struct spdk_nvme_ctrlr *ctrlr)
+{
+	int	rc;
+
+	nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY,
+			     ctrlr->opts.admin_timeout_ms);
+
+	rc = nvme_ctrlr_cmd_identify(ctrlr, SPDK_NVME_IDENTIFY_CTRLR, 0, 0,
+				     &ctrlr->cdata, sizeof(ctrlr->cdata),
+				     nvme_ctrlr_identify_done, ctrlr);
+	if (rc != 0) {
+		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
+		return rc;
+	}
+
+	return 0;
+}
+
+enum nvme_active_ns_state {
+	NVME_ACTIVE_NS_STATE_IDLE,
+	NVME_ACTIVE_NS_STATE_PROCESSING,
+	NVME_ACTIVE_NS_STATE_DONE,
+	NVME_ACTIVE_NS_STATE_ERROR
+};
+
+typedef void (*nvme_active_ns_ctx_deleter)(struct nvme_active_ns_ctx *);
+
+struct nvme_active_ns_ctx {
+	struct spdk_nvme_ctrlr *ctrlr;
+	uint32_t page;
+	uint32_t num_pages;
+	uint32_t next_nsid;
+	uint32_t *new_ns_list;
+	nvme_active_ns_ctx_deleter deleter;
+
+	enum nvme_active_ns_state state;
+};
+
+static struct nvme_active_ns_ctx *
+nvme_active_ns_ctx_create(struct spdk_nvme_ctrlr *ctrlr, nvme_active_ns_ctx_deleter deleter)
+{
+	struct nvme_active_ns_ctx *ctx;
+	uint32_t num_pages = 0;
+	uint32_t *new_ns_list = NULL;
+
+	ctx = calloc(1, sizeof(*ctx));
+	if (!ctx) {
+		SPDK_ERRLOG("Failed to allocate nvme_active_ns_ctx!\n");
+		return NULL;
+	}
+
+	if (ctrlr->num_ns) {
+		/* The allocated size must be a multiple of sizeof(struct spdk_nvme_ns_list) */
+		num_pages = (ctrlr->num_ns * sizeof(new_ns_list[0]) - 1) / sizeof(struct spdk_nvme_ns_list) + 1;
+		new_ns_list = spdk_zmalloc(num_pages * sizeof(struct spdk_nvme_ns_list), ctrlr->page_size,
+					   NULL, SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE);
+		if (!new_ns_list) {
+			SPDK_ERRLOG("Failed to allocate active_ns_list!\n");
+			free(ctx);
+			return NULL;
+		}
+	}
+
+	ctx->num_pages = num_pages;
+	ctx->new_ns_list = new_ns_list;
+	ctx->ctrlr = ctrlr;
+	ctx->deleter = deleter;
+
+	return ctx;
+}
+
+static void
+nvme_active_ns_ctx_destroy(struct nvme_active_ns_ctx *ctx)
+{
+	spdk_free(ctx->new_ns_list);
+	free(ctx);
+}
+
+static void
+nvme_ctrlr_identify_active_ns_swap(struct spdk_nvme_ctrlr *ctrlr, uint32_t **new_ns_list)
+{
+	spdk_free(ctrlr->active_ns_list);
+	ctrlr->active_ns_list = *new_ns_list;
+	*new_ns_list = NULL;
+}
+
+static void
+nvme_ctrlr_identify_active_ns_async_done(void *arg, const struct spdk_nvme_cpl *cpl)
+{
+	struct nvme_active_ns_ctx *ctx = arg;
+
+	if (spdk_nvme_cpl_is_error(cpl)) {
+		ctx->state = NVME_ACTIVE_NS_STATE_ERROR;
+		goto out;
+	}
+
+	ctx->next_nsid = ctx->new_ns_list[1024 * ctx->page + 1023];
+	if (ctx->next_nsid == 0 || ++ctx->page == ctx->num_pages) {
+		ctx->state = NVME_ACTIVE_NS_STATE_DONE;
+		goto out;
+	}
+
+	nvme_ctrlr_identify_active_ns_async(ctx);
+	return;
+
+out:
+	if (ctx->deleter) {
+		ctx->deleter(ctx);
+	}
+}
+
+static void
+nvme_ctrlr_identify_active_ns_async(struct nvme_active_ns_ctx *ctx)
+{
+	struct spdk_nvme_ctrlr *ctrlr = ctx->ctrlr;
+	uint32_t i;
+	int rc;
+
+	if (ctrlr->num_ns == 0) {
+		ctx->state = NVME_ACTIVE_NS_STATE_DONE;
+		goto out;
+	}
+
+	/*
+	 * If controller doesn't support active ns list CNS 0x02 dummy up
+	 * an active ns list, i.e. all namespaces report as active
+	 */
+	if (ctrlr->vs.raw < SPDK_NVME_VERSION(1, 1, 0) || ctrlr->quirks & NVME_QUIRK_IDENTIFY_CNS) {
+		for (i = 0; i < ctrlr->num_ns; i++) {
+			ctx->new_ns_list[i] = i + 1;
+		}
+
+		ctx->state = NVME_ACTIVE_NS_STATE_DONE;
+		goto out;
+	}
+
+	ctx->state = NVME_ACTIVE_NS_STATE_PROCESSING;
+	rc = nvme_ctrlr_cmd_identify(ctrlr, SPDK_NVME_IDENTIFY_ACTIVE_NS_LIST, 0, ctx->next_nsid,
+				     &ctx->new_ns_list[1024 * ctx->page], sizeof(struct spdk_nvme_ns_list),
+				     nvme_ctrlr_identify_active_ns_async_done, ctx);
+	if (rc != 0) {
+		ctx->state = NVME_ACTIVE_NS_STATE_ERROR;
+		goto out;
+	}
+
+	return;
+
+out:
+	if (ctx->deleter) {
+		ctx->deleter(ctx);
+	}
+}
+
+static void
+_nvme_active_ns_ctx_deleter(struct nvme_active_ns_ctx *ctx)
+{
+	struct spdk_nvme_ctrlr *ctrlr = ctx->ctrlr;
+
+	if (ctx->state == NVME_ACTIVE_NS_STATE_ERROR) {
+		nvme_ctrlr_destruct_namespaces(ctrlr);
+		nvme_active_ns_ctx_destroy(ctx);
+		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
+		return;
+	}
+
+	assert(ctx->state == NVME_ACTIVE_NS_STATE_DONE);
+	nvme_ctrlr_identify_active_ns_swap(ctrlr, &ctx->new_ns_list);
+	nvme_active_ns_ctx_destroy(ctx);
+	nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_NS, ctrlr->opts.admin_timeout_ms);
+}
+
+static void
+_nvme_ctrlr_identify_active_ns(struct spdk_nvme_ctrlr *ctrlr)
+{
+	struct nvme_active_ns_ctx *ctx;
+
+	ctx = nvme_active_ns_ctx_create(ctrlr, _nvme_active_ns_ctx_deleter);
+	if (!ctx) {
+		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
+		return;
+	}
+
+	nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ACTIVE_NS,
+			     ctrlr->opts.admin_timeout_ms);
+	nvme_ctrlr_identify_active_ns_async(ctx);
+}
+
+int
+nvme_ctrlr_identify_active_ns(struct spdk_nvme_ctrlr *ctrlr)
+{
+	struct nvme_active_ns_ctx *ctx;
+	int rc;
+
+	ctx = nvme_active_ns_ctx_create(ctrlr, NULL);
+	if (!ctx) {
+		return -ENOMEM;
+	}
+
+	nvme_ctrlr_identify_active_ns_async(ctx);
+	while (ctx->state == NVME_ACTIVE_NS_STATE_PROCESSING) {
+		rc = spdk_nvme_qpair_process_completions(ctrlr->adminq, 0);
+		if (rc < 0) {
+			ctx->state = NVME_ACTIVE_NS_STATE_ERROR;
+			break;
+		}
+	}
+
+	if (ctx->state == NVME_ACTIVE_NS_STATE_ERROR) {
+		nvme_active_ns_ctx_destroy(ctx);
+		return -ENXIO;
+	}
+
+	assert(ctx->state == NVME_ACTIVE_NS_STATE_DONE);
+	nvme_ctrlr_identify_active_ns_swap(ctrlr, &ctx->new_ns_list);
+	nvme_active_ns_ctx_destroy(ctx);
+
+	return 0;
+}
+
+static void
+nvme_ctrlr_identify_ns_async_done(void *arg, const struct spdk_nvme_cpl *cpl)
+{
+	struct spdk_nvme_ns *ns = (struct spdk_nvme_ns *)arg;
+	struct spdk_nvme_ctrlr *ctrlr = ns->ctrlr;
+	uint32_t nsid;
+	int rc;
+
+	if (spdk_nvme_cpl_is_error(cpl)) {
+		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
+		return;
+	} else {
+		nvme_ns_set_identify_data(ns);
+	}
+
+	/* move on to the next active NS */
+	nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, ns->id);
+	ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid);
+	if (ns == NULL) {
+		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_ID_DESCS,
+				     ctrlr->opts.admin_timeout_ms);
+		return;
+	}
+	ns->ctrlr = ctrlr;
+	ns->id = nsid;
+
+	rc = nvme_ctrlr_identify_ns_async(ns);
+	if (rc) {
+		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
+	}
+}
+
+static int
+nvme_ctrlr_identify_ns_async(struct spdk_nvme_ns *ns)
+{
+	struct spdk_nvme_ctrlr *ctrlr = ns->ctrlr;
+	struct spdk_nvme_ns_data *nsdata;
+
+	nsdata = &ctrlr->nsdata[ns->id - 1];
+
+	nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS,
+			     ctrlr->opts.admin_timeout_ms);
+	return nvme_ctrlr_cmd_identify(ns->ctrlr, SPDK_NVME_IDENTIFY_NS, 0, ns->id,
+				       nsdata, sizeof(*nsdata),
+				       nvme_ctrlr_identify_ns_async_done, ns);
+}
+
+static int
+nvme_ctrlr_identify_namespaces(struct spdk_nvme_ctrlr *ctrlr)
+{
+	uint32_t nsid;
+	struct spdk_nvme_ns *ns;
+	int rc;
+
+	nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr);
+	ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid);
+	if (ns == NULL) {
+		/* No active NS, move on to the next state */
+		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_CONFIGURE_AER,
+				     ctrlr->opts.admin_timeout_ms);
+		return 0;
+	}
+
+	ns->ctrlr = ctrlr;
+	ns->id = nsid;
+
+	rc = nvme_ctrlr_identify_ns_async(ns);
+	if (rc) {
+		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
+	}
+
+	return rc;
+}
+
+static void
+nvme_ctrlr_identify_id_desc_async_done(void *arg, const struct spdk_nvme_cpl *cpl)
+{
+	struct spdk_nvme_ns *ns = (struct spdk_nvme_ns *)arg;
+	struct spdk_nvme_ctrlr *ctrlr = ns->ctrlr;
+	uint32_t nsid;
+	int rc;
+
+	if (spdk_nvme_cpl_is_error(cpl)) {
+		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_CONFIGURE_AER,
+				     ctrlr->opts.admin_timeout_ms);
+		return;
+	}
+
+	/* move on to the next active NS */
+	nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, ns->id);
+	ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid);
+	if (ns == NULL) {
+		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_CONFIGURE_AER,
+				     ctrlr->opts.admin_timeout_ms);
+		return;
+	}
+
+	rc = nvme_ctrlr_identify_id_desc_async(ns);
+	if (rc) {
+		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
+	}
+}
+
+static int
+nvme_ctrlr_identify_id_desc_async(struct spdk_nvme_ns *ns)
+{
+	struct spdk_nvme_ctrlr *ctrlr = ns->ctrlr;
+
+	memset(ns->id_desc_list, 0, sizeof(ns->id_desc_list));
+
+	nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ID_DESCS,
+			     ctrlr->opts.admin_timeout_ms);
+	return nvme_ctrlr_cmd_identify(ns->ctrlr, SPDK_NVME_IDENTIFY_NS_ID_DESCRIPTOR_LIST,
+				       0, ns->id, ns->id_desc_list, sizeof(ns->id_desc_list),
+				       nvme_ctrlr_identify_id_desc_async_done, ns);
+}
+
+static int
+nvme_ctrlr_identify_id_desc_namespaces(struct spdk_nvme_ctrlr *ctrlr)
+{
+	uint32_t nsid;
+	struct spdk_nvme_ns *ns;
+	int rc;
+
+	if (ctrlr->vs.raw < SPDK_NVME_VERSION(1, 3, 0) ||
+	    (ctrlr->quirks & NVME_QUIRK_IDENTIFY_CNS)) {
+		SPDK_DEBUGLOG(SPDK_LOG_NVME, "Version < 1.3; not attempting to retrieve NS ID Descriptor List\n");
+		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_CONFIGURE_AER,
+				     ctrlr->opts.admin_timeout_ms);
+		return 0;
+	}
+
+	nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr);
+	ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid);
+	if (ns == NULL) {
+		/* No active NS, move on to the next state */
+		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_CONFIGURE_AER,
+				     ctrlr->opts.admin_timeout_ms);
+		return 0;
+	}
+
+	rc = nvme_ctrlr_identify_id_desc_async(ns);
+	if (rc) {
+		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
+	}
+
+	return rc;
+}
+
+static void
+nvme_ctrlr_update_nvmf_ioccsz(struct spdk_nvme_ctrlr *ctrlr)
+{
+	if (ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_RDMA ||
+	    ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_TCP ||
+	    ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_FC) {
+		if (ctrlr->cdata.nvmf_specific.ioccsz < 4) {
+			SPDK_ERRLOG("Incorrect IOCCSZ %u, the minimum value should be 4\n",
+				    ctrlr->cdata.nvmf_specific.ioccsz);
+			ctrlr->cdata.nvmf_specific.ioccsz = 4;
+			assert(0);
+		}
+		ctrlr->ioccsz_bytes = ctrlr->cdata.nvmf_specific.ioccsz * 16 - sizeof(struct spdk_nvme_cmd);
+		ctrlr->icdoff = ctrlr->cdata.nvmf_specific.icdoff;
+	}
+}
+
+static void
+nvme_ctrlr_set_num_queues_done(void *arg, const struct spdk_nvme_cpl *cpl)
+{
+	uint32_t cq_allocated, sq_allocated, min_allocated, i;
+	struct spdk_nvme_ctrlr *ctrlr = (struct spdk_nvme_ctrlr *)arg;
+
+	if (spdk_nvme_cpl_is_error(cpl)) {
+		SPDK_ERRLOG("Set Features - Number of Queues failed!\n");
+		ctrlr->opts.num_io_queues = 0;
+	} else {
+		/*
+		 * Data in cdw0 is 0-based.
+		 * Lower 16-bits indicate number of submission queues allocated.
+		 * Upper 16-bits indicate number of completion queues allocated.
+		 */
+		sq_allocated = (cpl->cdw0 & 0xFFFF) + 1;
+		cq_allocated = (cpl->cdw0 >> 16) + 1;
+
+		/*
+		 * For 1:1 queue mapping, set number of allocated queues to be minimum of
+		 * submission and completion queues.
+		 */
+		min_allocated = spdk_min(sq_allocated, cq_allocated);
+
+		/* Set number of queues to be minimum of requested and actually allocated. */
+		ctrlr->opts.num_io_queues = spdk_min(min_allocated, ctrlr->opts.num_io_queues);
+	}
+
+	ctrlr->free_io_qids = spdk_bit_array_create(ctrlr->opts.num_io_queues + 1);
+	if (ctrlr->free_io_qids == NULL) {
+		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
+		return;
+	}
+
+	/* Initialize list of free I/O queue IDs. QID 0 is the admin queue. */
+	spdk_bit_array_clear(ctrlr->free_io_qids, 0);
+	for (i = 1; i <= ctrlr->opts.num_io_queues; i++) {
+		spdk_bit_array_set(ctrlr->free_io_qids, i);
+	}
+	nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_CONSTRUCT_NS,
+			     ctrlr->opts.admin_timeout_ms);
+}
+
+static int
+nvme_ctrlr_set_num_queues(struct spdk_nvme_ctrlr *ctrlr)
+{
+	int rc;
+
+	if (ctrlr->opts.num_io_queues > SPDK_NVME_MAX_IO_QUEUES) {
+		SPDK_NOTICELOG("Limiting requested num_io_queues %u to max %d\n",
+			       ctrlr->opts.num_io_queues, SPDK_NVME_MAX_IO_QUEUES);
+		ctrlr->opts.num_io_queues = SPDK_NVME_MAX_IO_QUEUES;
+	} else if (ctrlr->opts.num_io_queues < 1) {
+		SPDK_NOTICELOG("Requested num_io_queues 0, increasing to 1\n");
+		ctrlr->opts.num_io_queues = 1;
+	}
+
+	nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_SET_NUM_QUEUES,
+			     ctrlr->opts.admin_timeout_ms);
+
+	rc = nvme_ctrlr_cmd_set_num_queues(ctrlr, ctrlr->opts.num_io_queues,
+					   nvme_ctrlr_set_num_queues_done, ctrlr);
+	if (rc != 0) {
+		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
+		return rc;
+	}
+
+	return 0;
+}
+
+static void
+nvme_ctrlr_set_keep_alive_timeout_done(void *arg, const struct spdk_nvme_cpl *cpl)
+{
+	uint32_t keep_alive_interval_ms;
+	struct spdk_nvme_ctrlr *ctrlr = (struct spdk_nvme_ctrlr *)arg;
+
+	if (spdk_nvme_cpl_is_error(cpl)) {
+		if ((cpl->status.sct == SPDK_NVME_SCT_GENERIC) &&
+		    (cpl->status.sc == SPDK_NVME_SC_INVALID_FIELD)) {
+			SPDK_DEBUGLOG(SPDK_LOG_NVME, "Keep alive timeout Get Feature is not supported\n");
+		} else {
+			SPDK_ERRLOG("Keep alive timeout Get Feature failed: SC %x SCT %x\n",
+				    cpl->status.sc, cpl->status.sct);
+			ctrlr->opts.keep_alive_timeout_ms = 0;
+			nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
+			return;
+		}
+	} else {
+		if (ctrlr->opts.keep_alive_timeout_ms != cpl->cdw0) {
+			SPDK_DEBUGLOG(SPDK_LOG_NVME, "Controller adjusted keep alive timeout to %u ms\n",
+				      cpl->cdw0);
+		}
+
+		ctrlr->opts.keep_alive_timeout_ms = cpl->cdw0;
+	}
+
+	keep_alive_interval_ms = ctrlr->opts.keep_alive_timeout_ms / 2;
+	if (keep_alive_interval_ms == 0) {
+		keep_alive_interval_ms = 1;
+	}
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "Sending keep alive every %u ms\n", keep_alive_interval_ms);
+
+	ctrlr->keep_alive_interval_ticks = (keep_alive_interval_ms * spdk_get_ticks_hz()) / UINT64_C(1000);
+
+	/* Schedule the first Keep Alive to be sent as soon as possible. */
+	ctrlr->next_keep_alive_tick = spdk_get_ticks();
+	nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_HOST_ID,
+			     ctrlr->opts.admin_timeout_ms);
+}
+
+static int
+nvme_ctrlr_set_keep_alive_timeout(struct spdk_nvme_ctrlr *ctrlr)
+{
+	int rc;
+
+	if (ctrlr->opts.keep_alive_timeout_ms == 0) {
+		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_HOST_ID,
+				     ctrlr->opts.admin_timeout_ms);
+		return 0;
+	}
+
+	if (ctrlr->cdata.kas == 0) {
+		SPDK_DEBUGLOG(SPDK_LOG_NVME, "Controller KAS is 0 - not enabling Keep Alive\n");
+		ctrlr->opts.keep_alive_timeout_ms = 0;
+		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_HOST_ID,
+				     ctrlr->opts.admin_timeout_ms);
+		return 0;
+	}
+
+	nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_KEEP_ALIVE_TIMEOUT,
+			     ctrlr->opts.admin_timeout_ms);
+
+	/* Retrieve actual keep alive timeout, since the controller may have adjusted it. */
+	rc = spdk_nvme_ctrlr_cmd_get_feature(ctrlr, SPDK_NVME_FEAT_KEEP_ALIVE_TIMER, 0, NULL, 0,
+					     nvme_ctrlr_set_keep_alive_timeout_done, ctrlr);
+	if (rc != 0) {
+		SPDK_ERRLOG("Keep alive timeout Get Feature failed: %d\n", rc);
+		ctrlr->opts.keep_alive_timeout_ms = 0;
+		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
+		return rc;
+	}
+
+	return 0;
+}
+
+static void
+nvme_ctrlr_set_host_id_done(void *arg, const struct spdk_nvme_cpl *cpl)
+{
+	struct spdk_nvme_ctrlr *ctrlr = (struct spdk_nvme_ctrlr *)arg;
+
+	if (spdk_nvme_cpl_is_error(cpl)) {
+		/*
+		 * Treat Set Features - Host ID failure as non-fatal, since the Host ID feature
+		 * is optional.
+		 */
+		SPDK_WARNLOG("Set Features - Host ID failed: SC 0x%x SCT 0x%x\n",
+			     cpl->status.sc, cpl->status.sct);
+	} else {
+		SPDK_DEBUGLOG(SPDK_LOG_NVME, "Set Features - Host ID was successful\n");
+	}
+
+	nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_READY, NVME_TIMEOUT_INFINITE);
+}
+
+static int
+nvme_ctrlr_set_host_id(struct spdk_nvme_ctrlr *ctrlr)
+{
+	uint8_t *host_id;
+	uint32_t host_id_size;
+	int rc;
+
+	if (ctrlr->trid.trtype != SPDK_NVME_TRANSPORT_PCIE) {
+		/*
+		 * NVMe-oF sends the host ID during Connect and doesn't allow
+		 * Set Features - Host Identifier after Connect, so we don't need to do anything here.
+		 */
+		SPDK_DEBUGLOG(SPDK_LOG_NVME, "NVMe-oF transport - not sending Set Features - Host ID\n");
+		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_READY, NVME_TIMEOUT_INFINITE);
+		return 0;
+	}
+
+	if (ctrlr->cdata.ctratt.host_id_exhid_supported) {
+		SPDK_DEBUGLOG(SPDK_LOG_NVME, "Using 128-bit extended host identifier\n");
+		host_id = ctrlr->opts.extended_host_id;
+		host_id_size = sizeof(ctrlr->opts.extended_host_id);
+	} else {
+		SPDK_DEBUGLOG(SPDK_LOG_NVME, "Using 64-bit host identifier\n");
+		host_id = ctrlr->opts.host_id;
+		host_id_size = sizeof(ctrlr->opts.host_id);
+	}
+
+	/* If the user specified an all-zeroes host identifier, don't send the command. */
+	if (spdk_mem_all_zero(host_id, host_id_size)) {
+		SPDK_DEBUGLOG(SPDK_LOG_NVME,
+			      "User did not specify host ID - not sending Set Features - Host ID\n");
+		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_READY, NVME_TIMEOUT_INFINITE);
+		return 0;
+	}
+
+	SPDK_LOGDUMP(SPDK_LOG_NVME, "host_id", host_id, host_id_size);
+
+	nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_HOST_ID,
+			     ctrlr->opts.admin_timeout_ms);
+
+	rc = nvme_ctrlr_cmd_set_host_id(ctrlr, host_id, host_id_size, nvme_ctrlr_set_host_id_done, ctrlr);
+	if (rc != 0) {
+		SPDK_ERRLOG("Set Features - Host ID failed: %d\n", rc);
+		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
+		return rc;
+	}
+
+	return 0;
+}
+
+static void
+nvme_ctrlr_destruct_namespaces(struct spdk_nvme_ctrlr *ctrlr)
+{
+	if (ctrlr->ns) {
+		uint32_t i, num_ns = ctrlr->num_ns;
+
+		for (i = 0; i < num_ns; i++) {
+			nvme_ns_destruct(&ctrlr->ns[i]);
+		}
+
+		spdk_free(ctrlr->ns);
+		ctrlr->ns = NULL;
+		ctrlr->num_ns = 0;
+	}
+
+	if (ctrlr->nsdata) {
+		spdk_free(ctrlr->nsdata);
+		ctrlr->nsdata = NULL;
+	}
+
+	spdk_free(ctrlr->active_ns_list);
+	ctrlr->active_ns_list = NULL;
+}
+
+static void
+nvme_ctrlr_update_namespaces(struct spdk_nvme_ctrlr *ctrlr)
+{
+	uint32_t i, nn = ctrlr->cdata.nn;
+	struct spdk_nvme_ns_data *nsdata;
+	bool ns_is_active;
+
+	for (i = 0; i < nn; i++) {
+		struct spdk_nvme_ns	*ns = &ctrlr->ns[i];
+		uint32_t		nsid = i + 1;
+
+		nsdata = &ctrlr->nsdata[nsid - 1];
+		ns_is_active = spdk_nvme_ctrlr_is_active_ns(ctrlr, nsid);
+
+		if (nsdata->ncap && ns_is_active) {
+			if (nvme_ns_update(ns) != 0) {
+				SPDK_ERRLOG("Failed to update active NS %u\n", nsid);
+				continue;
+			}
+		}
+
+		if ((nsdata->ncap == 0) && ns_is_active) {
+			if (nvme_ns_construct(ns, nsid, ctrlr) != 0) {
+				continue;
+			}
+		}
+
+		if (nsdata->ncap && !ns_is_active) {
+			nvme_ns_destruct(ns);
+		}
+	}
+}
+
+static int
+nvme_ctrlr_construct_namespaces(struct spdk_nvme_ctrlr *ctrlr)
+{
+	int rc = 0;
+	uint32_t nn = ctrlr->cdata.nn;
+
+	/* ctrlr->num_ns may be 0 (startup) or a different number of namespaces (reset),
+	 * so check if we need to reallocate.
+	 */
+	if (nn != ctrlr->num_ns) {
+		nvme_ctrlr_destruct_namespaces(ctrlr);
+
+		if (nn == 0) {
+			SPDK_WARNLOG("controller has 0 namespaces\n");
+			return 0;
+		}
+
+		ctrlr->ns = spdk_zmalloc(nn * sizeof(struct spdk_nvme_ns), 64, NULL,
+					 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_SHARE);
+		if (ctrlr->ns == NULL) {
+			rc = -ENOMEM;
+			goto fail;
+		}
+
+		ctrlr->nsdata = spdk_zmalloc(nn * sizeof(struct spdk_nvme_ns_data), 64,
+					     NULL, SPDK_ENV_SOCKET_ID_ANY,
+					     SPDK_MALLOC_SHARE | SPDK_MALLOC_DMA);
+		if (ctrlr->nsdata == NULL) {
+			rc = -ENOMEM;
+			goto fail;
+		}
+
+		ctrlr->num_ns = nn;
+	}
+
+	return 0;
+
+fail:
+	nvme_ctrlr_destruct_namespaces(ctrlr);
+	return rc;
+}
+
+static void
+nvme_ctrlr_async_event_cb(void *arg, const struct spdk_nvme_cpl *cpl)
+{
+	struct nvme_async_event_request	*aer = arg;
+	struct spdk_nvme_ctrlr		*ctrlr = aer->ctrlr;
+	struct spdk_nvme_ctrlr_process	*active_proc;
+	union spdk_nvme_async_event_completion	event;
+	int					rc;
+
+	if (cpl->status.sct == SPDK_NVME_SCT_GENERIC &&
+	    cpl->status.sc == SPDK_NVME_SC_ABORTED_SQ_DELETION) {
+		/*
+		 *  This is simulated when controller is being shut down, to
+		 *  effectively abort outstanding asynchronous event requests
+		 *  and make sure all memory is freed.  Do not repost the
+		 *  request in this case.
+		 */
+		return;
+	}
+
+	if (cpl->status.sct == SPDK_NVME_SCT_COMMAND_SPECIFIC &&
+	    cpl->status.sc == SPDK_NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED) {
+		/*
+		 *  SPDK will only send as many AERs as the device says it supports,
+		 *  so this status code indicates an out-of-spec device.  Do not repost
+		 *  the request in this case.
+		 */
+		SPDK_ERRLOG("Controller appears out-of-spec for asynchronous event request\n"
+			    "handling.  Do not repost this AER.\n");
+		return;
+	}
+
+	event.raw = cpl->cdw0;
+	if ((event.bits.async_event_type == SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE) &&
+	    (event.bits.async_event_info == SPDK_NVME_ASYNC_EVENT_NS_ATTR_CHANGED)) {
+		rc = nvme_ctrlr_identify_active_ns(ctrlr);
+		if (rc) {
+			return;
+		}
+		nvme_ctrlr_update_namespaces(ctrlr);
+		nvme_io_msg_ctrlr_update(ctrlr);
+	}
+
+	active_proc = nvme_ctrlr_get_current_process(ctrlr);
+	if (active_proc && active_proc->aer_cb_fn) {
+		active_proc->aer_cb_fn(active_proc->aer_cb_arg, cpl);
+	}
+
+	/* If the ctrlr was removed or in the destruct state, we should not send aer again */
+	if (ctrlr->is_removed || ctrlr->is_destructed) {
+		return;
+	}
+
+	/*
+	 * Repost another asynchronous event request to replace the one
+	 *  that just completed.
+	 */
+	if (nvme_ctrlr_construct_and_submit_aer(ctrlr, aer)) {
+		/*
+		 * We can't do anything to recover from a failure here,
+		 * so just print a warning message and leave the AER unsubmitted.
+		 */
+		SPDK_ERRLOG("resubmitting AER failed!\n");
+	}
+}
+
+static int
+nvme_ctrlr_construct_and_submit_aer(struct spdk_nvme_ctrlr *ctrlr,
+				    struct nvme_async_event_request *aer)
+{
+	struct nvme_request *req;
+
+	aer->ctrlr = ctrlr;
+	req = nvme_allocate_request_null(ctrlr->adminq, nvme_ctrlr_async_event_cb, aer);
+	aer->req = req;
+	if (req == NULL) {
+		return -1;
+	}
+
+	req->cmd.opc = SPDK_NVME_OPC_ASYNC_EVENT_REQUEST;
+	return nvme_ctrlr_submit_admin_request(ctrlr, req);
+}
+
+static void
+nvme_ctrlr_configure_aer_done(void *arg, const struct spdk_nvme_cpl *cpl)
+{
+	struct nvme_async_event_request		*aer;
+	int					rc;
+	uint32_t				i;
+	struct spdk_nvme_ctrlr *ctrlr =	(struct spdk_nvme_ctrlr *)arg;
+
+	if (spdk_nvme_cpl_is_error(cpl)) {
+		SPDK_NOTICELOG("nvme_ctrlr_configure_aer failed!\n");
+		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_SUPPORTED_LOG_PAGES,
+				     ctrlr->opts.admin_timeout_ms);
+		return;
+	}
+
+	/* aerl is a zero-based value, so we need to add 1 here. */
+	ctrlr->num_aers = spdk_min(NVME_MAX_ASYNC_EVENTS, (ctrlr->cdata.aerl + 1));
+
+	for (i = 0; i < ctrlr->num_aers; i++) {
+		aer = &ctrlr->aer[i];
+		rc = nvme_ctrlr_construct_and_submit_aer(ctrlr, aer);
+		if (rc) {
+			SPDK_ERRLOG("nvme_ctrlr_construct_and_submit_aer failed!\n");
+			nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
+			return;
+		}
+	}
+	nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_SUPPORTED_LOG_PAGES,
+			     ctrlr->opts.admin_timeout_ms);
+}
+
+static int
+nvme_ctrlr_configure_aer(struct spdk_nvme_ctrlr *ctrlr)
+{
+	union spdk_nvme_feat_async_event_configuration	config;
+	int						rc;
+
+	config.raw = 0;
+	config.bits.crit_warn.bits.available_spare = 1;
+	config.bits.crit_warn.bits.temperature = 1;
+	config.bits.crit_warn.bits.device_reliability = 1;
+	config.bits.crit_warn.bits.read_only = 1;
+	config.bits.crit_warn.bits.volatile_memory_backup = 1;
+
+	if (ctrlr->vs.raw >= SPDK_NVME_VERSION(1, 2, 0)) {
+		if (ctrlr->cdata.oaes.ns_attribute_notices) {
+			config.bits.ns_attr_notice = 1;
+		}
+		if (ctrlr->cdata.oaes.fw_activation_notices) {
+			config.bits.fw_activation_notice = 1;
+		}
+	}
+	if (ctrlr->vs.raw >= SPDK_NVME_VERSION(1, 3, 0) && ctrlr->cdata.lpa.telemetry) {
+		config.bits.telemetry_log_notice = 1;
+	}
+
+	nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_CONFIGURE_AER,
+			     ctrlr->opts.admin_timeout_ms);
+
+	rc = nvme_ctrlr_cmd_set_async_event_config(ctrlr, config,
+			nvme_ctrlr_configure_aer_done,
+			ctrlr);
+	if (rc != 0) {
+		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
+		return rc;
+	}
+
+	return 0;
+}
+
+struct spdk_nvme_ctrlr_process *
+nvme_ctrlr_get_process(struct spdk_nvme_ctrlr *ctrlr, pid_t pid)
+{
+	struct spdk_nvme_ctrlr_process	*active_proc;
+
+	TAILQ_FOREACH(active_proc, &ctrlr->active_procs, tailq) {
+		if (active_proc->pid == pid) {
+			return active_proc;
+		}
+	}
+
+	return NULL;
+}
+
+struct spdk_nvme_ctrlr_process *
+nvme_ctrlr_get_current_process(struct spdk_nvme_ctrlr *ctrlr)
+{
+	return nvme_ctrlr_get_process(ctrlr, getpid());
+}
+
+/**
+ * This function will be called when a process is using the controller.
+ *  1. For the primary process, it is called when constructing the controller.
+ *  2. For the secondary process, it is called at probing the controller.
+ * Note: will check whether the process is already added for the same process.
+ */
+int
+nvme_ctrlr_add_process(struct spdk_nvme_ctrlr *ctrlr, void *devhandle)
+{
+	struct spdk_nvme_ctrlr_process	*ctrlr_proc;
+	pid_t				pid = getpid();
+
+	/* Check whether the process is already added or not */
+	if (nvme_ctrlr_get_process(ctrlr, pid)) {
+		return 0;
+	}
+
+	/* Initialize the per process properties for this ctrlr */
+	ctrlr_proc = spdk_zmalloc(sizeof(struct spdk_nvme_ctrlr_process),
+				  64, NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_SHARE);
+	if (ctrlr_proc == NULL) {
+		SPDK_ERRLOG("failed to allocate memory to track the process props\n");
+
+		return -1;
+	}
+
+	ctrlr_proc->is_primary = spdk_process_is_primary();
+	ctrlr_proc->pid = pid;
+	STAILQ_INIT(&ctrlr_proc->active_reqs);
+	ctrlr_proc->devhandle = devhandle;
+	ctrlr_proc->ref = 0;
+	TAILQ_INIT(&ctrlr_proc->allocated_io_qpairs);
+
+	TAILQ_INSERT_TAIL(&ctrlr->active_procs, ctrlr_proc, tailq);
+
+	return 0;
+}
+
+/**
+ * This function will be called when the process detaches the controller.
+ * Note: the ctrlr_lock must be held when calling this function.
+ */
+static void
+nvme_ctrlr_remove_process(struct spdk_nvme_ctrlr *ctrlr,
+			  struct spdk_nvme_ctrlr_process *proc)
+{
+	struct spdk_nvme_qpair	*qpair, *tmp_qpair;
+
+	assert(STAILQ_EMPTY(&proc->active_reqs));
+
+	TAILQ_FOREACH_SAFE(qpair, &proc->allocated_io_qpairs, per_process_tailq, tmp_qpair) {
+		spdk_nvme_ctrlr_free_io_qpair(qpair);
+	}
+
+	TAILQ_REMOVE(&ctrlr->active_procs, proc, tailq);
+
+	if (ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) {
+		spdk_pci_device_detach(proc->devhandle);
+	}
+
+	spdk_free(proc);
+}
+
+/**
+ * This function will be called when the process exited unexpectedly
+ *  in order to free any incomplete nvme request, allocated IO qpairs
+ *  and allocated memory.
+ * Note: the ctrlr_lock must be held when calling this function.
+ */
+static void
+nvme_ctrlr_cleanup_process(struct spdk_nvme_ctrlr_process *proc)
+{
+	struct nvme_request	*req, *tmp_req;
+	struct spdk_nvme_qpair	*qpair, *tmp_qpair;
+
+	STAILQ_FOREACH_SAFE(req, &proc->active_reqs, stailq, tmp_req) {
+		STAILQ_REMOVE(&proc->active_reqs, req, nvme_request, stailq);
+
+		assert(req->pid == proc->pid);
+
+		nvme_free_request(req);
+	}
+
+	TAILQ_FOREACH_SAFE(qpair, &proc->allocated_io_qpairs, per_process_tailq, tmp_qpair) {
+		TAILQ_REMOVE(&proc->allocated_io_qpairs, qpair, per_process_tailq);
+
+		/*
+		 * The process may have been killed while some qpairs were in their
+		 *  completion context.  Clear that flag here to allow these IO
+		 *  qpairs to be deleted.
+		 */
+		qpair->in_completion_context = 0;
+
+		qpair->no_deletion_notification_needed = 1;
+
+		spdk_nvme_ctrlr_free_io_qpair(qpair);
+	}
+
+	spdk_free(proc);
+}
+
+/**
+ * This function will be called when destructing the controller.
+ *  1. There is no more admin request on this controller.
+ *  2. Clean up any left resource allocation when its associated process is gone.
+ */
+void
+nvme_ctrlr_free_processes(struct spdk_nvme_ctrlr *ctrlr)
+{
+	struct spdk_nvme_ctrlr_process	*active_proc, *tmp;
+
+	/* Free all the processes' properties and make sure no pending admin IOs */
+	TAILQ_FOREACH_SAFE(active_proc, &ctrlr->active_procs, tailq, tmp) {
+		TAILQ_REMOVE(&ctrlr->active_procs, active_proc, tailq);
+
+		assert(STAILQ_EMPTY(&active_proc->active_reqs));
+
+		spdk_free(active_proc);
+	}
+}
+
+/**
+ * This function will be called when any other process attaches or
+ *  detaches the controller in order to cleanup those unexpectedly
+ *  terminated processes.
+ * Note: the ctrlr_lock must be held when calling this function.
+ */
+static int
+nvme_ctrlr_remove_inactive_proc(struct spdk_nvme_ctrlr *ctrlr)
+{
+	struct spdk_nvme_ctrlr_process	*active_proc, *tmp;
+	int				active_proc_count = 0;
+
+	TAILQ_FOREACH_SAFE(active_proc, &ctrlr->active_procs, tailq, tmp) {
+		if ((kill(active_proc->pid, 0) == -1) && (errno == ESRCH)) {
+			SPDK_ERRLOG("process %d terminated unexpected\n", active_proc->pid);
+
+			TAILQ_REMOVE(&ctrlr->active_procs, active_proc, tailq);
+
+			nvme_ctrlr_cleanup_process(active_proc);
+		} else {
+			active_proc_count++;
+		}
+	}
+
+	return active_proc_count;
+}
+
+void
+nvme_ctrlr_proc_get_ref(struct spdk_nvme_ctrlr *ctrlr)
+{
+	struct spdk_nvme_ctrlr_process	*active_proc;
+
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+
+	nvme_ctrlr_remove_inactive_proc(ctrlr);
+
+	active_proc = nvme_ctrlr_get_current_process(ctrlr);
+	if (active_proc) {
+		active_proc->ref++;
+	}
+
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+}
+
+void
+nvme_ctrlr_proc_put_ref(struct spdk_nvme_ctrlr *ctrlr)
+{
+	struct spdk_nvme_ctrlr_process	*active_proc;
+	int				proc_count;
+
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+
+	proc_count = nvme_ctrlr_remove_inactive_proc(ctrlr);
+
+	active_proc = nvme_ctrlr_get_current_process(ctrlr);
+	if (active_proc) {
+		active_proc->ref--;
+		assert(active_proc->ref >= 0);
+
+		/*
+		 * The last active process will be removed at the end of
+		 * the destruction of the controller.
+		 */
+		if (active_proc->ref == 0 && proc_count != 1) {
+			nvme_ctrlr_remove_process(ctrlr, active_proc);
+		}
+	}
+
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+}
+
+int
+nvme_ctrlr_get_ref_count(struct spdk_nvme_ctrlr *ctrlr)
+{
+	struct spdk_nvme_ctrlr_process	*active_proc;
+	int				ref = 0;
+
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+
+	nvme_ctrlr_remove_inactive_proc(ctrlr);
+
+	TAILQ_FOREACH(active_proc, &ctrlr->active_procs, tailq) {
+		ref += active_proc->ref;
+	}
+
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+
+	return ref;
+}
+
+/**
+ *  Get the PCI device handle which is only visible to its associated process.
+ */
+struct spdk_pci_device *
+nvme_ctrlr_proc_get_devhandle(struct spdk_nvme_ctrlr *ctrlr)
+{
+	struct spdk_nvme_ctrlr_process	*active_proc;
+	struct spdk_pci_device		*devhandle = NULL;
+
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+
+	active_proc = nvme_ctrlr_get_current_process(ctrlr);
+	if (active_proc) {
+		devhandle = active_proc->devhandle;
+	}
+
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+
+	return devhandle;
+}
+
+/**
+ * This function will be called repeatedly during initialization until the controller is ready.
+ */
+int
+nvme_ctrlr_process_init(struct spdk_nvme_ctrlr *ctrlr)
+{
+	union spdk_nvme_cc_register cc;
+	union spdk_nvme_csts_register csts;
+	uint32_t ready_timeout_in_ms;
+	int rc = 0;
+
+	/*
+	 * May need to avoid accessing any register on the target controller
+	 * for a while. Return early without touching the FSM.
+	 * Check sleep_timeout_tsc > 0 for unit test.
+	 */
+	if ((ctrlr->sleep_timeout_tsc > 0) &&
+	    (spdk_get_ticks() <= ctrlr->sleep_timeout_tsc)) {
+		return 0;
+	}
+	ctrlr->sleep_timeout_tsc = 0;
+
+	if (nvme_ctrlr_get_cc(ctrlr, &cc) ||
+	    nvme_ctrlr_get_csts(ctrlr, &csts)) {
+		if (ctrlr->state_timeout_tsc != NVME_TIMEOUT_INFINITE) {
+			/* While a device is resetting, it may be unable to service MMIO reads
+			 * temporarily. Allow for this case.
+			 */
+			SPDK_ERRLOG("Get registers failed while waiting for CSTS.RDY == 0\n");
+			goto init_timeout;
+		}
+		SPDK_ERRLOG("Failed to read CC and CSTS in state %d\n", ctrlr->state);
+		return -EIO;
+	}
+
+	ready_timeout_in_ms = 500 * ctrlr->cap.bits.to;
+
+	/*
+	 * Check if the current initialization step is done or has timed out.
+	 */
+	switch (ctrlr->state) {
+	case NVME_CTRLR_STATE_INIT_DELAY:
+		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_INIT, ready_timeout_in_ms);
+		if (ctrlr->quirks & NVME_QUIRK_DELAY_BEFORE_INIT) {
+			/*
+			 * Controller may need some delay before it's enabled.
+			 *
+			 * This is a workaround for an issue where the PCIe-attached NVMe controller
+			 * is not ready after VFIO reset. We delay the initialization rather than the
+			 * enabling itself, because this is required only for the very first enabling
+			 * - directly after a VFIO reset.
+			 */
+			SPDK_DEBUGLOG(SPDK_LOG_NVME, "Adding 2 second delay before initializing the controller\n");
+			ctrlr->sleep_timeout_tsc = spdk_get_ticks() + (2000 * spdk_get_ticks_hz() / 1000);
+		}
+		break;
+
+	case NVME_CTRLR_STATE_INIT:
+		/* Begin the hardware initialization by making sure the controller is disabled. */
+		if (cc.bits.en) {
+			SPDK_DEBUGLOG(SPDK_LOG_NVME, "CC.EN = 1\n");
+			/*
+			 * Controller is currently enabled. We need to disable it to cause a reset.
+			 *
+			 * If CC.EN = 1 && CSTS.RDY = 0, the controller is in the process of becoming ready.
+			 *  Wait for the ready bit to be 1 before disabling the controller.
+			 */
+			if (csts.bits.rdy == 0) {
+				SPDK_DEBUGLOG(SPDK_LOG_NVME, "CC.EN = 1 && CSTS.RDY = 0 - waiting for reset to complete\n");
+				nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1, ready_timeout_in_ms);
+				return 0;
+			}
+
+			/* CC.EN = 1 && CSTS.RDY == 1, so we can immediately disable the controller. */
+			SPDK_DEBUGLOG(SPDK_LOG_NVME, "Setting CC.EN = 0\n");
+			cc.bits.en = 0;
+			if (nvme_ctrlr_set_cc(ctrlr, &cc)) {
+				SPDK_ERRLOG("set_cc() failed\n");
+				return -EIO;
+			}
+			nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0, ready_timeout_in_ms);
+
+			/*
+			 * Wait 2.5 seconds before accessing PCI registers.
+			 * Not using sleep() to avoid blocking other controller's initialization.
+			 */
+			if (ctrlr->quirks & NVME_QUIRK_DELAY_BEFORE_CHK_RDY) {
+				SPDK_DEBUGLOG(SPDK_LOG_NVME, "Applying quirk: delay 2.5 seconds before reading registers\n");
+				ctrlr->sleep_timeout_tsc = spdk_get_ticks() + (2500 * spdk_get_ticks_hz() / 1000);
+			}
+			return 0;
+		} else {
+			if (csts.bits.rdy == 1) {
+				SPDK_DEBUGLOG(SPDK_LOG_NVME, "CC.EN = 0 && CSTS.RDY = 1 - waiting for shutdown to complete\n");
+			}
+
+			nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0, ready_timeout_in_ms);
+			return 0;
+		}
+		break;
+
+	case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1:
+		if (csts.bits.rdy == 1) {
+			SPDK_DEBUGLOG(SPDK_LOG_NVME, "CC.EN = 1 && CSTS.RDY = 1 - disabling controller\n");
+			/* CC.EN = 1 && CSTS.RDY = 1, so we can set CC.EN = 0 now. */
+			SPDK_DEBUGLOG(SPDK_LOG_NVME, "Setting CC.EN = 0\n");
+			cc.bits.en = 0;
+			if (nvme_ctrlr_set_cc(ctrlr, &cc)) {
+				SPDK_ERRLOG("set_cc() failed\n");
+				return -EIO;
+			}
+			nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0, ready_timeout_in_ms);
+			return 0;
+		}
+		break;
+
+	case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0:
+		if (csts.bits.rdy == 0) {
+			SPDK_DEBUGLOG(SPDK_LOG_NVME, "CC.EN = 0 && CSTS.RDY = 0\n");
+			nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ENABLE, ready_timeout_in_ms);
+			/*
+			 * Delay 100us before setting CC.EN = 1.  Some NVMe SSDs miss CC.EN getting
+			 *  set to 1 if it is too soon after CSTS.RDY is reported as 0.
+			 */
+			spdk_delay_us(100);
+			return 0;
+		}
+		break;
+
+	case NVME_CTRLR_STATE_ENABLE:
+		SPDK_DEBUGLOG(SPDK_LOG_NVME, "Setting CC.EN = 1\n");
+		rc = nvme_ctrlr_enable(ctrlr);
+		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1, ready_timeout_in_ms);
+		return rc;
+
+	case NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1:
+		if (csts.bits.rdy == 1) {
+			SPDK_DEBUGLOG(SPDK_LOG_NVME, "CC.EN = 1 && CSTS.RDY = 1 - controller is ready\n");
+			/*
+			 * The controller has been enabled.
+			 *  Perform the rest of initialization serially.
+			 */
+			nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_RESET_ADMIN_QUEUE,
+					     ctrlr->opts.admin_timeout_ms);
+			return 0;
+		}
+		break;
+
+	case NVME_CTRLR_STATE_RESET_ADMIN_QUEUE:
+		nvme_transport_qpair_reset(ctrlr->adminq);
+		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY,
+				     ctrlr->opts.admin_timeout_ms);
+		break;
+
+	case NVME_CTRLR_STATE_IDENTIFY:
+		rc = nvme_ctrlr_identify(ctrlr);
+		break;
+
+	case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY:
+		spdk_nvme_qpair_process_completions(ctrlr->adminq, 0);
+		break;
+
+	case NVME_CTRLR_STATE_SET_NUM_QUEUES:
+		nvme_ctrlr_update_nvmf_ioccsz(ctrlr);
+		rc = nvme_ctrlr_set_num_queues(ctrlr);
+		break;
+
+	case NVME_CTRLR_STATE_WAIT_FOR_SET_NUM_QUEUES:
+		spdk_nvme_qpair_process_completions(ctrlr->adminq, 0);
+		break;
+
+	case NVME_CTRLR_STATE_CONSTRUCT_NS:
+		rc = nvme_ctrlr_construct_namespaces(ctrlr);
+		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_ACTIVE_NS,
+				     ctrlr->opts.admin_timeout_ms);
+		break;
+
+	case NVME_CTRLR_STATE_IDENTIFY_ACTIVE_NS:
+		_nvme_ctrlr_identify_active_ns(ctrlr);
+		break;
+
+	case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ACTIVE_NS:
+		spdk_nvme_qpair_process_completions(ctrlr->adminq, 0);
+		break;
+
+	case NVME_CTRLR_STATE_IDENTIFY_NS:
+		rc = nvme_ctrlr_identify_namespaces(ctrlr);
+		break;
+
+	case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS:
+		spdk_nvme_qpair_process_completions(ctrlr->adminq, 0);
+		break;
+
+	case NVME_CTRLR_STATE_IDENTIFY_ID_DESCS:
+		rc = nvme_ctrlr_identify_id_desc_namespaces(ctrlr);
+		break;
+
+	case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ID_DESCS:
+		spdk_nvme_qpair_process_completions(ctrlr->adminq, 0);
+		break;
+
+	case NVME_CTRLR_STATE_CONFIGURE_AER:
+		rc = nvme_ctrlr_configure_aer(ctrlr);
+		break;
+
+	case NVME_CTRLR_STATE_WAIT_FOR_CONFIGURE_AER:
+		spdk_nvme_qpair_process_completions(ctrlr->adminq, 0);
+		break;
+
+	case NVME_CTRLR_STATE_SET_SUPPORTED_LOG_PAGES:
+		rc = nvme_ctrlr_set_supported_log_pages(ctrlr);
+		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_SUPPORTED_FEATURES,
+				     ctrlr->opts.admin_timeout_ms);
+		break;
+
+	case NVME_CTRLR_STATE_SET_SUPPORTED_FEATURES:
+		nvme_ctrlr_set_supported_features(ctrlr);
+		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_DB_BUF_CFG,
+				     ctrlr->opts.admin_timeout_ms);
+		break;
+
+	case NVME_CTRLR_STATE_SET_DB_BUF_CFG:
+		rc = nvme_ctrlr_set_doorbell_buffer_config(ctrlr);
+		break;
+
+	case NVME_CTRLR_STATE_WAIT_FOR_DB_BUF_CFG:
+		spdk_nvme_qpair_process_completions(ctrlr->adminq, 0);
+		break;
+
+	case NVME_CTRLR_STATE_SET_KEEP_ALIVE_TIMEOUT:
+		rc = nvme_ctrlr_set_keep_alive_timeout(ctrlr);
+		break;
+
+	case NVME_CTRLR_STATE_WAIT_FOR_KEEP_ALIVE_TIMEOUT:
+		spdk_nvme_qpair_process_completions(ctrlr->adminq, 0);
+		break;
+
+	case NVME_CTRLR_STATE_SET_HOST_ID:
+		rc = nvme_ctrlr_set_host_id(ctrlr);
+		break;
+
+	case NVME_CTRLR_STATE_WAIT_FOR_HOST_ID:
+		spdk_nvme_qpair_process_completions(ctrlr->adminq, 0);
+		break;
+
+	case NVME_CTRLR_STATE_READY:
+		SPDK_DEBUGLOG(SPDK_LOG_NVME, "Ctrlr already in ready state\n");
+		return 0;
+
+	case NVME_CTRLR_STATE_ERROR:
+		SPDK_ERRLOG("Ctrlr %s is in error state\n", ctrlr->trid.traddr);
+		return -1;
+
+	default:
+		assert(0);
+		return -1;
+	}
+
+init_timeout:
+	if (ctrlr->state_timeout_tsc != NVME_TIMEOUT_INFINITE &&
+	    spdk_get_ticks() > ctrlr->state_timeout_tsc) {
+		SPDK_ERRLOG("Initialization timed out in state %d\n", ctrlr->state);
+		return -1;
+	}
+
+	return rc;
+}
+
+int
+nvme_robust_mutex_init_recursive_shared(pthread_mutex_t *mtx)
+{
+	pthread_mutexattr_t attr;
+	int rc = 0;
+
+	if (pthread_mutexattr_init(&attr)) {
+		return -1;
+	}
+	if (pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE) ||
+#ifndef __FreeBSD__
+	    pthread_mutexattr_setrobust(&attr, PTHREAD_MUTEX_ROBUST) ||
+	    pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED) ||
+#endif
+	    pthread_mutex_init(mtx, &attr)) {
+		rc = -1;
+	}
+	pthread_mutexattr_destroy(&attr);
+	return rc;
+}
+
+int
+nvme_ctrlr_construct(struct spdk_nvme_ctrlr *ctrlr)
+{
+	int rc;
+
+	if (ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) {
+		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_INIT_DELAY, NVME_TIMEOUT_INFINITE);
+	} else {
+		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_INIT, NVME_TIMEOUT_INFINITE);
+	}
+
+	if (ctrlr->opts.admin_queue_size > SPDK_NVME_ADMIN_QUEUE_MAX_ENTRIES) {
+		SPDK_ERRLOG("admin_queue_size %u exceeds max defined by NVMe spec, use max value\n",
+			    ctrlr->opts.admin_queue_size);
+		ctrlr->opts.admin_queue_size = SPDK_NVME_ADMIN_QUEUE_MAX_ENTRIES;
+	}
+
+	if (ctrlr->opts.admin_queue_size < SPDK_NVME_ADMIN_QUEUE_MIN_ENTRIES) {
+		SPDK_ERRLOG("admin_queue_size %u is less than minimum defined by NVMe spec, use min value\n",
+			    ctrlr->opts.admin_queue_size);
+		ctrlr->opts.admin_queue_size = SPDK_NVME_ADMIN_QUEUE_MIN_ENTRIES;
+	}
+
+	ctrlr->flags = 0;
+	ctrlr->free_io_qids = NULL;
+	ctrlr->is_resetting = false;
+	ctrlr->is_failed = false;
+	ctrlr->is_destructed = false;
+
+	TAILQ_INIT(&ctrlr->active_io_qpairs);
+	STAILQ_INIT(&ctrlr->queued_aborts);
+	ctrlr->outstanding_aborts = 0;
+
+	rc = nvme_robust_mutex_init_recursive_shared(&ctrlr->ctrlr_lock);
+	if (rc != 0) {
+		return rc;
+	}
+
+	TAILQ_INIT(&ctrlr->active_procs);
+
+	return rc;
+}
+
+/* This function should be called once at ctrlr initialization to set up constant properties. */
+void
+nvme_ctrlr_init_cap(struct spdk_nvme_ctrlr *ctrlr, const union spdk_nvme_cap_register *cap,
+		    const union spdk_nvme_vs_register *vs)
+{
+	ctrlr->cap = *cap;
+	ctrlr->vs = *vs;
+
+	if (ctrlr->cap.bits.ams & SPDK_NVME_CAP_AMS_WRR) {
+		ctrlr->flags |= SPDK_NVME_CTRLR_WRR_SUPPORTED;
+	}
+
+	ctrlr->min_page_size = 1u << (12 + ctrlr->cap.bits.mpsmin);
+
+	/* For now, always select page_size == min_page_size. */
+	ctrlr->page_size = ctrlr->min_page_size;
+
+	ctrlr->opts.io_queue_size = spdk_max(ctrlr->opts.io_queue_size, SPDK_NVME_IO_QUEUE_MIN_ENTRIES);
+	ctrlr->opts.io_queue_size = spdk_min(ctrlr->opts.io_queue_size, MAX_IO_QUEUE_ENTRIES);
+	ctrlr->opts.io_queue_size = spdk_min(ctrlr->opts.io_queue_size, ctrlr->cap.bits.mqes + 1u);
+
+	ctrlr->opts.io_queue_requests = spdk_max(ctrlr->opts.io_queue_requests, ctrlr->opts.io_queue_size);
+}
+
+void
+nvme_ctrlr_destruct_finish(struct spdk_nvme_ctrlr *ctrlr)
+{
+	pthread_mutex_destroy(&ctrlr->ctrlr_lock);
+}
+
+void
+nvme_ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr)
+{
+	struct spdk_nvme_qpair *qpair, *tmp;
+
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "Prepare to destruct SSD: %s\n", ctrlr->trid.traddr);
+
+	ctrlr->is_destructed = true;
+
+	spdk_nvme_qpair_process_completions(ctrlr->adminq, 0);
+
+	nvme_ctrlr_abort_queued_aborts(ctrlr);
+	nvme_transport_admin_qpair_abort_aers(ctrlr->adminq);
+
+	TAILQ_FOREACH_SAFE(qpair, &ctrlr->active_io_qpairs, tailq, tmp) {
+		spdk_nvme_ctrlr_free_io_qpair(qpair);
+	}
+
+	nvme_ctrlr_free_doorbell_buffer(ctrlr);
+
+	if (ctrlr->opts.no_shn_notification) {
+		SPDK_INFOLOG(SPDK_LOG_NVME, "Disable SSD: %s without shutdown notification\n",
+			     ctrlr->trid.traddr);
+		nvme_ctrlr_disable(ctrlr);
+	} else {
+		nvme_ctrlr_shutdown(ctrlr);
+	}
+
+	nvme_ctrlr_destruct_namespaces(ctrlr);
+
+	spdk_bit_array_free(&ctrlr->free_io_qids);
+
+	nvme_transport_ctrlr_destruct(ctrlr);
+}
+
+int
+nvme_ctrlr_submit_admin_request(struct spdk_nvme_ctrlr *ctrlr,
+				struct nvme_request *req)
+{
+	return nvme_qpair_submit_request(ctrlr->adminq, req);
+}
+
+static void
+nvme_keep_alive_completion(void *cb_ctx, const struct spdk_nvme_cpl *cpl)
+{
+	/* Do nothing */
+}
+
+/*
+ * Check if we need to send a Keep Alive command.
+ * Caller must hold ctrlr->ctrlr_lock.
+ */
+static void
+nvme_ctrlr_keep_alive(struct spdk_nvme_ctrlr *ctrlr)
+{
+	uint64_t now;
+	struct nvme_request *req;
+	struct spdk_nvme_cmd *cmd;
+	int rc;
+
+	now = spdk_get_ticks();
+	if (now < ctrlr->next_keep_alive_tick) {
+		return;
+	}
+
+	req = nvme_allocate_request_null(ctrlr->adminq, nvme_keep_alive_completion, NULL);
+	if (req == NULL) {
+		return;
+	}
+
+	cmd = &req->cmd;
+	cmd->opc = SPDK_NVME_OPC_KEEP_ALIVE;
+
+	rc = nvme_ctrlr_submit_admin_request(ctrlr, req);
+	if (rc != 0) {
+		SPDK_ERRLOG("Submitting Keep Alive failed\n");
+	}
+
+	ctrlr->next_keep_alive_tick = now + ctrlr->keep_alive_interval_ticks;
+}
+
+int32_t
+spdk_nvme_ctrlr_process_admin_completions(struct spdk_nvme_ctrlr *ctrlr)
+{
+	int32_t num_completions;
+	int32_t rc;
+
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+
+	if (ctrlr->keep_alive_interval_ticks) {
+		nvme_ctrlr_keep_alive(ctrlr);
+	}
+
+	rc = nvme_io_msg_process(ctrlr);
+	if (rc < 0) {
+		nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+		return rc;
+	}
+	num_completions = rc;
+
+	rc = spdk_nvme_qpair_process_completions(ctrlr->adminq, 0);
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+
+	if (rc < 0) {
+		num_completions = rc;
+	} else {
+		num_completions += rc;
+	}
+
+	return num_completions;
+}
+
+const struct spdk_nvme_ctrlr_data *
+spdk_nvme_ctrlr_get_data(struct spdk_nvme_ctrlr *ctrlr)
+{
+	return &ctrlr->cdata;
+}
+
+union spdk_nvme_csts_register spdk_nvme_ctrlr_get_regs_csts(struct spdk_nvme_ctrlr *ctrlr)
+{
+	union spdk_nvme_csts_register csts;
+
+	if (nvme_ctrlr_get_csts(ctrlr, &csts)) {
+		csts.raw = 0xFFFFFFFFu;
+	}
+	return csts;
+}
+
+union spdk_nvme_cap_register spdk_nvme_ctrlr_get_regs_cap(struct spdk_nvme_ctrlr *ctrlr)
+{
+	return ctrlr->cap;
+}
+
+union spdk_nvme_vs_register spdk_nvme_ctrlr_get_regs_vs(struct spdk_nvme_ctrlr *ctrlr)
+{
+	return ctrlr->vs;
+}
+
+union spdk_nvme_cmbsz_register spdk_nvme_ctrlr_get_regs_cmbsz(struct spdk_nvme_ctrlr *ctrlr)
+{
+	union spdk_nvme_cmbsz_register cmbsz;
+
+	if (nvme_ctrlr_get_cmbsz(ctrlr, &cmbsz)) {
+		cmbsz.raw = 0;
+	}
+
+	return cmbsz;
+}
+
+uint32_t
+spdk_nvme_ctrlr_get_num_ns(struct spdk_nvme_ctrlr *ctrlr)
+{
+	return ctrlr->num_ns;
+}
+
+static int32_t
+nvme_ctrlr_active_ns_idx(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid)
+{
+	int32_t result = -1;
+
+	if (ctrlr->active_ns_list == NULL || nsid == 0 || nsid > ctrlr->num_ns) {
+		return result;
+	}
+
+	int32_t lower = 0;
+	int32_t upper = ctrlr->num_ns - 1;
+	int32_t mid;
+
+	while (lower <= upper) {
+		mid = lower + (upper - lower) / 2;
+		if (ctrlr->active_ns_list[mid] == nsid) {
+			result = mid;
+			break;
+		} else {
+			if (ctrlr->active_ns_list[mid] != 0 && ctrlr->active_ns_list[mid] < nsid) {
+				lower = mid + 1;
+			} else {
+				upper = mid - 1;
+			}
+
+		}
+	}
+
+	return result;
+}
+
+bool
+spdk_nvme_ctrlr_is_active_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid)
+{
+	return nvme_ctrlr_active_ns_idx(ctrlr, nsid) != -1;
+}
+
+uint32_t
+spdk_nvme_ctrlr_get_first_active_ns(struct spdk_nvme_ctrlr *ctrlr)
+{
+	return ctrlr->active_ns_list ? ctrlr->active_ns_list[0] : 0;
+}
+
+uint32_t
+spdk_nvme_ctrlr_get_next_active_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t prev_nsid)
+{
+	int32_t nsid_idx = nvme_ctrlr_active_ns_idx(ctrlr, prev_nsid);
+	if (ctrlr->active_ns_list && nsid_idx >= 0 && (uint32_t)nsid_idx < ctrlr->num_ns - 1) {
+		return ctrlr->active_ns_list[nsid_idx + 1];
+	}
+	return 0;
+}
+
+struct spdk_nvme_ns *
+spdk_nvme_ctrlr_get_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid)
+{
+	if (nsid < 1 || nsid > ctrlr->num_ns) {
+		return NULL;
+	}
+
+	return &ctrlr->ns[nsid - 1];
+}
+
+struct spdk_pci_device *
+spdk_nvme_ctrlr_get_pci_device(struct spdk_nvme_ctrlr *ctrlr)
+{
+	if (ctrlr == NULL) {
+		return NULL;
+	}
+
+	if (ctrlr->trid.trtype != SPDK_NVME_TRANSPORT_PCIE) {
+		return NULL;
+	}
+
+	return nvme_ctrlr_proc_get_devhandle(ctrlr);
+}
+
+uint32_t
+spdk_nvme_ctrlr_get_max_xfer_size(const struct spdk_nvme_ctrlr *ctrlr)
+{
+	return ctrlr->max_xfer_size;
+}
+
+void
+spdk_nvme_ctrlr_register_aer_callback(struct spdk_nvme_ctrlr *ctrlr,
+				      spdk_nvme_aer_cb aer_cb_fn,
+				      void *aer_cb_arg)
+{
+	struct spdk_nvme_ctrlr_process *active_proc;
+
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+
+	active_proc = nvme_ctrlr_get_current_process(ctrlr);
+	if (active_proc) {
+		active_proc->aer_cb_fn = aer_cb_fn;
+		active_proc->aer_cb_arg = aer_cb_arg;
+	}
+
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+}
+
+void
+spdk_nvme_ctrlr_register_timeout_callback(struct spdk_nvme_ctrlr *ctrlr,
+		uint64_t timeout_us, spdk_nvme_timeout_cb cb_fn, void *cb_arg)
+{
+	struct spdk_nvme_ctrlr_process	*active_proc;
+
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+
+	active_proc = nvme_ctrlr_get_current_process(ctrlr);
+	if (active_proc) {
+		active_proc->timeout_ticks = timeout_us * spdk_get_ticks_hz() / 1000000ULL;
+		active_proc->timeout_cb_fn = cb_fn;
+		active_proc->timeout_cb_arg = cb_arg;
+	}
+
+	ctrlr->timeout_enabled = true;
+
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+}
+
+bool
+spdk_nvme_ctrlr_is_log_page_supported(struct spdk_nvme_ctrlr *ctrlr, uint8_t log_page)
+{
+	/* No bounds check necessary, since log_page is uint8_t and log_page_supported has 256 entries */
+	SPDK_STATIC_ASSERT(sizeof(ctrlr->log_page_supported) == 256, "log_page_supported size mismatch");
+	return ctrlr->log_page_supported[log_page];
+}
+
+bool
+spdk_nvme_ctrlr_is_feature_supported(struct spdk_nvme_ctrlr *ctrlr, uint8_t feature_code)
+{
+	/* No bounds check necessary, since feature_code is uint8_t and feature_supported has 256 entries */
+	SPDK_STATIC_ASSERT(sizeof(ctrlr->feature_supported) == 256, "feature_supported size mismatch");
+	return ctrlr->feature_supported[feature_code];
+}
+
+int
+spdk_nvme_ctrlr_attach_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid,
+			  struct spdk_nvme_ctrlr_list *payload)
+{
+	struct nvme_completion_poll_status	*status;
+	int					res;
+	struct spdk_nvme_ns			*ns;
+
+	status = calloc(1, sizeof(*status));
+	if (!status) {
+		SPDK_ERRLOG("Failed to allocate status tracker\n");
+		return -ENOMEM;
+	}
+
+	res = nvme_ctrlr_cmd_attach_ns(ctrlr, nsid, payload,
+				       nvme_completion_poll_cb, status);
+	if (res) {
+		free(status);
+		return res;
+	}
+	if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) {
+		SPDK_ERRLOG("spdk_nvme_ctrlr_attach_ns failed!\n");
+		if (!status->timed_out) {
+			free(status);
+		}
+		return -ENXIO;
+	}
+	free(status);
+
+	res = nvme_ctrlr_identify_active_ns(ctrlr);
+	if (res) {
+		return res;
+	}
+
+	ns = &ctrlr->ns[nsid - 1];
+	return nvme_ns_construct(ns, nsid, ctrlr);
+}
+
+int
+spdk_nvme_ctrlr_detach_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid,
+			  struct spdk_nvme_ctrlr_list *payload)
+{
+	struct nvme_completion_poll_status	*status;
+	int					res;
+	struct spdk_nvme_ns			*ns;
+
+	status = calloc(1, sizeof(*status));
+	if (!status) {
+		SPDK_ERRLOG("Failed to allocate status tracker\n");
+		return -ENOMEM;
+	}
+
+	res = nvme_ctrlr_cmd_detach_ns(ctrlr, nsid, payload,
+				       nvme_completion_poll_cb, status);
+	if (res) {
+		free(status);
+		return res;
+	}
+	if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) {
+		SPDK_ERRLOG("spdk_nvme_ctrlr_detach_ns failed!\n");
+		if (!status->timed_out) {
+			free(status);
+		}
+		return -ENXIO;
+	}
+	free(status);
+
+	res = nvme_ctrlr_identify_active_ns(ctrlr);
+	if (res) {
+		return res;
+	}
+
+	ns = &ctrlr->ns[nsid - 1];
+	/* Inactive NS */
+	nvme_ns_destruct(ns);
+
+	return 0;
+}
+
+uint32_t
+spdk_nvme_ctrlr_create_ns(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns_data *payload)
+{
+	struct nvme_completion_poll_status	*status;
+	int					res;
+	uint32_t				nsid;
+	struct spdk_nvme_ns			*ns;
+
+	status = calloc(1, sizeof(*status));
+	if (!status) {
+		SPDK_ERRLOG("Failed to allocate status tracker\n");
+		return 0;
+	}
+
+	res = nvme_ctrlr_cmd_create_ns(ctrlr, payload, nvme_completion_poll_cb, status);
+	if (res) {
+		free(status);
+		return 0;
+	}
+	if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) {
+		SPDK_ERRLOG("spdk_nvme_ctrlr_create_ns failed!\n");
+		if (!status->timed_out) {
+			free(status);
+		}
+		return 0;
+	}
+
+	nsid = status->cpl.cdw0;
+	ns = &ctrlr->ns[nsid - 1];
+	free(status);
+	/* Inactive NS */
+	res = nvme_ns_construct(ns, nsid, ctrlr);
+	if (res) {
+		return 0;
+	}
+
+	/* Return the namespace ID that was created */
+	return nsid;
+}
+
+int
+spdk_nvme_ctrlr_delete_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid)
+{
+	struct nvme_completion_poll_status	*status;
+	int					res;
+	struct spdk_nvme_ns			*ns;
+
+	status = calloc(1, sizeof(*status));
+	if (!status) {
+		SPDK_ERRLOG("Failed to allocate status tracker\n");
+		return -ENOMEM;
+	}
+
+	res = nvme_ctrlr_cmd_delete_ns(ctrlr, nsid, nvme_completion_poll_cb, status);
+	if (res) {
+		free(status);
+		return res;
+	}
+	if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) {
+		SPDK_ERRLOG("spdk_nvme_ctrlr_delete_ns failed!\n");
+		if (!status->timed_out) {
+			free(status);
+		}
+		return -ENXIO;
+	}
+	free(status);
+
+	res = nvme_ctrlr_identify_active_ns(ctrlr);
+	if (res) {
+		return res;
+	}
+
+	ns = &ctrlr->ns[nsid - 1];
+	nvme_ns_destruct(ns);
+
+	return 0;
+}
+
+int
+spdk_nvme_ctrlr_format(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid,
+		       struct spdk_nvme_format *format)
+{
+	struct nvme_completion_poll_status	*status;
+	int					res;
+
+	status = calloc(1, sizeof(*status));
+	if (!status) {
+		SPDK_ERRLOG("Failed to allocate status tracker\n");
+		return -ENOMEM;
+	}
+
+	res = nvme_ctrlr_cmd_format(ctrlr, nsid, format, nvme_completion_poll_cb,
+				    status);
+	if (res) {
+		free(status);
+		return res;
+	}
+	if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) {
+		SPDK_ERRLOG("spdk_nvme_ctrlr_format failed!\n");
+		if (!status->timed_out) {
+			free(status);
+		}
+		return -ENXIO;
+	}
+	free(status);
+
+	return spdk_nvme_ctrlr_reset(ctrlr);
+}
+
+int
+spdk_nvme_ctrlr_update_firmware(struct spdk_nvme_ctrlr *ctrlr, void *payload, uint32_t size,
+				int slot, enum spdk_nvme_fw_commit_action commit_action, struct spdk_nvme_status *completion_status)
+{
+	struct spdk_nvme_fw_commit		fw_commit;
+	struct nvme_completion_poll_status	*status;
+	int					res;
+	unsigned int				size_remaining;
+	unsigned int				offset;
+	unsigned int				transfer;
+	void					*p;
+
+	if (!completion_status) {
+		return -EINVAL;
+	}
+	memset(completion_status, 0, sizeof(struct spdk_nvme_status));
+	if (size % 4) {
+		SPDK_ERRLOG("spdk_nvme_ctrlr_update_firmware invalid size!\n");
+		return -1;
+	}
+
+	/* Current support only for SPDK_NVME_FW_COMMIT_REPLACE_IMG
+	 * and SPDK_NVME_FW_COMMIT_REPLACE_AND_ENABLE_IMG
+	 */
+	if ((commit_action != SPDK_NVME_FW_COMMIT_REPLACE_IMG) &&
+	    (commit_action != SPDK_NVME_FW_COMMIT_REPLACE_AND_ENABLE_IMG)) {
+		SPDK_ERRLOG("spdk_nvme_ctrlr_update_firmware invalid command!\n");
+		return -1;
+	}
+
+	status = calloc(1, sizeof(*status));
+	if (!status) {
+		SPDK_ERRLOG("Failed to allocate status tracker\n");
+		return -ENOMEM;
+	}
+
+	/* Firmware download */
+	size_remaining = size;
+	offset = 0;
+	p = payload;
+
+	while (size_remaining > 0) {
+		transfer = spdk_min(size_remaining, ctrlr->min_page_size);
+
+		memset(status, 0, sizeof(*status));
+		res = nvme_ctrlr_cmd_fw_image_download(ctrlr, transfer, offset, p,
+						       nvme_completion_poll_cb,
+						       status);
+		if (res) {
+			free(status);
+			return res;
+		}
+
+		if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) {
+			SPDK_ERRLOG("spdk_nvme_ctrlr_fw_image_download failed!\n");
+			if (!status->timed_out) {
+				free(status);
+			}
+			return -ENXIO;
+		}
+		p += transfer;
+		offset += transfer;
+		size_remaining -= transfer;
+	}
+
+	/* Firmware commit */
+	memset(&fw_commit, 0, sizeof(struct spdk_nvme_fw_commit));
+	fw_commit.fs = slot;
+	fw_commit.ca = commit_action;
+
+	memset(status, 0, sizeof(*status));
+	res = nvme_ctrlr_cmd_fw_commit(ctrlr, &fw_commit, nvme_completion_poll_cb,
+				       status);
+	if (res) {
+		free(status);
+		return res;
+	}
+
+	res = nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock);
+
+	memcpy(completion_status, &status->cpl.status, sizeof(struct spdk_nvme_status));
+
+	if (!status->timed_out) {
+		free(status);
+	}
+
+	if (res) {
+		if (completion_status->sct != SPDK_NVME_SCT_COMMAND_SPECIFIC ||
+		    completion_status->sc != SPDK_NVME_SC_FIRMWARE_REQ_NVM_RESET) {
+			if (completion_status->sct == SPDK_NVME_SCT_COMMAND_SPECIFIC  &&
+			    completion_status->sc == SPDK_NVME_SC_FIRMWARE_REQ_CONVENTIONAL_RESET) {
+				SPDK_NOTICELOG("firmware activation requires conventional reset to be performed. !\n");
+			} else {
+				SPDK_ERRLOG("nvme_ctrlr_cmd_fw_commit failed!\n");
+			}
+			return -ENXIO;
+		}
+	}
+
+	return spdk_nvme_ctrlr_reset(ctrlr);
+}
+
+int
+spdk_nvme_ctrlr_reserve_cmb(struct spdk_nvme_ctrlr *ctrlr)
+{
+	int rc, size;
+	union spdk_nvme_cmbsz_register cmbsz;
+
+	cmbsz = spdk_nvme_ctrlr_get_regs_cmbsz(ctrlr);
+
+	if (cmbsz.bits.rds == 0 || cmbsz.bits.wds == 0) {
+		return -ENOTSUP;
+	}
+
+	size = cmbsz.bits.sz * (0x1000 << (cmbsz.bits.szu * 4));
+
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+	rc = nvme_transport_ctrlr_reserve_cmb(ctrlr);
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+
+	if (rc < 0) {
+		return rc;
+	}
+
+	return size;
+}
+
+void *
+spdk_nvme_ctrlr_map_cmb(struct spdk_nvme_ctrlr *ctrlr, size_t *size)
+{
+	void *buf;
+
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+	buf = nvme_transport_ctrlr_map_cmb(ctrlr, size);
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+
+	return buf;
+}
+
+void
+spdk_nvme_ctrlr_unmap_cmb(struct spdk_nvme_ctrlr *ctrlr)
+{
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+	nvme_transport_ctrlr_unmap_cmb(ctrlr);
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+}
+
+bool
+spdk_nvme_ctrlr_is_discovery(struct spdk_nvme_ctrlr *ctrlr)
+{
+	assert(ctrlr);
+
+	return !strncmp(ctrlr->trid.subnqn, SPDK_NVMF_DISCOVERY_NQN,
+			strlen(SPDK_NVMF_DISCOVERY_NQN));
+}
+
+int
+spdk_nvme_ctrlr_security_receive(struct spdk_nvme_ctrlr *ctrlr, uint8_t secp,
+				 uint16_t spsp, uint8_t nssf, void *payload, size_t size)
+{
+	struct nvme_completion_poll_status	*status;
+	int					res;
+
+	status = calloc(1, sizeof(*status));
+	if (!status) {
+		SPDK_ERRLOG("Failed to allocate status tracker\n");
+		return -ENOMEM;
+	}
+
+	res = spdk_nvme_ctrlr_cmd_security_receive(ctrlr, secp, spsp, nssf, payload, size,
+			nvme_completion_poll_cb, status);
+	if (res) {
+		free(status);
+		return res;
+	}
+	if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) {
+		SPDK_ERRLOG("spdk_nvme_ctrlr_cmd_security_receive failed!\n");
+		if (!status->timed_out) {
+			free(status);
+		}
+		return -ENXIO;
+	}
+	free(status);
+
+	return 0;
+}
+
+int
+spdk_nvme_ctrlr_security_send(struct spdk_nvme_ctrlr *ctrlr, uint8_t secp,
+			      uint16_t spsp, uint8_t nssf, void *payload, size_t size)
+{
+	struct nvme_completion_poll_status	*status;
+	int					res;
+
+	status = calloc(1, sizeof(*status));
+	if (!status) {
+		SPDK_ERRLOG("Failed to allocate status tracker\n");
+		return -ENOMEM;
+	}
+
+	res = spdk_nvme_ctrlr_cmd_security_send(ctrlr, secp, spsp, nssf, payload, size,
+						nvme_completion_poll_cb,
+						status);
+	if (res) {
+		free(status);
+		return res;
+	}
+	if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) {
+		SPDK_ERRLOG("spdk_nvme_ctrlr_cmd_security_send failed!\n");
+		if (!status->timed_out) {
+			free(status);
+		}
+		return -ENXIO;
+	}
+
+	free(status);
+
+	return 0;
+}
+
+uint64_t
+spdk_nvme_ctrlr_get_flags(struct spdk_nvme_ctrlr *ctrlr)
+{
+	return ctrlr->flags;
+}
+
+const struct spdk_nvme_transport_id *
+spdk_nvme_ctrlr_get_transport_id(struct spdk_nvme_ctrlr *ctrlr)
+{
+	return &ctrlr->trid;
+}
+
+/* FIXME need to specify max number of iovs */
+int
+spdk_nvme_map_prps(void *prv, struct spdk_nvme_cmd *cmd, struct iovec *iovs,
+		   uint32_t len, size_t mps,
+		   void *(*gpa_to_vva)(void *prv, uint64_t addr, uint64_t len))
+{
+	uint64_t prp1, prp2;
+	void *vva;
+	uint32_t i;
+	uint32_t residue_len, nents;
+	uint64_t *prp_list;
+	int iovcnt;
+
+	prp1 = cmd->dptr.prp.prp1;
+	prp2 = cmd->dptr.prp.prp2;
+
+	/* PRP1 may started with unaligned page address */
+	residue_len = mps - (prp1 % mps);
+	residue_len = spdk_min(len, residue_len);
+
+	vva = gpa_to_vva(prv, prp1, residue_len);
+	if (spdk_unlikely(vva == NULL)) {
+		SPDK_ERRLOG("GPA to VVA failed\n");
+		return -1;
+	}
+	iovs[0].iov_base = vva;
+	iovs[0].iov_len = residue_len;
+	len -= residue_len;
+
+	if (len) {
+		if (spdk_unlikely(prp2 == 0)) {
+			SPDK_ERRLOG("no PRP2, %d remaining\n", len);
+			return -1;
+		}
+
+		if (len <= mps) {
+			/* 2 PRP used */
+			iovcnt = 2;
+			vva = gpa_to_vva(prv, prp2, len);
+			if (spdk_unlikely(vva == NULL)) {
+				SPDK_ERRLOG("no VVA for %#lx, len%#x\n",
+					    prp2, len);
+				return -1;
+			}
+			iovs[1].iov_base = vva;
+			iovs[1].iov_len = len;
+		} else {
+			/* PRP list used */
+			nents = (len + mps - 1) / mps;
+			vva = gpa_to_vva(prv, prp2, nents * sizeof(*prp_list));
+			if (spdk_unlikely(vva == NULL)) {
+				SPDK_ERRLOG("no VVA for %#lx, nents=%#x\n",
+					    prp2, nents);
+				return -1;
+			}
+			prp_list = vva;
+			i = 0;
+			while (len != 0) {
+				residue_len = spdk_min(len, mps);
+				vva = gpa_to_vva(prv, prp_list[i], residue_len);
+				if (spdk_unlikely(vva == NULL)) {
+					SPDK_ERRLOG("no VVA for %#lx, residue_len=%#x\n",
+						    prp_list[i], residue_len);
+					return -1;
+				}
+				iovs[i + 1].iov_base = vva;
+				iovs[i + 1].iov_len = residue_len;
+				len -= residue_len;
+				i++;
+			}
+			iovcnt = i + 1;
+		}
+	} else {
+		/* 1 PRP used */
+		iovcnt = 1;
+	}
+
+	return iovcnt;
+}
diff --git a/src/spdk/lib/nvme/nvme_ctrlr_cmd.c b/src/spdk/lib/nvme/nvme_ctrlr_cmd.c
new file mode 100644
index 000000000..9b16c8d6f
--- /dev/null
+++ b/src/spdk/lib/nvme/nvme_ctrlr_cmd.c
@@ -0,0 +1,966 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "nvme_internal.h"
+
+int
+spdk_nvme_ctrlr_io_cmd_raw_no_payload_build(struct spdk_nvme_ctrlr *ctrlr,
+		struct spdk_nvme_qpair *qpair,
+		struct spdk_nvme_cmd *cmd,
+		spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	struct nvme_request *req;
+	struct nvme_payload payload;
+
+	if (ctrlr->trid.trtype != SPDK_NVME_TRANSPORT_PCIE) {
+		return -EINVAL;
+	}
+
+	memset(&payload, 0, sizeof(payload));
+	req = nvme_allocate_request(qpair, &payload, 0, 0, cb_fn, cb_arg);
+
+	if (req == NULL) {
+		return -ENOMEM;
+	}
+
+	memcpy(&req->cmd, cmd, sizeof(req->cmd));
+
+	return nvme_qpair_submit_request(qpair, req);
+}
+
+int
+spdk_nvme_ctrlr_cmd_io_raw(struct spdk_nvme_ctrlr *ctrlr,
+			   struct spdk_nvme_qpair *qpair,
+			   struct spdk_nvme_cmd *cmd,
+			   void *buf, uint32_t len,
+			   spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	struct nvme_request	*req;
+
+	req = nvme_allocate_request_contig(qpair, buf, len, cb_fn, cb_arg);
+
+	if (req == NULL) {
+		return -ENOMEM;
+	}
+
+	memcpy(&req->cmd, cmd, sizeof(req->cmd));
+
+	return nvme_qpair_submit_request(qpair, req);
+}
+
+int
+spdk_nvme_ctrlr_cmd_io_raw_with_md(struct spdk_nvme_ctrlr *ctrlr,
+				   struct spdk_nvme_qpair *qpair,
+				   struct spdk_nvme_cmd *cmd,
+				   void *buf, uint32_t len, void *md_buf,
+				   spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	struct nvme_request *req;
+	struct nvme_payload payload;
+	uint32_t md_len = 0;
+
+	payload = NVME_PAYLOAD_CONTIG(buf, md_buf);
+
+	/* Caculate metadata length */
+	if (md_buf) {
+		struct spdk_nvme_ns *ns = &ctrlr->ns[cmd->nsid - 1];
+
+		assert(ns->sector_size != 0);
+		md_len =  len / ns->sector_size * ns->md_size;
+	}
+
+	req = nvme_allocate_request(qpair, &payload, len, md_len, cb_fn, cb_arg);
+	if (req == NULL) {
+		return -ENOMEM;
+	}
+
+	memcpy(&req->cmd, cmd, sizeof(req->cmd));
+
+	return nvme_qpair_submit_request(qpair, req);
+}
+
+int
+spdk_nvme_ctrlr_cmd_admin_raw(struct spdk_nvme_ctrlr *ctrlr,
+			      struct spdk_nvme_cmd *cmd,
+			      void *buf, uint32_t len,
+			      spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	struct nvme_request	*req;
+	int			rc;
+
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+	req = nvme_allocate_request_contig(ctrlr->adminq, buf, len, cb_fn, cb_arg);
+	if (req == NULL) {
+		nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+		return -ENOMEM;
+	}
+
+	memcpy(&req->cmd, cmd, sizeof(req->cmd));
+
+	rc = nvme_ctrlr_submit_admin_request(ctrlr, req);
+
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+	return rc;
+}
+
+int
+nvme_ctrlr_cmd_identify(struct spdk_nvme_ctrlr *ctrlr, uint8_t cns, uint16_t cntid, uint32_t nsid,
+			void *payload, size_t payload_size,
+			spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	struct nvme_request *req;
+	struct spdk_nvme_cmd *cmd;
+
+	req = nvme_allocate_request_user_copy(ctrlr->adminq,
+					      payload, payload_size,
+					      cb_fn, cb_arg, false);
+	if (req == NULL) {
+		return -ENOMEM;
+	}
+
+	cmd = &req->cmd;
+	cmd->opc = SPDK_NVME_OPC_IDENTIFY;
+	cmd->cdw10_bits.identify.cns = cns;
+	cmd->cdw10_bits.identify.cntid = cntid;
+	cmd->nsid = nsid;
+
+	return nvme_ctrlr_submit_admin_request(ctrlr, req);
+}
+
+int
+nvme_ctrlr_cmd_attach_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid,
+			 struct spdk_nvme_ctrlr_list *payload, spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	struct nvme_request			*req;
+	struct spdk_nvme_cmd			*cmd;
+	int					rc;
+
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+	req = nvme_allocate_request_user_copy(ctrlr->adminq,
+					      payload, sizeof(struct spdk_nvme_ctrlr_list),
+					      cb_fn, cb_arg, true);
+	if (req == NULL) {
+		nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+		return -ENOMEM;
+	}
+
+	cmd = &req->cmd;
+	cmd->opc = SPDK_NVME_OPC_NS_ATTACHMENT;
+	cmd->nsid = nsid;
+	cmd->cdw10_bits.ns_attach.sel = SPDK_NVME_NS_CTRLR_ATTACH;
+
+	rc = nvme_ctrlr_submit_admin_request(ctrlr, req);
+
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+	return rc;
+}
+
+int
+nvme_ctrlr_cmd_detach_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid,
+			 struct spdk_nvme_ctrlr_list *payload, spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	struct nvme_request			*req;
+	struct spdk_nvme_cmd			*cmd;
+	int					rc;
+
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+	req = nvme_allocate_request_user_copy(ctrlr->adminq,
+					      payload, sizeof(struct spdk_nvme_ctrlr_list),
+					      cb_fn, cb_arg, true);
+	if (req == NULL) {
+		nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+		return -ENOMEM;
+	}
+
+	cmd = &req->cmd;
+	cmd->opc = SPDK_NVME_OPC_NS_ATTACHMENT;
+	cmd->nsid = nsid;
+	cmd->cdw10_bits.ns_attach.sel = SPDK_NVME_NS_CTRLR_DETACH;
+
+	rc = nvme_ctrlr_submit_admin_request(ctrlr, req);
+
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+	return rc;
+}
+
+int
+nvme_ctrlr_cmd_create_ns(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns_data *payload,
+			 spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	struct nvme_request			*req;
+	struct spdk_nvme_cmd			*cmd;
+	int					rc;
+
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+	req = nvme_allocate_request_user_copy(ctrlr->adminq,
+					      payload, sizeof(struct spdk_nvme_ns_data),
+					      cb_fn, cb_arg, true);
+	if (req == NULL) {
+		nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+		return -ENOMEM;
+	}
+
+	cmd = &req->cmd;
+	cmd->opc = SPDK_NVME_OPC_NS_MANAGEMENT;
+	cmd->cdw10_bits.ns_manage.sel = SPDK_NVME_NS_MANAGEMENT_CREATE;
+
+	rc = nvme_ctrlr_submit_admin_request(ctrlr, req);
+
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+	return rc;
+}
+
+int
+nvme_ctrlr_cmd_delete_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, spdk_nvme_cmd_cb cb_fn,
+			 void *cb_arg)
+{
+	struct nvme_request			*req;
+	struct spdk_nvme_cmd			*cmd;
+	int					rc;
+
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+	req = nvme_allocate_request_null(ctrlr->adminq, cb_fn, cb_arg);
+	if (req == NULL) {
+		nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+		return -ENOMEM;
+	}
+
+	cmd = &req->cmd;
+	cmd->opc = SPDK_NVME_OPC_NS_MANAGEMENT;
+	cmd->cdw10_bits.ns_manage.sel = SPDK_NVME_NS_MANAGEMENT_DELETE;
+	cmd->nsid = nsid;
+
+	rc = nvme_ctrlr_submit_admin_request(ctrlr, req);
+
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+	return rc;
+}
+
+int
+nvme_ctrlr_cmd_doorbell_buffer_config(struct spdk_nvme_ctrlr *ctrlr, uint64_t prp1, uint64_t prp2,
+				      spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	struct nvme_request			*req;
+	struct spdk_nvme_cmd			*cmd;
+	int					rc;
+
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+	req = nvme_allocate_request_null(ctrlr->adminq, cb_fn, cb_arg);
+	if (req == NULL) {
+		nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+		return -ENOMEM;
+	}
+
+	cmd = &req->cmd;
+	cmd->opc = SPDK_NVME_OPC_DOORBELL_BUFFER_CONFIG;
+	cmd->dptr.prp.prp1 = prp1;
+	cmd->dptr.prp.prp2 = prp2;
+
+	rc = nvme_ctrlr_submit_admin_request(ctrlr, req);
+
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+	return rc;
+}
+
+int
+nvme_ctrlr_cmd_format(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, struct spdk_nvme_format *format,
+		      spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	struct nvme_request *req;
+	struct spdk_nvme_cmd *cmd;
+	int rc;
+
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+	req = nvme_allocate_request_null(ctrlr->adminq, cb_fn, cb_arg);
+	if (req == NULL) {
+		nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+		return -ENOMEM;
+	}
+
+	cmd = &req->cmd;
+	cmd->opc = SPDK_NVME_OPC_FORMAT_NVM;
+	cmd->nsid = nsid;
+	memcpy(&cmd->cdw10, format, sizeof(uint32_t));
+
+	rc = nvme_ctrlr_submit_admin_request(ctrlr, req);
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+
+	return rc;
+}
+
+int
+spdk_nvme_ctrlr_cmd_set_feature(struct spdk_nvme_ctrlr *ctrlr, uint8_t feature,
+				uint32_t cdw11, uint32_t cdw12, void *payload, uint32_t payload_size,
+				spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	struct nvme_request *req;
+	struct spdk_nvme_cmd *cmd;
+	int rc;
+
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+	req = nvme_allocate_request_user_copy(ctrlr->adminq, payload, payload_size, cb_fn, cb_arg,
+					      true);
+	if (req == NULL) {
+		nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+		return -ENOMEM;
+	}
+
+	cmd = &req->cmd;
+	cmd->opc = SPDK_NVME_OPC_SET_FEATURES;
+	cmd->cdw10_bits.set_features.fid = feature;
+	cmd->cdw11 = cdw11;
+	cmd->cdw12 = cdw12;
+
+	rc = nvme_ctrlr_submit_admin_request(ctrlr, req);
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+
+	return rc;
+}
+
+int
+spdk_nvme_ctrlr_cmd_get_feature(struct spdk_nvme_ctrlr *ctrlr, uint8_t feature,
+				uint32_t cdw11, void *payload, uint32_t payload_size,
+				spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	struct nvme_request *req;
+	struct spdk_nvme_cmd *cmd;
+	int rc;
+
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+	req = nvme_allocate_request_user_copy(ctrlr->adminq, payload, payload_size, cb_fn, cb_arg,
+					      false);
+	if (req == NULL) {
+		nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+		return -ENOMEM;
+	}
+
+	cmd = &req->cmd;
+	cmd->opc = SPDK_NVME_OPC_GET_FEATURES;
+	cmd->cdw10_bits.get_features.fid = feature;
+	cmd->cdw11 = cdw11;
+
+	rc = nvme_ctrlr_submit_admin_request(ctrlr, req);
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+
+	return rc;
+}
+
+int
+spdk_nvme_ctrlr_cmd_get_feature_ns(struct spdk_nvme_ctrlr *ctrlr, uint8_t feature,
+				   uint32_t cdw11, void *payload,
+				   uint32_t payload_size, spdk_nvme_cmd_cb cb_fn,
+				   void *cb_arg, uint32_t ns_id)
+{
+	struct nvme_request *req;
+	struct spdk_nvme_cmd *cmd;
+	int rc;
+
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+	req = nvme_allocate_request_user_copy(ctrlr->adminq, payload, payload_size, cb_fn, cb_arg,
+					      false);
+	if (req == NULL) {
+		nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+		return -ENOMEM;
+	}
+
+	cmd = &req->cmd;
+	cmd->opc = SPDK_NVME_OPC_GET_FEATURES;
+	cmd->cdw10_bits.get_features.fid = feature;
+	cmd->cdw11 = cdw11;
+	cmd->nsid = ns_id;
+
+	rc = nvme_ctrlr_submit_admin_request(ctrlr, req);
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+
+	return rc;
+}
+
+int spdk_nvme_ctrlr_cmd_set_feature_ns(struct spdk_nvme_ctrlr *ctrlr, uint8_t feature,
+				       uint32_t cdw11, uint32_t cdw12, void *payload,
+				       uint32_t payload_size, spdk_nvme_cmd_cb cb_fn,
+				       void *cb_arg, uint32_t ns_id)
+{
+	struct nvme_request *req;
+	struct spdk_nvme_cmd *cmd;
+	int rc;
+
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+	req = nvme_allocate_request_user_copy(ctrlr->adminq, payload, payload_size, cb_fn, cb_arg,
+					      true);
+	if (req == NULL) {
+		nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+		return -ENOMEM;
+	}
+
+	cmd = &req->cmd;
+	cmd->opc = SPDK_NVME_OPC_SET_FEATURES;
+	cmd->cdw10_bits.set_features.fid = feature;
+	cmd->cdw11 = cdw11;
+	cmd->cdw12 = cdw12;
+	cmd->nsid = ns_id;
+
+	rc = nvme_ctrlr_submit_admin_request(ctrlr, req);
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+
+	return rc;
+}
+
+int
+nvme_ctrlr_cmd_set_num_queues(struct spdk_nvme_ctrlr *ctrlr,
+			      uint32_t num_queues, spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	union spdk_nvme_feat_number_of_queues feat_num_queues;
+
+	feat_num_queues.raw = 0;
+	feat_num_queues.bits.nsqr = num_queues - 1;
+	feat_num_queues.bits.ncqr = num_queues - 1;
+
+	return spdk_nvme_ctrlr_cmd_set_feature(ctrlr, SPDK_NVME_FEAT_NUMBER_OF_QUEUES, feat_num_queues.raw,
+					       0,
+					       NULL, 0, cb_fn, cb_arg);
+}
+
+int
+nvme_ctrlr_cmd_get_num_queues(struct spdk_nvme_ctrlr *ctrlr,
+			      spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	return spdk_nvme_ctrlr_cmd_get_feature(ctrlr, SPDK_NVME_FEAT_NUMBER_OF_QUEUES, 0, NULL, 0,
+					       cb_fn, cb_arg);
+}
+
+int
+nvme_ctrlr_cmd_set_async_event_config(struct spdk_nvme_ctrlr *ctrlr,
+				      union spdk_nvme_feat_async_event_configuration config, spdk_nvme_cmd_cb cb_fn,
+				      void *cb_arg)
+{
+	uint32_t cdw11;
+
+	cdw11 = config.raw;
+	return spdk_nvme_ctrlr_cmd_set_feature(ctrlr, SPDK_NVME_FEAT_ASYNC_EVENT_CONFIGURATION, cdw11, 0,
+					       NULL, 0,
+					       cb_fn, cb_arg);
+}
+
+int
+nvme_ctrlr_cmd_set_host_id(struct spdk_nvme_ctrlr *ctrlr, void *host_id, uint32_t host_id_size,
+			   spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	union spdk_nvme_feat_host_identifier feat_host_identifier;
+
+	feat_host_identifier.raw = 0;
+	if (host_id_size == 16) {
+		/* 128-bit extended host identifier */
+		feat_host_identifier.bits.exhid = 1;
+	} else if (host_id_size == 8) {
+		/* 64-bit host identifier */
+		feat_host_identifier.bits.exhid = 0;
+	} else {
+		SPDK_ERRLOG("Invalid host ID size %u\n", host_id_size);
+		return -EINVAL;
+	}
+
+	return spdk_nvme_ctrlr_cmd_set_feature(ctrlr, SPDK_NVME_FEAT_HOST_IDENTIFIER,
+					       feat_host_identifier.raw, 0,
+					       host_id, host_id_size, cb_fn, cb_arg);
+}
+
+int
+spdk_nvme_ctrlr_cmd_get_log_page_ext(struct spdk_nvme_ctrlr *ctrlr, uint8_t log_page,
+				     uint32_t nsid, void *payload, uint32_t payload_size,
+				     uint64_t offset, uint32_t cdw10,
+				     uint32_t cdw11, uint32_t cdw14,
+				     spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	struct nvme_request *req;
+	struct spdk_nvme_cmd *cmd;
+	uint32_t numd, numdl, numdu;
+	uint32_t lpol, lpou;
+	int rc;
+
+	if (payload_size == 0) {
+		return -EINVAL;
+	}
+
+	if (offset & 3) {
+		return -EINVAL;
+	}
+
+	numd = payload_size / sizeof(uint32_t) - 1u;
+	numdl = numd & 0xFFFFu;
+	numdu = (numd >> 16) & 0xFFFFu;
+
+	lpol = (uint32_t)offset;
+	lpou = (uint32_t)(offset >> 32);
+
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+
+	if (offset && !ctrlr->cdata.lpa.edlp) {
+		nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+		return -EINVAL;
+	}
+
+	req = nvme_allocate_request_user_copy(ctrlr->adminq,
+					      payload, payload_size, cb_fn, cb_arg, false);
+	if (req == NULL) {
+		nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+		return -ENOMEM;
+	}
+
+	cmd = &req->cmd;
+	cmd->opc = SPDK_NVME_OPC_GET_LOG_PAGE;
+	cmd->nsid = nsid;
+	cmd->cdw10 = cdw10;
+	cmd->cdw10_bits.get_log_page.numdl = numdl;
+	cmd->cdw10_bits.get_log_page.lid = log_page;
+
+	cmd->cdw11 = cdw11;
+	cmd->cdw11_bits.get_log_page.numdu = numdu;
+	cmd->cdw12 = lpol;
+	cmd->cdw13 = lpou;
+	cmd->cdw14 = cdw14;
+
+	rc = nvme_ctrlr_submit_admin_request(ctrlr, req);
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+
+	return rc;
+}
+
+int
+spdk_nvme_ctrlr_cmd_get_log_page(struct spdk_nvme_ctrlr *ctrlr, uint8_t log_page,
+				 uint32_t nsid, void *payload, uint32_t payload_size,
+				 uint64_t offset, spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	return spdk_nvme_ctrlr_cmd_get_log_page_ext(ctrlr, log_page, nsid, payload,
+			payload_size, offset, 0, 0, 0, cb_fn, cb_arg);
+}
+
+static void
+nvme_ctrlr_retry_queued_abort(struct spdk_nvme_ctrlr *ctrlr)
+{
+	struct nvme_request	*next, *tmp;
+	int rc;
+
+	if (ctrlr->is_resetting || ctrlr->is_destructed) {
+		return;
+	}
+
+	STAILQ_FOREACH_SAFE(next, &ctrlr->queued_aborts, stailq, tmp) {
+		STAILQ_REMOVE_HEAD(&ctrlr->queued_aborts, stailq);
+		ctrlr->outstanding_aborts++;
+		rc = nvme_ctrlr_submit_admin_request(ctrlr, next);
+		if (rc < 0) {
+			SPDK_ERRLOG("Failed to submit queued abort.\n");
+			memset(&next->cpl, 0, sizeof(next->cpl));
+			next->cpl.status.sct = SPDK_NVME_SCT_GENERIC;
+			next->cpl.status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
+			next->cpl.status.dnr = 1;
+			nvme_complete_request(next->cb_fn, next->cb_arg, next->qpair, next, &next->cpl);
+			nvme_free_request(next);
+		} else {
+			/* If the first abort succeeds, stop iterating. */
+			break;
+		}
+	}
+}
+
+static int
+_nvme_ctrlr_submit_abort_request(struct spdk_nvme_ctrlr *ctrlr,
+				 struct nvme_request *req)
+{
+	/* ACL is a 0's based value. */
+	if (ctrlr->outstanding_aborts >= ctrlr->cdata.acl + 1U) {
+		STAILQ_INSERT_TAIL(&ctrlr->queued_aborts, req, stailq);
+		return 0;
+	} else {
+		ctrlr->outstanding_aborts++;
+		return nvme_ctrlr_submit_admin_request(ctrlr, req);
+	}
+}
+
+static void
+nvme_ctrlr_cmd_abort_cpl(void *ctx, const struct spdk_nvme_cpl *cpl)
+{
+	struct nvme_request	*req = ctx;
+	struct spdk_nvme_ctrlr	*ctrlr;
+
+	ctrlr = req->qpair->ctrlr;
+
+	ctrlr->outstanding_aborts--;
+	nvme_ctrlr_retry_queued_abort(ctrlr);
+
+	req->user_cb_fn(req->user_cb_arg, cpl);
+}
+
+int
+spdk_nvme_ctrlr_cmd_abort(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair,
+			  uint16_t cid, spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	int rc;
+	struct nvme_request *req;
+	struct spdk_nvme_cmd *cmd;
+
+	if (qpair == NULL) {
+		qpair = ctrlr->adminq;
+	}
+
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+	req = nvme_allocate_request_null(ctrlr->adminq, nvme_ctrlr_cmd_abort_cpl, NULL);
+	if (req == NULL) {
+		nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+		return -ENOMEM;
+	}
+	req->cb_arg = req;
+	req->user_cb_fn = cb_fn;
+	req->user_cb_arg = cb_arg;
+
+	cmd = &req->cmd;
+	cmd->opc = SPDK_NVME_OPC_ABORT;
+	cmd->cdw10_bits.abort.sqid = qpair->id;
+	cmd->cdw10_bits.abort.cid = cid;
+
+	rc = _nvme_ctrlr_submit_abort_request(ctrlr, req);
+
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+	return rc;
+}
+
+static void
+nvme_complete_abort_request(void *ctx, const struct spdk_nvme_cpl *cpl)
+{
+	struct nvme_request *req = ctx;
+	struct nvme_request *parent = req->parent;
+	struct spdk_nvme_ctrlr *ctrlr;
+
+	ctrlr = req->qpair->ctrlr;
+
+	ctrlr->outstanding_aborts--;
+	nvme_ctrlr_retry_queued_abort(ctrlr);
+
+	nvme_request_remove_child(parent, req);
+
+	if (!spdk_nvme_cpl_is_abort_success(cpl)) {
+		parent->parent_status.cdw0 |= 1U;
+	}
+
+	if (parent->num_children == 0) {
+		nvme_complete_request(parent->cb_fn, parent->cb_arg, parent->qpair,
+				      parent, &parent->parent_status);
+		nvme_free_request(parent);
+	}
+}
+
+static int
+nvme_request_add_abort(struct nvme_request *req, void *arg)
+{
+	struct nvme_request *parent = arg;
+	struct nvme_request *child;
+	void *cmd_cb_arg;
+
+	cmd_cb_arg = parent->user_cb_arg;
+
+	if (req->cb_arg != cmd_cb_arg &&
+	    (req->parent == NULL || req->parent->cb_arg != cmd_cb_arg)) {
+		return 0;
+	}
+
+	child = nvme_allocate_request_null(parent->qpair->ctrlr->adminq,
+					   nvme_complete_abort_request, NULL);
+	if (child == NULL) {
+		return -ENOMEM;
+	}
+
+	child->cb_arg = child;
+
+	child->cmd.opc = SPDK_NVME_OPC_ABORT;
+	/* Copy SQID from the parent. */
+	child->cmd.cdw10_bits.abort.sqid = parent->cmd.cdw10_bits.abort.sqid;
+	child->cmd.cdw10_bits.abort.cid = req->cmd.cid;
+
+	child->parent = parent;
+
+	TAILQ_INSERT_TAIL(&parent->children, child, child_tailq);
+	parent->num_children++;
+
+	return 0;
+}
+
+int
+spdk_nvme_ctrlr_cmd_abort_ext(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair,
+			      void *cmd_cb_arg,
+			      spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	int rc = 0;
+	struct nvme_request *parent, *child, *tmp;
+	bool child_failed = false;
+	int aborted = 0;
+
+	if (cmd_cb_arg == NULL) {
+		return -EINVAL;
+	}
+
+	pthread_mutex_lock(&ctrlr->ctrlr_lock);
+
+	if (qpair == NULL) {
+		qpair = ctrlr->adminq;
+	}
+
+	parent = nvme_allocate_request_null(ctrlr->adminq, cb_fn, cb_arg);
+	if (parent == NULL) {
+		pthread_mutex_unlock(&ctrlr->ctrlr_lock);
+
+		return -ENOMEM;
+	}
+
+	TAILQ_INIT(&parent->children);
+	parent->num_children = 0;
+
+	parent->cmd.opc = SPDK_NVME_OPC_ABORT;
+	memset(&parent->parent_status, 0, sizeof(struct spdk_nvme_cpl));
+
+	/* Hold SQID that the requests to abort are associated with.
+	 * This will be copied to the children.
+	 *
+	 * CID is not set here because the parent is not submitted directly
+	 * and CID is not determined until request to abort is found.
+	 */
+	parent->cmd.cdw10_bits.abort.sqid = qpair->id;
+
+	/* This is used to find request to abort. */
+	parent->user_cb_arg = cmd_cb_arg;
+
+	/* Add an abort request for each outstanding request which has cmd_cb_arg
+	 * as its callback context.
+	 */
+	rc = nvme_transport_qpair_iterate_requests(qpair, nvme_request_add_abort, parent);
+	if (rc != 0) {
+		/* Free abort requests already added. */
+		child_failed = true;
+	}
+
+	TAILQ_FOREACH_SAFE(child, &parent->children, child_tailq, tmp) {
+		if (spdk_likely(!child_failed)) {
+			rc = _nvme_ctrlr_submit_abort_request(ctrlr, child);
+			if (spdk_unlikely(rc != 0)) {
+				child_failed = true;
+			}
+		} else {
+			/* Free remaining abort requests. */
+			nvme_request_remove_child(parent, child);
+			nvme_free_request(child);
+		}
+	}
+
+	if (spdk_likely(!child_failed)) {
+		/* There is no error so far. Abort requests were submitted successfully
+		 * or there was no outstanding request to abort.
+		 *
+		 * Hence abort queued requests which has cmd_cb_arg as its callback
+		 * context next.
+		 */
+		aborted = nvme_qpair_abort_queued_reqs(qpair, cmd_cb_arg);
+		if (parent->num_children == 0) {
+			/* There was no outstanding request to abort. */
+			if (aborted > 0) {
+				/* The queued requests were successfully aborted. Hence
+				 * complete the parent request with success synchronously.
+				 */
+				nvme_complete_request(parent->cb_fn, parent->cb_arg, parent->qpair,
+						      parent, &parent->parent_status);
+				nvme_free_request(parent);
+			} else {
+				/* There was no queued request to abort. */
+				rc = -ENOENT;
+			}
+		}
+	} else {
+		/* Failed to add or submit abort request. */
+		if (parent->num_children != 0) {
+			/* Return success since we must wait for those children
+			 * to complete but set the parent request to failure.
+			 */
+			parent->parent_status.cdw0 |= 1U;
+			rc = 0;
+		}
+	}
+
+	if (rc != 0) {
+		nvme_free_request(parent);
+	}
+
+	pthread_mutex_unlock(&ctrlr->ctrlr_lock);
+	return rc;
+}
+
+int
+nvme_ctrlr_cmd_fw_commit(struct spdk_nvme_ctrlr *ctrlr,
+			 const struct spdk_nvme_fw_commit *fw_commit,
+			 spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	struct nvme_request *req;
+	struct spdk_nvme_cmd *cmd;
+	int rc;
+
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+	req = nvme_allocate_request_null(ctrlr->adminq, cb_fn, cb_arg);
+	if (req == NULL) {
+		nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+		return -ENOMEM;
+	}
+
+	cmd = &req->cmd;
+	cmd->opc = SPDK_NVME_OPC_FIRMWARE_COMMIT;
+	memcpy(&cmd->cdw10, fw_commit, sizeof(uint32_t));
+
+	rc = nvme_ctrlr_submit_admin_request(ctrlr, req);
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+
+	return rc;
+
+}
+
+int
+nvme_ctrlr_cmd_fw_image_download(struct spdk_nvme_ctrlr *ctrlr,
+				 uint32_t size, uint32_t offset, void *payload,
+				 spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	struct nvme_request *req;
+	struct spdk_nvme_cmd *cmd;
+	int rc;
+
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+	req = nvme_allocate_request_user_copy(ctrlr->adminq, payload, size, cb_fn, cb_arg, true);
+	if (req == NULL) {
+		nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+		return -ENOMEM;
+	}
+
+	cmd = &req->cmd;
+	cmd->opc = SPDK_NVME_OPC_FIRMWARE_IMAGE_DOWNLOAD;
+	cmd->cdw10 = (size >> 2) - 1;
+	cmd->cdw11 = offset >> 2;
+
+	rc = nvme_ctrlr_submit_admin_request(ctrlr, req);
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+
+	return rc;
+}
+
+int
+spdk_nvme_ctrlr_cmd_security_receive(struct spdk_nvme_ctrlr *ctrlr, uint8_t secp,
+				     uint16_t spsp, uint8_t nssf, void *payload,
+				     uint32_t payload_size, spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	struct nvme_request *req;
+	struct spdk_nvme_cmd *cmd;
+	int rc;
+
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+	req = nvme_allocate_request_user_copy(ctrlr->adminq, payload, payload_size,
+					      cb_fn, cb_arg, false);
+	if (req == NULL) {
+		nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+		return -ENOMEM;
+	}
+
+	cmd = &req->cmd;
+	cmd->opc = SPDK_NVME_OPC_SECURITY_RECEIVE;
+	cmd->cdw10_bits.sec_send_recv.nssf = nssf;
+	cmd->cdw10_bits.sec_send_recv.spsp0 = (uint8_t)spsp;
+	cmd->cdw10_bits.sec_send_recv.spsp1 = (uint8_t)(spsp >> 8);
+	cmd->cdw10_bits.sec_send_recv.secp = secp;
+	cmd->cdw11 = payload_size;
+
+	rc = nvme_ctrlr_submit_admin_request(ctrlr, req);
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+
+	return rc;
+}
+
+int
+spdk_nvme_ctrlr_cmd_security_send(struct spdk_nvme_ctrlr *ctrlr, uint8_t secp,
+				  uint16_t spsp, uint8_t nssf, void *payload,
+				  uint32_t payload_size, spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	struct nvme_request *req;
+	struct spdk_nvme_cmd *cmd;
+	int rc;
+
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+	req = nvme_allocate_request_user_copy(ctrlr->adminq, payload, payload_size,
+					      cb_fn, cb_arg, true);
+	if (req == NULL) {
+		nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+		return -ENOMEM;
+	}
+
+	cmd = &req->cmd;
+	cmd->opc = SPDK_NVME_OPC_SECURITY_SEND;
+	cmd->cdw10_bits.sec_send_recv.nssf = nssf;
+	cmd->cdw10_bits.sec_send_recv.spsp0 = (uint8_t)spsp;
+	cmd->cdw10_bits.sec_send_recv.spsp1 = (uint8_t)(spsp >> 8);
+	cmd->cdw10_bits.sec_send_recv.secp = secp;
+	cmd->cdw11 = payload_size;
+
+	rc = nvme_ctrlr_submit_admin_request(ctrlr, req);
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+
+	return rc;
+}
+
+int
+nvme_ctrlr_cmd_sanitize(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid,
+			struct spdk_nvme_sanitize *sanitize, uint32_t cdw11,
+			spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	struct nvme_request *req;
+	struct spdk_nvme_cmd *cmd;
+	int rc;
+
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+	req = nvme_allocate_request_null(ctrlr->adminq, cb_fn, cb_arg);
+	if (req == NULL) {
+		nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+		return -ENOMEM;
+	}
+
+	cmd = &req->cmd;
+	cmd->opc = SPDK_NVME_OPC_SANITIZE;
+	cmd->nsid = nsid;
+	cmd->cdw11 = cdw11;
+	memcpy(&cmd->cdw10, sanitize, sizeof(cmd->cdw10));
+
+	rc = nvme_ctrlr_submit_admin_request(ctrlr, req);
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+
+	return rc;
+}
diff --git a/src/spdk/lib/nvme/nvme_ctrlr_ocssd_cmd.c b/src/spdk/lib/nvme/nvme_ctrlr_ocssd_cmd.c
new file mode 100644
index 000000000..2eba219ce
--- /dev/null
+++ b/src/spdk/lib/nvme/nvme_ctrlr_ocssd_cmd.c
@@ -0,0 +1,88 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/nvme_ocssd.h"
+#include "nvme_internal.h"
+
+bool
+spdk_nvme_ctrlr_is_ocssd_supported(struct spdk_nvme_ctrlr *ctrlr)
+{
+	if (ctrlr->quirks & NVME_QUIRK_OCSSD) {
+		/* TODO: There isn't a standardized way to identify Open-Channel SSD
+		 * different verdors may have different conditions.
+		 */
+
+		/*
+		 * Current QEMU OpenChannel Device needs to check nsdata->vs[0].
+		 * Here check nsdata->vs[0] of the first namespace.
+		 */
+		if (ctrlr->cdata.vid == SPDK_PCI_VID_CNEXLABS) {
+			if (ctrlr->num_ns && ctrlr->nsdata[0].vendor_specific[0] == 0x1) {
+				return true;
+			}
+		}
+	}
+	return false;
+}
+
+
+int
+spdk_nvme_ocssd_ctrlr_cmd_geometry(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid,
+				   void *payload, uint32_t payload_size,
+				   spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	struct nvme_request *req;
+	struct spdk_nvme_cmd *cmd;
+	int rc;
+
+	if (!payload || (payload_size != sizeof(struct spdk_ocssd_geometry_data))) {
+		return -EINVAL;
+	}
+
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+	req = nvme_allocate_request_user_copy(ctrlr->adminq,
+					      payload, payload_size, cb_fn, cb_arg, false);
+	if (req == NULL) {
+		nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+		return -ENOMEM;
+	}
+
+	cmd = &req->cmd;
+	cmd->opc = SPDK_OCSSD_OPC_GEOMETRY;
+	cmd->nsid = nsid;
+
+	rc = nvme_ctrlr_submit_admin_request(ctrlr, req);
+
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+	return rc;
+}
diff --git a/src/spdk/lib/nvme/nvme_cuse.c b/src/spdk/lib/nvme/nvme_cuse.c
new file mode 100644
index 000000000..9a5ee1f0d
--- /dev/null
+++ b/src/spdk/lib/nvme/nvme_cuse.c
@@ -0,0 +1,1115 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#define FUSE_USE_VERSION 31
+
+#include <fuse3/cuse_lowlevel.h>
+
+#include <linux/nvme_ioctl.h>
+#include <linux/fs.h>
+
+#include "nvme_internal.h"
+#include "nvme_io_msg.h"
+#include "nvme_cuse.h"
+
+struct cuse_device {
+	bool				is_started;
+
+	char				dev_name[128];
+	uint32_t			index;
+	int				claim_fd;
+	char				lock_name[64];
+
+	struct spdk_nvme_ctrlr		*ctrlr;		/**< NVMe controller */
+	uint32_t			nsid;		/**< NVMe name space id, or 0 */
+
+	pthread_t			tid;
+	struct fuse_session		*session;
+
+	struct cuse_device		*ctrlr_device;
+	struct cuse_device		*ns_devices;	/**< Array of cuse ns devices */
+
+	TAILQ_ENTRY(cuse_device)	tailq;
+};
+
+static pthread_mutex_t g_cuse_mtx = PTHREAD_MUTEX_INITIALIZER;
+static TAILQ_HEAD(, cuse_device) g_ctrlr_ctx_head = TAILQ_HEAD_INITIALIZER(g_ctrlr_ctx_head);
+static struct spdk_bit_array *g_ctrlr_started;
+
+struct cuse_io_ctx {
+	struct spdk_nvme_cmd		nvme_cmd;
+	enum spdk_nvme_data_transfer	data_transfer;
+
+	uint64_t			lba;
+	uint32_t			lba_count;
+
+	void				*data;
+	int				data_len;
+
+	fuse_req_t			req;
+};
+
+static void
+cuse_io_ctx_free(struct cuse_io_ctx *ctx)
+{
+	spdk_free(ctx->data);
+	free(ctx);
+}
+
+#define FUSE_REPLY_CHECK_BUFFER(req, arg, out_bufsz, val)		\
+	if (out_bufsz == 0) {						\
+		struct iovec out_iov;					\
+		out_iov.iov_base = (void *)arg;				\
+		out_iov.iov_len = sizeof(val);				\
+		fuse_reply_ioctl_retry(req, NULL, 0, &out_iov, 1);	\
+		return;							\
+	}
+
+static void
+cuse_nvme_admin_cmd_cb(void *arg, const struct spdk_nvme_cpl *cpl)
+{
+	struct cuse_io_ctx *ctx = arg;
+	struct iovec out_iov[2];
+	struct spdk_nvme_cpl _cpl;
+
+	if (ctx->data_transfer == SPDK_NVME_DATA_HOST_TO_CONTROLLER) {
+		fuse_reply_ioctl_iov(ctx->req, cpl->status.sc, NULL, 0);
+	} else {
+		memcpy(&_cpl, cpl, sizeof(struct spdk_nvme_cpl));
+
+		out_iov[0].iov_base = &_cpl.cdw0;
+		out_iov[0].iov_len = sizeof(_cpl.cdw0);
+
+		if (ctx->data_len > 0) {
+			out_iov[1].iov_base = ctx->data;
+			out_iov[1].iov_len = ctx->data_len;
+			fuse_reply_ioctl_iov(ctx->req, cpl->status.sc, out_iov, 2);
+		} else {
+			fuse_reply_ioctl_iov(ctx->req, cpl->status.sc, out_iov, 1);
+		}
+	}
+
+	cuse_io_ctx_free(ctx);
+}
+
+static void
+cuse_nvme_admin_cmd_execute(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, void *arg)
+{
+	int rc;
+	struct cuse_io_ctx *ctx = arg;
+
+	rc = spdk_nvme_ctrlr_cmd_admin_raw(ctrlr, &ctx->nvme_cmd, ctx->data, ctx->data_len,
+					   cuse_nvme_admin_cmd_cb, (void *)ctx);
+	if (rc < 0) {
+		fuse_reply_err(ctx->req, EINVAL);
+		cuse_io_ctx_free(ctx);
+	}
+}
+
+static void
+cuse_nvme_admin_cmd_send(fuse_req_t req, struct nvme_admin_cmd *admin_cmd,
+			 const void *data)
+{
+	struct cuse_io_ctx *ctx;
+	struct cuse_device *cuse_device = fuse_req_userdata(req);
+	int rv;
+
+	ctx = (struct cuse_io_ctx *)calloc(1, sizeof(struct cuse_io_ctx));
+	if (!ctx) {
+		SPDK_ERRLOG("Cannot allocate memory for cuse_io_ctx\n");
+		fuse_reply_err(req, ENOMEM);
+		return;
+	}
+
+	ctx->req = req;
+	ctx->data_transfer = spdk_nvme_opc_get_data_transfer(admin_cmd->opcode);
+
+	memset(&ctx->nvme_cmd, 0, sizeof(ctx->nvme_cmd));
+	ctx->nvme_cmd.opc = admin_cmd->opcode;
+	ctx->nvme_cmd.nsid = admin_cmd->nsid;
+	ctx->nvme_cmd.cdw10 = admin_cmd->cdw10;
+	ctx->nvme_cmd.cdw11 = admin_cmd->cdw11;
+	ctx->nvme_cmd.cdw12 = admin_cmd->cdw12;
+	ctx->nvme_cmd.cdw13 = admin_cmd->cdw13;
+	ctx->nvme_cmd.cdw14 = admin_cmd->cdw14;
+	ctx->nvme_cmd.cdw15 = admin_cmd->cdw15;
+
+	ctx->data_len = admin_cmd->data_len;
+
+	if (ctx->data_len > 0) {
+		ctx->data = spdk_malloc(ctx->data_len, 0, NULL, SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
+		if (!ctx->data) {
+			SPDK_ERRLOG("Cannot allocate memory for data\n");
+			fuse_reply_err(req, ENOMEM);
+			free(ctx);
+			return;
+		}
+		if (data != NULL) {
+			memcpy(ctx->data, data, ctx->data_len);
+		}
+	}
+
+	rv = nvme_io_msg_send(cuse_device->ctrlr, 0, cuse_nvme_admin_cmd_execute, ctx);
+	if (rv) {
+		SPDK_ERRLOG("Cannot send io msg to the controller\n");
+		fuse_reply_err(req, -rv);
+		cuse_io_ctx_free(ctx);
+		return;
+	}
+}
+
+static void
+cuse_nvme_admin_cmd(fuse_req_t req, int cmd, void *arg,
+		    struct fuse_file_info *fi, unsigned flags,
+		    const void *in_buf, size_t in_bufsz, size_t out_bufsz)
+{
+	struct nvme_admin_cmd *admin_cmd;
+	struct iovec in_iov[2], out_iov[2];
+
+	in_iov[0].iov_base = (void *)arg;
+	in_iov[0].iov_len = sizeof(*admin_cmd);
+	if (in_bufsz == 0) {
+		fuse_reply_ioctl_retry(req, in_iov, 1, NULL, 0);
+		return;
+	}
+
+	admin_cmd = (struct nvme_admin_cmd *)in_buf;
+
+	switch (spdk_nvme_opc_get_data_transfer(admin_cmd->opcode)) {
+	case SPDK_NVME_DATA_NONE:
+		SPDK_ERRLOG("SPDK_NVME_DATA_NONE not implemented\n");
+		fuse_reply_err(req, EINVAL);
+		return;
+	case SPDK_NVME_DATA_HOST_TO_CONTROLLER:
+		if (admin_cmd->addr != 0) {
+			in_iov[1].iov_base = (void *)admin_cmd->addr;
+			in_iov[1].iov_len = admin_cmd->data_len;
+			if (in_bufsz == sizeof(*admin_cmd)) {
+				fuse_reply_ioctl_retry(req, in_iov, 2, NULL, 0);
+				return;
+			}
+			cuse_nvme_admin_cmd_send(req, admin_cmd, in_buf + sizeof(*admin_cmd));
+		} else {
+			cuse_nvme_admin_cmd_send(req, admin_cmd, NULL);
+		}
+		return;
+	case SPDK_NVME_DATA_CONTROLLER_TO_HOST:
+		if (out_bufsz == 0) {
+			out_iov[0].iov_base = &((struct nvme_admin_cmd *)arg)->result;
+			out_iov[0].iov_len = sizeof(uint32_t);
+			if (admin_cmd->data_len > 0) {
+				out_iov[1].iov_base = (void *)admin_cmd->addr;
+				out_iov[1].iov_len = admin_cmd->data_len;
+				fuse_reply_ioctl_retry(req, in_iov, 1, out_iov, 2);
+			} else {
+				fuse_reply_ioctl_retry(req, in_iov, 1, out_iov, 1);
+			}
+			return;
+		}
+
+		cuse_nvme_admin_cmd_send(req, admin_cmd, NULL);
+
+		return;
+	case SPDK_NVME_DATA_BIDIRECTIONAL:
+		fuse_reply_err(req, EINVAL);
+		return;
+	}
+}
+
+static void
+cuse_nvme_reset_execute(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, void *arg)
+{
+	int rc;
+	fuse_req_t req = arg;
+
+	rc = spdk_nvme_ctrlr_reset(ctrlr);
+	if (rc) {
+		fuse_reply_err(req, rc);
+		return;
+	}
+
+	fuse_reply_ioctl_iov(req, 0, NULL, 0);
+}
+
+static void
+cuse_nvme_reset(fuse_req_t req, int cmd, void *arg,
+		struct fuse_file_info *fi, unsigned flags,
+		const void *in_buf, size_t in_bufsz, size_t out_bufsz)
+{
+	int rv;
+	struct cuse_device *cuse_device = fuse_req_userdata(req);
+
+	if (cuse_device->nsid) {
+		SPDK_ERRLOG("Namespace reset not supported\n");
+		fuse_reply_err(req, EINVAL);
+		return;
+	}
+
+	rv = nvme_io_msg_send(cuse_device->ctrlr, cuse_device->nsid, cuse_nvme_reset_execute, (void *)req);
+	if (rv) {
+		SPDK_ERRLOG("Cannot send reset\n");
+		fuse_reply_err(req, EINVAL);
+	}
+}
+
+/*****************************************************************************
+ * Namespace IO requests
+ */
+
+static void
+cuse_nvme_submit_io_write_done(void *ref, const struct spdk_nvme_cpl *cpl)
+{
+	struct cuse_io_ctx *ctx = (struct cuse_io_ctx *)ref;
+
+	fuse_reply_ioctl_iov(ctx->req, cpl->status.sc, NULL, 0);
+
+	cuse_io_ctx_free(ctx);
+}
+
+static void
+cuse_nvme_submit_io_write_cb(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, void *arg)
+{
+	int rc;
+	struct cuse_io_ctx *ctx = arg;
+	struct spdk_nvme_ns *ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid);
+
+	rc = spdk_nvme_ns_cmd_write(ns, ctrlr->external_io_msgs_qpair, ctx->data,
+				    ctx->lba, /* LBA start */
+				    ctx->lba_count, /* number of LBAs */
+				    cuse_nvme_submit_io_write_done, ctx, 0);
+
+	if (rc != 0) {
+		SPDK_ERRLOG("write failed: rc = %d\n", rc);
+		fuse_reply_err(ctx->req, rc);
+		cuse_io_ctx_free(ctx);
+		return;
+	}
+}
+
+static void
+cuse_nvme_submit_io_write(fuse_req_t req, int cmd, void *arg,
+			  struct fuse_file_info *fi, unsigned flags,
+			  const void *in_buf, size_t in_bufsz, size_t out_bufsz)
+{
+	const struct nvme_user_io *user_io = in_buf;
+	struct cuse_io_ctx *ctx;
+	struct spdk_nvme_ns *ns;
+	uint32_t block_size;
+	int rc;
+	struct cuse_device *cuse_device = fuse_req_userdata(req);
+
+	ctx = (struct cuse_io_ctx *)calloc(1, sizeof(struct cuse_io_ctx));
+	if (!ctx) {
+		SPDK_ERRLOG("Cannot allocate memory for context\n");
+		fuse_reply_err(req, ENOMEM);
+		return;
+	}
+
+	ctx->req = req;
+
+	ns = spdk_nvme_ctrlr_get_ns(cuse_device->ctrlr, cuse_device->nsid);
+	block_size = spdk_nvme_ns_get_sector_size(ns);
+
+	ctx->lba = user_io->slba;
+	ctx->lba_count = user_io->nblocks + 1;
+	ctx->data_len = ctx->lba_count * block_size;
+
+	ctx->data = spdk_zmalloc(ctx->data_len, 0x1000, NULL, SPDK_ENV_SOCKET_ID_ANY,
+				 SPDK_MALLOC_DMA);
+	if (ctx->data == NULL) {
+		SPDK_ERRLOG("Write buffer allocation failed\n");
+		fuse_reply_err(ctx->req, ENOMEM);
+		free(ctx);
+		return;
+	}
+
+	memcpy(ctx->data, in_buf + sizeof(*user_io), ctx->data_len);
+
+	rc = nvme_io_msg_send(cuse_device->ctrlr, cuse_device->nsid, cuse_nvme_submit_io_write_cb,
+			      ctx);
+	if (rc < 0) {
+		SPDK_ERRLOG("Cannot send write io\n");
+		fuse_reply_err(ctx->req, rc);
+		cuse_io_ctx_free(ctx);
+	}
+}
+
+static void
+cuse_nvme_submit_io_read_done(void *ref, const struct spdk_nvme_cpl *cpl)
+{
+	struct cuse_io_ctx *ctx = (struct cuse_io_ctx *)ref;
+	struct iovec iov;
+
+	iov.iov_base = ctx->data;
+	iov.iov_len = ctx->data_len;
+
+	fuse_reply_ioctl_iov(ctx->req, cpl->status.sc, &iov, 1);
+
+	cuse_io_ctx_free(ctx);
+}
+
+static void
+cuse_nvme_submit_io_read_cb(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, void *arg)
+{
+	int rc;
+	struct cuse_io_ctx *ctx = arg;
+	struct spdk_nvme_ns *ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid);
+
+	rc = spdk_nvme_ns_cmd_read(ns, ctrlr->external_io_msgs_qpair, ctx->data,
+				   ctx->lba, /* LBA start */
+				   ctx->lba_count, /* number of LBAs */
+				   cuse_nvme_submit_io_read_done, ctx, 0);
+
+	if (rc != 0) {
+		SPDK_ERRLOG("read failed: rc = %d\n", rc);
+		fuse_reply_err(ctx->req, rc);
+		cuse_io_ctx_free(ctx);
+		return;
+	}
+}
+
+static void
+cuse_nvme_submit_io_read(fuse_req_t req, int cmd, void *arg,
+			 struct fuse_file_info *fi, unsigned flags,
+			 const void *in_buf, size_t in_bufsz, size_t out_bufsz)
+{
+	int rc;
+	struct cuse_io_ctx *ctx;
+	const struct nvme_user_io *user_io = in_buf;
+	struct cuse_device *cuse_device = fuse_req_userdata(req);
+	struct spdk_nvme_ns *ns;
+	uint32_t block_size;
+
+	ctx = (struct cuse_io_ctx *)calloc(1, sizeof(struct cuse_io_ctx));
+	if (!ctx) {
+		SPDK_ERRLOG("Cannot allocate memory for context\n");
+		fuse_reply_err(req, ENOMEM);
+		return;
+	}
+
+	ctx->req = req;
+	ctx->lba = user_io->slba;
+	ctx->lba_count = user_io->nblocks;
+
+	ns = spdk_nvme_ctrlr_get_ns(cuse_device->ctrlr, cuse_device->nsid);
+	block_size = spdk_nvme_ns_get_sector_size(ns);
+
+	ctx->data_len = ctx->lba_count * block_size;
+	ctx->data = spdk_zmalloc(ctx->data_len, 0x1000, NULL, SPDK_ENV_SOCKET_ID_ANY,
+				 SPDK_MALLOC_DMA);
+	if (ctx->data == NULL) {
+		SPDK_ERRLOG("Read buffer allocation failed\n");
+		fuse_reply_err(ctx->req, ENOMEM);
+		free(ctx);
+		return;
+	}
+
+	rc = nvme_io_msg_send(cuse_device->ctrlr, cuse_device->nsid, cuse_nvme_submit_io_read_cb, ctx);
+	if (rc < 0) {
+		SPDK_ERRLOG("Cannot send read io\n");
+		fuse_reply_err(ctx->req, rc);
+		cuse_io_ctx_free(ctx);
+	}
+}
+
+
+static void
+cuse_nvme_submit_io(fuse_req_t req, int cmd, void *arg,
+		    struct fuse_file_info *fi, unsigned flags,
+		    const void *in_buf, size_t in_bufsz, size_t out_bufsz)
+{
+	const struct nvme_user_io *user_io;
+	struct iovec in_iov[2], out_iov;
+
+	in_iov[0].iov_base = (void *)arg;
+	in_iov[0].iov_len = sizeof(*user_io);
+	if (in_bufsz == 0) {
+		fuse_reply_ioctl_retry(req, in_iov, 1, NULL, 0);
+		return;
+	}
+
+	user_io = in_buf;
+
+	switch (user_io->opcode) {
+	case SPDK_NVME_OPC_READ:
+		out_iov.iov_base = (void *)user_io->addr;
+		out_iov.iov_len = (user_io->nblocks + 1) * 512;
+		if (out_bufsz == 0) {
+			fuse_reply_ioctl_retry(req, in_iov, 1, &out_iov, 1);
+			return;
+		}
+
+		cuse_nvme_submit_io_read(req, cmd, arg, fi, flags, in_buf,
+					 in_bufsz, out_bufsz);
+		break;
+	case SPDK_NVME_OPC_WRITE:
+		in_iov[1].iov_base = (void *)user_io->addr;
+		in_iov[1].iov_len = (user_io->nblocks + 1) * 512;
+		if (in_bufsz == sizeof(*user_io)) {
+			fuse_reply_ioctl_retry(req, in_iov, 2, NULL, 0);
+			return;
+		}
+
+		cuse_nvme_submit_io_write(req, cmd, arg, fi, flags, in_buf,
+					  in_bufsz, out_bufsz);
+
+		break;
+	default:
+		SPDK_ERRLOG("SUBMIT_IO: opc:%d not valid\n", user_io->opcode);
+		fuse_reply_err(req, EINVAL);
+		return;
+	}
+
+}
+
+/*****************************************************************************
+ * Other namespace IOCTLs
+ */
+static void
+cuse_blkgetsize64(fuse_req_t req, int cmd, void *arg,
+		  struct fuse_file_info *fi, unsigned flags,
+		  const void *in_buf, size_t in_bufsz, size_t out_bufsz)
+{
+	uint64_t size;
+	struct spdk_nvme_ns *ns;
+	struct cuse_device *cuse_device = fuse_req_userdata(req);
+
+	FUSE_REPLY_CHECK_BUFFER(req, arg, out_bufsz, size);
+
+	ns = spdk_nvme_ctrlr_get_ns(cuse_device->ctrlr, cuse_device->nsid);
+	size = spdk_nvme_ns_get_num_sectors(ns);
+	fuse_reply_ioctl(req, 0, &size, sizeof(size));
+}
+
+static void
+cuse_blkpbszget(fuse_req_t req, int cmd, void *arg,
+		struct fuse_file_info *fi, unsigned flags,
+		const void *in_buf, size_t in_bufsz, size_t out_bufsz)
+{
+	int pbsz;
+	struct spdk_nvme_ns *ns;
+	struct cuse_device *cuse_device = fuse_req_userdata(req);
+
+	FUSE_REPLY_CHECK_BUFFER(req, arg, out_bufsz, pbsz);
+
+	ns = spdk_nvme_ctrlr_get_ns(cuse_device->ctrlr, cuse_device->nsid);
+	pbsz = spdk_nvme_ns_get_sector_size(ns);
+	fuse_reply_ioctl(req, 0, &pbsz, sizeof(pbsz));
+}
+
+static void
+cuse_blkgetsize(fuse_req_t req, int cmd, void *arg,
+		struct fuse_file_info *fi, unsigned flags,
+		const void *in_buf, size_t in_bufsz, size_t out_bufsz)
+{
+	long size;
+	struct spdk_nvme_ns *ns;
+	struct cuse_device *cuse_device = fuse_req_userdata(req);
+
+	FUSE_REPLY_CHECK_BUFFER(req, arg, out_bufsz, size);
+
+	ns = spdk_nvme_ctrlr_get_ns(cuse_device->ctrlr, cuse_device->nsid);
+
+	/* return size in 512 bytes blocks */
+	size = spdk_nvme_ns_get_num_sectors(ns) * 512 / spdk_nvme_ns_get_sector_size(ns);
+	fuse_reply_ioctl(req, 0, &size, sizeof(size));
+}
+
+static void
+cuse_getid(fuse_req_t req, int cmd, void *arg,
+	   struct fuse_file_info *fi, unsigned flags,
+	   const void *in_buf, size_t in_bufsz, size_t out_bufsz)
+{
+	struct cuse_device *cuse_device = fuse_req_userdata(req);
+
+	fuse_reply_ioctl(req, cuse_device->nsid, NULL, 0);
+}
+
+static void
+cuse_ctrlr_ioctl(fuse_req_t req, int cmd, void *arg,
+		 struct fuse_file_info *fi, unsigned flags,
+		 const void *in_buf, size_t in_bufsz, size_t out_bufsz)
+{
+	if (flags & FUSE_IOCTL_COMPAT) {
+		fuse_reply_err(req, ENOSYS);
+		return;
+	}
+
+	switch (cmd) {
+	case NVME_IOCTL_ADMIN_CMD:
+		cuse_nvme_admin_cmd(req, cmd, arg, fi, flags, in_buf, in_bufsz, out_bufsz);
+		break;
+
+	case NVME_IOCTL_RESET:
+		cuse_nvme_reset(req, cmd, arg, fi, flags, in_buf, in_bufsz, out_bufsz);
+		break;
+
+	default:
+		SPDK_ERRLOG("Unsupported IOCTL 0x%X.\n", cmd);
+		fuse_reply_err(req, EINVAL);
+	}
+}
+
+static void
+cuse_ns_ioctl(fuse_req_t req, int cmd, void *arg,
+	      struct fuse_file_info *fi, unsigned flags,
+	      const void *in_buf, size_t in_bufsz, size_t out_bufsz)
+{
+	if (flags & FUSE_IOCTL_COMPAT) {
+		fuse_reply_err(req, ENOSYS);
+		return;
+	}
+
+	switch (cmd) {
+	case NVME_IOCTL_ADMIN_CMD:
+		cuse_nvme_admin_cmd(req, cmd, arg, fi, flags, in_buf, in_bufsz, out_bufsz);
+		break;
+
+	case NVME_IOCTL_SUBMIT_IO:
+		cuse_nvme_submit_io(req, cmd, arg, fi, flags, in_buf, in_bufsz, out_bufsz);
+		break;
+
+	case NVME_IOCTL_ID:
+		cuse_getid(req, cmd, arg, fi, flags, in_buf, in_bufsz, out_bufsz);
+		break;
+
+	case BLKPBSZGET:
+		cuse_blkpbszget(req, cmd, arg, fi, flags, in_buf, in_bufsz, out_bufsz);
+		break;
+
+	case BLKGETSIZE:
+		/* Returns the device size as a number of 512-byte blocks (returns pointer to long) */
+		cuse_blkgetsize(req, cmd, arg, fi, flags, in_buf, in_bufsz, out_bufsz);
+		break;
+
+	case BLKGETSIZE64:
+		/* Returns the device size in sectors (returns pointer to uint64_t) */
+		cuse_blkgetsize64(req, cmd, arg, fi, flags, in_buf, in_bufsz, out_bufsz);
+		break;
+
+	default:
+		SPDK_ERRLOG("Unsupported IOCTL 0x%X.\n", cmd);
+		fuse_reply_err(req, EINVAL);
+	}
+}
+
+/*****************************************************************************
+ * CUSE threads initialization.
+ */
+
+static void cuse_open(fuse_req_t req, struct fuse_file_info *fi)
+{
+	fuse_reply_open(req, fi);
+}
+
+static const struct cuse_lowlevel_ops cuse_ctrlr_clop = {
+	.open		= cuse_open,
+	.ioctl		= cuse_ctrlr_ioctl,
+};
+
+static const struct cuse_lowlevel_ops cuse_ns_clop = {
+	.open		= cuse_open,
+	.ioctl		= cuse_ns_ioctl,
+};
+
+static void *
+cuse_thread(void *arg)
+{
+	struct cuse_device *cuse_device = arg;
+	char *cuse_argv[] = { "cuse", "-f" };
+	int cuse_argc = SPDK_COUNTOF(cuse_argv);
+	char devname_arg[128 + 8];
+	const char *dev_info_argv[] = { devname_arg };
+	struct cuse_info ci;
+	int multithreaded;
+	int rc;
+	struct fuse_buf buf = { .mem = NULL };
+	struct pollfd fds;
+	int timeout_msecs = 500;
+
+	spdk_unaffinitize_thread();
+
+	snprintf(devname_arg, sizeof(devname_arg), "DEVNAME=%s", cuse_device->dev_name);
+
+	memset(&ci, 0, sizeof(ci));
+	ci.dev_info_argc = 1;
+	ci.dev_info_argv = dev_info_argv;
+	ci.flags = CUSE_UNRESTRICTED_IOCTL;
+
+	if (cuse_device->nsid) {
+		cuse_device->session = cuse_lowlevel_setup(cuse_argc, cuse_argv, &ci, &cuse_ns_clop,
+				       &multithreaded, cuse_device);
+	} else {
+		cuse_device->session = cuse_lowlevel_setup(cuse_argc, cuse_argv, &ci, &cuse_ctrlr_clop,
+				       &multithreaded, cuse_device);
+	}
+	if (!cuse_device->session) {
+		SPDK_ERRLOG("Cannot create cuse session\n");
+		goto err;
+	}
+
+	SPDK_NOTICELOG("fuse session for device %s created\n", cuse_device->dev_name);
+
+	/* Receive and process fuse requests */
+	fds.fd = fuse_session_fd(cuse_device->session);
+	fds.events = POLLIN;
+	while (!fuse_session_exited(cuse_device->session)) {
+		rc = poll(&fds, 1, timeout_msecs);
+		if (rc <= 0) {
+			continue;
+		}
+		rc = fuse_session_receive_buf(cuse_device->session, &buf);
+		if (rc > 0) {
+			fuse_session_process_buf(cuse_device->session, &buf);
+		}
+	}
+	free(buf.mem);
+	fuse_session_reset(cuse_device->session);
+	cuse_lowlevel_teardown(cuse_device->session);
+err:
+	pthread_exit(NULL);
+}
+
+/*****************************************************************************
+ * CUSE devices management
+ */
+
+static int
+cuse_nvme_ns_start(struct cuse_device *ctrlr_device, uint32_t nsid)
+{
+	struct cuse_device *ns_device;
+	int rv;
+
+	ns_device = &ctrlr_device->ns_devices[nsid - 1];
+	if (ns_device->is_started) {
+		return 0;
+	}
+
+	ns_device->ctrlr = ctrlr_device->ctrlr;
+	ns_device->ctrlr_device = ctrlr_device;
+	ns_device->nsid = nsid;
+	rv = snprintf(ns_device->dev_name, sizeof(ns_device->dev_name), "%sn%d",
+		      ctrlr_device->dev_name, ns_device->nsid);
+	if (rv < 0) {
+		SPDK_ERRLOG("Device name too long.\n");
+		free(ns_device);
+		return -ENAMETOOLONG;
+	}
+
+	rv = pthread_create(&ns_device->tid, NULL, cuse_thread, ns_device);
+	if (rv != 0) {
+		SPDK_ERRLOG("pthread_create failed\n");
+		return -rv;
+	}
+
+	ns_device->is_started = true;
+
+	return 0;
+}
+
+static void
+cuse_nvme_ns_stop(struct cuse_device *ctrlr_device, uint32_t nsid)
+{
+	struct cuse_device *ns_device;
+
+	ns_device = &ctrlr_device->ns_devices[nsid - 1];
+	if (!ns_device->is_started) {
+		return;
+	}
+
+	fuse_session_exit(ns_device->session);
+	pthread_join(ns_device->tid, NULL);
+	ns_device->is_started = false;
+}
+
+static int
+nvme_cuse_claim(struct cuse_device *ctrlr_device, uint32_t index)
+{
+	int dev_fd;
+	int pid;
+	void *dev_map;
+	struct flock cusedev_lock = {
+		.l_type = F_WRLCK,
+		.l_whence = SEEK_SET,
+		.l_start = 0,
+		.l_len = 0,
+	};
+
+	snprintf(ctrlr_device->lock_name, sizeof(ctrlr_device->lock_name),
+		 "/tmp/spdk_nvme_cuse_lock_%" PRIu32, index);
+
+	dev_fd = open(ctrlr_device->lock_name, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR);
+	if (dev_fd == -1) {
+		SPDK_ERRLOG("could not open %s\n", ctrlr_device->lock_name);
+		return -errno;
+	}
+
+	if (ftruncate(dev_fd, sizeof(int)) != 0) {
+		SPDK_ERRLOG("could not truncate %s\n", ctrlr_device->lock_name);
+		close(dev_fd);
+		return -errno;
+	}
+
+	dev_map = mmap(NULL, sizeof(int), PROT_READ | PROT_WRITE,
+		       MAP_SHARED, dev_fd, 0);
+	if (dev_map == MAP_FAILED) {
+		SPDK_ERRLOG("could not mmap dev %s (%d)\n", ctrlr_device->lock_name, errno);
+		close(dev_fd);
+		return -errno;
+	}
+
+	if (fcntl(dev_fd, F_SETLK, &cusedev_lock) != 0) {
+		pid = *(int *)dev_map;
+		SPDK_ERRLOG("Cannot create lock on device %s, probably"
+			    " process %d has claimed it\n", ctrlr_device->lock_name, pid);
+		munmap(dev_map, sizeof(int));
+		close(dev_fd);
+		/* F_SETLK returns unspecified errnos, normalize them */
+		return -EACCES;
+	}
+
+	*(int *)dev_map = (int)getpid();
+	munmap(dev_map, sizeof(int));
+	ctrlr_device->claim_fd = dev_fd;
+	ctrlr_device->index = index;
+	/* Keep dev_fd open to maintain the lock. */
+	return 0;
+}
+
+static void
+nvme_cuse_unclaim(struct cuse_device *ctrlr_device)
+{
+	close(ctrlr_device->claim_fd);
+	ctrlr_device->claim_fd = -1;
+	unlink(ctrlr_device->lock_name);
+}
+
+static void
+cuse_nvme_ctrlr_stop(struct cuse_device *ctrlr_device)
+{
+	uint32_t i;
+	uint32_t num_ns = spdk_nvme_ctrlr_get_num_ns(ctrlr_device->ctrlr);
+
+	for (i = 1; i <= num_ns; i++) {
+		cuse_nvme_ns_stop(ctrlr_device, i);
+	}
+
+	fuse_session_exit(ctrlr_device->session);
+	pthread_join(ctrlr_device->tid, NULL);
+	TAILQ_REMOVE(&g_ctrlr_ctx_head, ctrlr_device, tailq);
+	spdk_bit_array_clear(g_ctrlr_started, ctrlr_device->index);
+	if (spdk_bit_array_count_set(g_ctrlr_started) == 0) {
+		spdk_bit_array_free(&g_ctrlr_started);
+	}
+	nvme_cuse_unclaim(ctrlr_device);
+	free(ctrlr_device->ns_devices);
+	free(ctrlr_device);
+}
+
+static int
+cuse_nvme_ctrlr_update_namespaces(struct cuse_device *ctrlr_device)
+{
+	uint32_t nsid;
+	uint32_t num_ns = spdk_nvme_ctrlr_get_num_ns(ctrlr_device->ctrlr);
+
+	for (nsid = 1; nsid <= num_ns; nsid++) {
+		if (!spdk_nvme_ctrlr_is_active_ns(ctrlr_device->ctrlr, nsid)) {
+			cuse_nvme_ns_stop(ctrlr_device, nsid);
+			continue;
+		}
+
+		if (cuse_nvme_ns_start(ctrlr_device, nsid) < 0) {
+			SPDK_ERRLOG("Cannot start CUSE namespace device.");
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+static int
+nvme_cuse_start(struct spdk_nvme_ctrlr *ctrlr)
+{
+	int rv = 0;
+	struct cuse_device *ctrlr_device;
+	uint32_t num_ns = spdk_nvme_ctrlr_get_num_ns(ctrlr);
+
+	SPDK_NOTICELOG("Creating cuse device for controller\n");
+
+	if (g_ctrlr_started == NULL) {
+		g_ctrlr_started = spdk_bit_array_create(128);
+		if (g_ctrlr_started == NULL) {
+			SPDK_ERRLOG("Cannot create bit array\n");
+			return -ENOMEM;
+		}
+	}
+
+	ctrlr_device = (struct cuse_device *)calloc(1, sizeof(struct cuse_device));
+	if (!ctrlr_device) {
+		SPDK_ERRLOG("Cannot allocate memory for ctrlr_device.");
+		rv = -ENOMEM;
+		goto err2;
+	}
+
+	ctrlr_device->ctrlr = ctrlr;
+
+	/* Check if device already exists, if not increment index until success */
+	ctrlr_device->index = 0;
+	while (1) {
+		ctrlr_device->index = spdk_bit_array_find_first_clear(g_ctrlr_started, ctrlr_device->index);
+		if (ctrlr_device->index == UINT32_MAX) {
+			SPDK_ERRLOG("Too many registered controllers\n");
+			goto err2;
+		}
+
+		if (nvme_cuse_claim(ctrlr_device, ctrlr_device->index) == 0) {
+			break;
+		}
+		ctrlr_device->index++;
+	}
+	spdk_bit_array_set(g_ctrlr_started, ctrlr_device->index);
+	snprintf(ctrlr_device->dev_name, sizeof(ctrlr_device->dev_name), "spdk/nvme%d",
+		 ctrlr_device->index);
+
+	rv = pthread_create(&ctrlr_device->tid, NULL, cuse_thread, ctrlr_device);
+	if (rv != 0) {
+		SPDK_ERRLOG("pthread_create failed\n");
+		rv = -rv;
+		goto err3;
+	}
+	TAILQ_INSERT_TAIL(&g_ctrlr_ctx_head, ctrlr_device, tailq);
+
+	ctrlr_device->ns_devices = (struct cuse_device *)calloc(num_ns, sizeof(struct cuse_device));
+	/* Start all active namespaces */
+	if (cuse_nvme_ctrlr_update_namespaces(ctrlr_device) < 0) {
+		SPDK_ERRLOG("Cannot start CUSE namespace devices.");
+		cuse_nvme_ctrlr_stop(ctrlr_device);
+		rv = -1;
+		goto err3;
+	}
+
+	return 0;
+
+err3:
+	spdk_bit_array_clear(g_ctrlr_started, ctrlr_device->index);
+err2:
+	free(ctrlr_device);
+	if (spdk_bit_array_count_set(g_ctrlr_started) == 0) {
+		spdk_bit_array_free(&g_ctrlr_started);
+	}
+	return rv;
+}
+
+static struct cuse_device *
+nvme_cuse_get_cuse_ctrlr_device(struct spdk_nvme_ctrlr *ctrlr)
+{
+	struct cuse_device *ctrlr_device = NULL;
+
+	TAILQ_FOREACH(ctrlr_device, &g_ctrlr_ctx_head, tailq) {
+		if (ctrlr_device->ctrlr == ctrlr) {
+			break;
+		}
+	}
+
+	return ctrlr_device;
+}
+
+static struct cuse_device *
+nvme_cuse_get_cuse_ns_device(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid)
+{
+	struct cuse_device *ctrlr_device = NULL;
+	uint32_t num_ns = spdk_nvme_ctrlr_get_num_ns(ctrlr);
+
+	if (nsid < 1 || nsid > num_ns) {
+		return NULL;
+	}
+
+	ctrlr_device = nvme_cuse_get_cuse_ctrlr_device(ctrlr);
+	if (!ctrlr_device) {
+		return NULL;
+	}
+
+	if (!ctrlr_device->ns_devices[nsid - 1].is_started) {
+		return NULL;
+	}
+
+	return &ctrlr_device->ns_devices[nsid - 1];
+}
+
+static void
+nvme_cuse_stop(struct spdk_nvme_ctrlr *ctrlr)
+{
+	struct cuse_device *ctrlr_device;
+
+	pthread_mutex_lock(&g_cuse_mtx);
+
+	ctrlr_device = nvme_cuse_get_cuse_ctrlr_device(ctrlr);
+	if (!ctrlr_device) {
+		SPDK_ERRLOG("Cannot find associated CUSE device\n");
+		pthread_mutex_unlock(&g_cuse_mtx);
+		return;
+	}
+
+	cuse_nvme_ctrlr_stop(ctrlr_device);
+
+	pthread_mutex_unlock(&g_cuse_mtx);
+}
+
+static void
+nvme_cuse_update(struct spdk_nvme_ctrlr *ctrlr)
+{
+	struct cuse_device *ctrlr_device;
+
+	pthread_mutex_lock(&g_cuse_mtx);
+
+	ctrlr_device = nvme_cuse_get_cuse_ctrlr_device(ctrlr);
+	if (!ctrlr_device) {
+		pthread_mutex_unlock(&g_cuse_mtx);
+		return;
+	}
+
+	cuse_nvme_ctrlr_update_namespaces(ctrlr_device);
+
+	pthread_mutex_unlock(&g_cuse_mtx);
+}
+
+static struct nvme_io_msg_producer cuse_nvme_io_msg_producer = {
+	.name = "cuse",
+	.stop = nvme_cuse_stop,
+	.update = nvme_cuse_update,
+};
+
+int
+spdk_nvme_cuse_register(struct spdk_nvme_ctrlr *ctrlr)
+{
+	int rc;
+
+	rc = nvme_io_msg_ctrlr_register(ctrlr, &cuse_nvme_io_msg_producer);
+	if (rc) {
+		return rc;
+	}
+
+	pthread_mutex_lock(&g_cuse_mtx);
+
+	rc = nvme_cuse_start(ctrlr);
+	if (rc) {
+		nvme_io_msg_ctrlr_unregister(ctrlr, &cuse_nvme_io_msg_producer);
+	}
+
+	pthread_mutex_unlock(&g_cuse_mtx);
+
+	return rc;
+}
+
+int
+spdk_nvme_cuse_unregister(struct spdk_nvme_ctrlr *ctrlr)
+{
+	struct cuse_device *ctrlr_device;
+
+	pthread_mutex_lock(&g_cuse_mtx);
+
+	ctrlr_device = nvme_cuse_get_cuse_ctrlr_device(ctrlr);
+	if (!ctrlr_device) {
+		SPDK_ERRLOG("Cannot find associated CUSE device\n");
+		pthread_mutex_unlock(&g_cuse_mtx);
+		return -ENODEV;
+	}
+
+	cuse_nvme_ctrlr_stop(ctrlr_device);
+
+	pthread_mutex_unlock(&g_cuse_mtx);
+
+	nvme_io_msg_ctrlr_unregister(ctrlr, &cuse_nvme_io_msg_producer);
+
+	return 0;
+}
+
+void
+spdk_nvme_cuse_update_namespaces(struct spdk_nvme_ctrlr *ctrlr)
+{
+	nvme_cuse_update(ctrlr);
+}
+
+int
+spdk_nvme_cuse_get_ctrlr_name(struct spdk_nvme_ctrlr *ctrlr, char *name, size_t *size)
+{
+	struct cuse_device *ctrlr_device;
+	size_t req_len;
+
+	pthread_mutex_lock(&g_cuse_mtx);
+
+	ctrlr_device = nvme_cuse_get_cuse_ctrlr_device(ctrlr);
+	if (!ctrlr_device) {
+		pthread_mutex_unlock(&g_cuse_mtx);
+		return -ENODEV;
+	}
+
+	req_len = strnlen(ctrlr_device->dev_name, sizeof(ctrlr_device->dev_name));
+	if (*size < req_len) {
+		*size = req_len;
+		pthread_mutex_unlock(&g_cuse_mtx);
+		return -ENOSPC;
+	}
+	snprintf(name, req_len + 1, "%s", ctrlr_device->dev_name);
+
+	pthread_mutex_unlock(&g_cuse_mtx);
+
+	return 0;
+}
+
+int
+spdk_nvme_cuse_get_ns_name(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, char *name, size_t *size)
+{
+	struct cuse_device *ns_device;
+	size_t req_len;
+
+	pthread_mutex_lock(&g_cuse_mtx);
+
+	ns_device = nvme_cuse_get_cuse_ns_device(ctrlr, nsid);
+	if (!ns_device) {
+		pthread_mutex_unlock(&g_cuse_mtx);
+		return -ENODEV;
+	}
+
+	req_len = strnlen(ns_device->dev_name, sizeof(ns_device->dev_name));
+	if (*size < req_len) {
+		*size = req_len;
+		pthread_mutex_unlock(&g_cuse_mtx);
+		return -ENOSPC;
+	}
+	snprintf(name, req_len + 1, "%s", ns_device->dev_name);
+
+	pthread_mutex_unlock(&g_cuse_mtx);
+
+	return 0;
+}
diff --git a/src/spdk/lib/nvme/nvme_cuse.h b/src/spdk/lib/nvme/nvme_cuse.h
new file mode 100644
index 000000000..92b475190
--- /dev/null
+++ b/src/spdk/lib/nvme/nvme_cuse.h
@@ -0,0 +1,42 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __NVME_CUSE_H__
+#define __NVME_CUSE_H__
+
+#include "spdk/nvme.h"
+
+int nvme_cuse_register(struct spdk_nvme_ctrlr *ctrlr, const char *dev_path);
+void nvme_cuse_unregister(struct spdk_nvme_ctrlr *ctrlr);
+
+#endif /* __NVME_CUSE_H__ */
diff --git a/src/spdk/lib/nvme/nvme_fabric.c b/src/spdk/lib/nvme/nvme_fabric.c
new file mode 100644
index 000000000..9fff20873
--- /dev/null
+++ b/src/spdk/lib/nvme/nvme_fabric.c
@@ -0,0 +1,475 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation. All rights reserved.
+ *   Copyright (c) 2020 Mellanox Technologies LTD. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * NVMe over Fabrics transport-independent functions
+ */
+
+#include "nvme_internal.h"
+
+#include "spdk/endian.h"
+#include "spdk/string.h"
+
+static int
+nvme_fabric_prop_set_cmd(struct spdk_nvme_ctrlr *ctrlr,
+			 uint32_t offset, uint8_t size, uint64_t value)
+{
+	struct spdk_nvmf_fabric_prop_set_cmd cmd = {};
+	struct nvme_completion_poll_status *status;
+	int rc;
+
+	assert(size == SPDK_NVMF_PROP_SIZE_4 || size == SPDK_NVMF_PROP_SIZE_8);
+
+	status = calloc(1, sizeof(*status));
+	if (!status) {
+		SPDK_ERRLOG("Failed to allocate status tracker\n");
+		return -ENOMEM;
+	}
+
+	cmd.opcode = SPDK_NVME_OPC_FABRIC;
+	cmd.fctype = SPDK_NVMF_FABRIC_COMMAND_PROPERTY_SET;
+	cmd.ofst = offset;
+	cmd.attrib.size = size;
+	cmd.value.u64 = value;
+
+	rc = spdk_nvme_ctrlr_cmd_admin_raw(ctrlr, (struct spdk_nvme_cmd *)&cmd,
+					   NULL, 0,
+					   nvme_completion_poll_cb, status);
+	if (rc < 0) {
+		free(status);
+		return rc;
+	}
+
+	if (nvme_wait_for_completion(ctrlr->adminq, status)) {
+		if (!status->timed_out) {
+			free(status);
+		}
+		SPDK_ERRLOG("Property Set failed\n");
+		return -1;
+	}
+	free(status);
+
+	return 0;
+}
+
+static int
+nvme_fabric_prop_get_cmd(struct spdk_nvme_ctrlr *ctrlr,
+			 uint32_t offset, uint8_t size, uint64_t *value)
+{
+	struct spdk_nvmf_fabric_prop_set_cmd cmd = {};
+	struct nvme_completion_poll_status *status;
+	struct spdk_nvmf_fabric_prop_get_rsp *response;
+	int rc;
+
+	assert(size == SPDK_NVMF_PROP_SIZE_4 || size == SPDK_NVMF_PROP_SIZE_8);
+
+	status = calloc(1, sizeof(*status));
+	if (!status) {
+		SPDK_ERRLOG("Failed to allocate status tracker\n");
+		return -ENOMEM;
+	}
+
+	cmd.opcode = SPDK_NVME_OPC_FABRIC;
+	cmd.fctype = SPDK_NVMF_FABRIC_COMMAND_PROPERTY_GET;
+	cmd.ofst = offset;
+	cmd.attrib.size = size;
+
+	rc = spdk_nvme_ctrlr_cmd_admin_raw(ctrlr, (struct spdk_nvme_cmd *)&cmd,
+					   NULL, 0, nvme_completion_poll_cb,
+					   status);
+	if (rc < 0) {
+		free(status);
+		return rc;
+	}
+
+	if (nvme_wait_for_completion(ctrlr->adminq, status)) {
+		if (!status->timed_out) {
+			free(status);
+		}
+		SPDK_ERRLOG("Property Get failed\n");
+		return -1;
+	}
+
+	response = (struct spdk_nvmf_fabric_prop_get_rsp *)&status->cpl;
+
+	if (size == SPDK_NVMF_PROP_SIZE_4) {
+		*value = response->value.u32.low;
+	} else {
+		*value = response->value.u64;
+	}
+
+	free(status);
+
+	return 0;
+}
+
+int
+nvme_fabric_ctrlr_set_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t value)
+{
+	return nvme_fabric_prop_set_cmd(ctrlr, offset, SPDK_NVMF_PROP_SIZE_4, value);
+}
+
+int
+nvme_fabric_ctrlr_set_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t value)
+{
+	return nvme_fabric_prop_set_cmd(ctrlr, offset, SPDK_NVMF_PROP_SIZE_8, value);
+}
+
+int
+nvme_fabric_ctrlr_get_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t *value)
+{
+	uint64_t tmp_value;
+	int rc;
+	rc = nvme_fabric_prop_get_cmd(ctrlr, offset, SPDK_NVMF_PROP_SIZE_4, &tmp_value);
+
+	if (!rc) {
+		*value = (uint32_t)tmp_value;
+	}
+	return rc;
+}
+
+int
+nvme_fabric_ctrlr_get_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t *value)
+{
+	return nvme_fabric_prop_get_cmd(ctrlr, offset, SPDK_NVMF_PROP_SIZE_8, value);
+}
+
+static void
+nvme_fabric_discover_probe(struct spdk_nvmf_discovery_log_page_entry *entry,
+			   struct spdk_nvme_probe_ctx *probe_ctx,
+			   int discover_priority)
+{
+	struct spdk_nvme_transport_id trid;
+	uint8_t *end;
+	size_t len;
+
+	memset(&trid, 0, sizeof(trid));
+
+	if (entry->subtype == SPDK_NVMF_SUBTYPE_DISCOVERY) {
+		SPDK_WARNLOG("Skipping unsupported discovery service referral\n");
+		return;
+	} else if (entry->subtype != SPDK_NVMF_SUBTYPE_NVME) {
+		SPDK_WARNLOG("Skipping unknown subtype %u\n", entry->subtype);
+		return;
+	}
+
+	trid.trtype = entry->trtype;
+	spdk_nvme_transport_id_populate_trstring(&trid, spdk_nvme_transport_id_trtype_str(entry->trtype));
+	if (!spdk_nvme_transport_available_by_name(trid.trstring)) {
+		SPDK_WARNLOG("NVMe transport type %u not available; skipping probe\n",
+			     trid.trtype);
+		return;
+	}
+
+	snprintf(trid.trstring, sizeof(trid.trstring), "%s", probe_ctx->trid.trstring);
+	trid.adrfam = entry->adrfam;
+
+	/* Ensure that subnqn is null terminated. */
+	end = memchr(entry->subnqn, '\0', SPDK_NVMF_NQN_MAX_LEN + 1);
+	if (!end) {
+		SPDK_ERRLOG("Discovery entry SUBNQN is not null terminated\n");
+		return;
+	}
+	len = end - entry->subnqn;
+	memcpy(trid.subnqn, entry->subnqn, len);
+	trid.subnqn[len] = '\0';
+
+	/* Convert traddr to a null terminated string. */
+	len = spdk_strlen_pad(entry->traddr, sizeof(entry->traddr), ' ');
+	memcpy(trid.traddr, entry->traddr, len);
+	if (spdk_str_chomp(trid.traddr) != 0) {
+		SPDK_DEBUGLOG(SPDK_LOG_NVME, "Trailing newlines removed from discovery TRADDR\n");
+	}
+
+	/* Convert trsvcid to a null terminated string. */
+	len = spdk_strlen_pad(entry->trsvcid, sizeof(entry->trsvcid), ' ');
+	memcpy(trid.trsvcid, entry->trsvcid, len);
+	if (spdk_str_chomp(trid.trsvcid) != 0) {
+		SPDK_DEBUGLOG(SPDK_LOG_NVME, "Trailing newlines removed from discovery TRSVCID\n");
+	}
+
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "subnqn=%s, trtype=%u, traddr=%s, trsvcid=%s\n",
+		      trid.subnqn, trid.trtype,
+		      trid.traddr, trid.trsvcid);
+
+	/* Copy the priority from the discovery ctrlr */
+	trid.priority = discover_priority;
+
+	nvme_ctrlr_probe(&trid, probe_ctx, NULL);
+}
+
+static int
+nvme_fabric_get_discovery_log_page(struct spdk_nvme_ctrlr *ctrlr,
+				   void *log_page, uint32_t size, uint64_t offset)
+{
+	struct nvme_completion_poll_status *status;
+	int rc;
+
+	status = calloc(1, sizeof(*status));
+	if (!status) {
+		SPDK_ERRLOG("Failed to allocate status tracker\n");
+		return -ENOMEM;
+	}
+
+	rc = spdk_nvme_ctrlr_cmd_get_log_page(ctrlr, SPDK_NVME_LOG_DISCOVERY, 0, log_page, size, offset,
+					      nvme_completion_poll_cb, status);
+	if (rc < 0) {
+		free(status);
+		return -1;
+	}
+
+	if (nvme_wait_for_completion(ctrlr->adminq, status)) {
+		if (!status->timed_out) {
+			free(status);
+		}
+		return -1;
+	}
+	free(status);
+
+	return 0;
+}
+
+int
+nvme_fabric_ctrlr_scan(struct spdk_nvme_probe_ctx *probe_ctx,
+		       bool direct_connect)
+{
+	struct spdk_nvme_ctrlr_opts discovery_opts;
+	struct spdk_nvme_ctrlr *discovery_ctrlr;
+	union spdk_nvme_cc_register cc;
+	int rc;
+	struct nvme_completion_poll_status *status;
+
+	if (strcmp(probe_ctx->trid.subnqn, SPDK_NVMF_DISCOVERY_NQN) != 0) {
+		/* It is not a discovery_ctrlr info and try to directly connect it */
+		rc = nvme_ctrlr_probe(&probe_ctx->trid, probe_ctx, NULL);
+		return rc;
+	}
+
+	spdk_nvme_ctrlr_get_default_ctrlr_opts(&discovery_opts, sizeof(discovery_opts));
+	/* For discovery_ctrlr set the timeout to 0 */
+	discovery_opts.keep_alive_timeout_ms = 0;
+
+	discovery_ctrlr = nvme_transport_ctrlr_construct(&probe_ctx->trid, &discovery_opts, NULL);
+	if (discovery_ctrlr == NULL) {
+		return -1;
+	}
+	nvme_qpair_set_state(discovery_ctrlr->adminq, NVME_QPAIR_ENABLED);
+
+	/* TODO: this should be using the normal NVMe controller initialization process +1 */
+	cc.raw = 0;
+	cc.bits.en = 1;
+	cc.bits.iosqes = 6; /* SQ entry size == 64 == 2^6 */
+	cc.bits.iocqes = 4; /* CQ entry size == 16 == 2^4 */
+	rc = nvme_transport_ctrlr_set_reg_4(discovery_ctrlr, offsetof(struct spdk_nvme_registers, cc.raw),
+					    cc.raw);
+	if (rc < 0) {
+		SPDK_ERRLOG("Failed to set cc\n");
+		nvme_ctrlr_destruct(discovery_ctrlr);
+		return -1;
+	}
+
+	status = calloc(1, sizeof(*status));
+	if (!status) {
+		SPDK_ERRLOG("Failed to allocate status tracker\n");
+		nvme_ctrlr_destruct(discovery_ctrlr);
+		return -ENOMEM;
+	}
+
+	/* get the cdata info */
+	rc = nvme_ctrlr_cmd_identify(discovery_ctrlr, SPDK_NVME_IDENTIFY_CTRLR, 0, 0,
+				     &discovery_ctrlr->cdata, sizeof(discovery_ctrlr->cdata),
+				     nvme_completion_poll_cb, status);
+	if (rc != 0) {
+		SPDK_ERRLOG("Failed to identify cdata\n");
+		nvme_ctrlr_destruct(discovery_ctrlr);
+		free(status);
+		return rc;
+	}
+
+	if (nvme_wait_for_completion(discovery_ctrlr->adminq, status)) {
+		SPDK_ERRLOG("nvme_identify_controller failed!\n");
+		nvme_ctrlr_destruct(discovery_ctrlr);
+		if (!status->timed_out) {
+			free(status);
+		}
+		return -ENXIO;
+	}
+
+	free(status);
+
+	/* Direct attach through spdk_nvme_connect() API */
+	if (direct_connect == true) {
+		/* Set the ready state to skip the normal init process */
+		discovery_ctrlr->state = NVME_CTRLR_STATE_READY;
+		nvme_ctrlr_connected(probe_ctx, discovery_ctrlr);
+		nvme_ctrlr_add_process(discovery_ctrlr, 0);
+		return 0;
+	}
+
+	rc = nvme_fabric_ctrlr_discover(discovery_ctrlr, probe_ctx);
+	nvme_ctrlr_destruct(discovery_ctrlr);
+	return rc;
+}
+
+int
+nvme_fabric_ctrlr_discover(struct spdk_nvme_ctrlr *ctrlr,
+			   struct spdk_nvme_probe_ctx *probe_ctx)
+{
+	struct spdk_nvmf_discovery_log_page *log_page;
+	struct spdk_nvmf_discovery_log_page_entry *log_page_entry;
+	char buffer[4096];
+	int rc;
+	uint64_t i, numrec, buffer_max_entries_first, buffer_max_entries, log_page_offset = 0;
+	uint64_t remaining_num_rec = 0;
+	uint16_t recfmt;
+
+	memset(buffer, 0x0, 4096);
+	buffer_max_entries_first = (sizeof(buffer) - offsetof(struct spdk_nvmf_discovery_log_page,
+				    entries[0])) /
+				   sizeof(struct spdk_nvmf_discovery_log_page_entry);
+	buffer_max_entries = sizeof(buffer) / sizeof(struct spdk_nvmf_discovery_log_page_entry);
+	do {
+		rc = nvme_fabric_get_discovery_log_page(ctrlr, buffer, sizeof(buffer), log_page_offset);
+		if (rc < 0) {
+			SPDK_DEBUGLOG(SPDK_LOG_NVME, "Get Log Page - Discovery error\n");
+			return rc;
+		}
+
+		if (!remaining_num_rec) {
+			log_page = (struct spdk_nvmf_discovery_log_page *)buffer;
+			recfmt = from_le16(&log_page->recfmt);
+			if (recfmt != 0) {
+				SPDK_ERRLOG("Unrecognized discovery log record format %" PRIu16 "\n", recfmt);
+				return -EPROTO;
+			}
+			remaining_num_rec = log_page->numrec;
+			log_page_offset = offsetof(struct spdk_nvmf_discovery_log_page, entries[0]);
+			log_page_entry = &log_page->entries[0];
+			numrec = spdk_min(remaining_num_rec, buffer_max_entries_first);
+		} else {
+			numrec = spdk_min(remaining_num_rec, buffer_max_entries);
+			log_page_entry = (struct spdk_nvmf_discovery_log_page_entry *)buffer;
+		}
+
+		for (i = 0; i < numrec; i++) {
+			nvme_fabric_discover_probe(log_page_entry++, probe_ctx, ctrlr->trid.priority);
+		}
+		remaining_num_rec -= numrec;
+		log_page_offset += numrec * sizeof(struct spdk_nvmf_discovery_log_page_entry);
+	} while (remaining_num_rec != 0);
+
+	return 0;
+}
+
+int
+nvme_fabric_qpair_connect(struct spdk_nvme_qpair *qpair, uint32_t num_entries)
+{
+	struct nvme_completion_poll_status *status;
+	struct spdk_nvmf_fabric_connect_rsp *rsp;
+	struct spdk_nvmf_fabric_connect_cmd cmd;
+	struct spdk_nvmf_fabric_connect_data *nvmf_data;
+	struct spdk_nvme_ctrlr *ctrlr;
+	int rc;
+
+	if (num_entries == 0 || num_entries > SPDK_NVME_IO_QUEUE_MAX_ENTRIES) {
+		return -EINVAL;
+	}
+
+	ctrlr = qpair->ctrlr;
+	if (!ctrlr) {
+		return -EINVAL;
+	}
+
+	nvmf_data = spdk_zmalloc(sizeof(*nvmf_data), 0, NULL,
+				 SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
+	if (!nvmf_data) {
+		SPDK_ERRLOG("nvmf_data allocation error\n");
+		return -ENOMEM;
+	}
+
+	status = calloc(1, sizeof(*status));
+	if (!status) {
+		SPDK_ERRLOG("Failed to allocate status tracker\n");
+		spdk_free(nvmf_data);
+		return -ENOMEM;
+	}
+
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.opcode = SPDK_NVME_OPC_FABRIC;
+	cmd.fctype = SPDK_NVMF_FABRIC_COMMAND_CONNECT;
+	cmd.qid = qpair->id;
+	cmd.sqsize = num_entries - 1;
+	cmd.kato = ctrlr->opts.keep_alive_timeout_ms;
+
+	if (nvme_qpair_is_admin_queue(qpair)) {
+		nvmf_data->cntlid = 0xFFFF;
+	} else {
+		nvmf_data->cntlid = ctrlr->cntlid;
+	}
+
+	SPDK_STATIC_ASSERT(sizeof(nvmf_data->hostid) == sizeof(ctrlr->opts.extended_host_id),
+			   "host ID size mismatch");
+	memcpy(nvmf_data->hostid, ctrlr->opts.extended_host_id, sizeof(nvmf_data->hostid));
+	snprintf(nvmf_data->hostnqn, sizeof(nvmf_data->hostnqn), "%s", ctrlr->opts.hostnqn);
+	snprintf(nvmf_data->subnqn, sizeof(nvmf_data->subnqn), "%s", ctrlr->trid.subnqn);
+
+	rc = spdk_nvme_ctrlr_cmd_io_raw(ctrlr, qpair,
+					(struct spdk_nvme_cmd *)&cmd,
+					nvmf_data, sizeof(*nvmf_data),
+					nvme_completion_poll_cb, status);
+	if (rc < 0) {
+		SPDK_ERRLOG("Connect command failed\n");
+		spdk_free(nvmf_data);
+		free(status);
+		return rc;
+	}
+
+	if (nvme_wait_for_completion(qpair, status)) {
+		SPDK_ERRLOG("Connect command failed\n");
+		spdk_free(nvmf_data);
+		if (!status->timed_out) {
+			free(status);
+		}
+		return -EIO;
+	}
+
+	if (nvme_qpair_is_admin_queue(qpair)) {
+		rsp = (struct spdk_nvmf_fabric_connect_rsp *)&status->cpl;
+		ctrlr->cntlid = rsp->status_code_specific.success.cntlid;
+		SPDK_DEBUGLOG(SPDK_LOG_NVME, "CNTLID 0x%04" PRIx16 "\n", ctrlr->cntlid);
+	}
+
+	spdk_free(nvmf_data);
+	free(status);
+	return 0;
+}
diff --git a/src/spdk/lib/nvme/nvme_internal.h b/src/spdk/lib/nvme/nvme_internal.h
new file mode 100644
index 000000000..98fec279d
--- /dev/null
+++ b/src/spdk/lib/nvme/nvme_internal.h
@@ -0,0 +1,1233 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation. All rights reserved.
+ *   Copyright (c) 2020 Mellanox Technologies LTD. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __NVME_INTERNAL_H__
+#define __NVME_INTERNAL_H__
+
+#include "spdk/config.h"
+#include "spdk/likely.h"
+#include "spdk/stdinc.h"
+
+#include "spdk/nvme.h"
+
+#if defined(__i386__) || defined(__x86_64__)
+#include <x86intrin.h>
+#endif
+
+#include "spdk/queue.h"
+#include "spdk/barrier.h"
+#include "spdk/bit_array.h"
+#include "spdk/mmio.h"
+#include "spdk/pci_ids.h"
+#include "spdk/util.h"
+#include "spdk/memory.h"
+#include "spdk/nvme_intel.h"
+#include "spdk/nvmf_spec.h"
+#include "spdk/uuid.h"
+
+#include "spdk_internal/assert.h"
+#include "spdk_internal/log.h"
+
+extern pid_t g_spdk_nvme_pid;
+
+/*
+ * Some Intel devices support vendor-unique read latency log page even
+ * though the log page directory says otherwise.
+ */
+#define NVME_INTEL_QUIRK_READ_LATENCY 0x1
+
+/*
+ * Some Intel devices support vendor-unique write latency log page even
+ * though the log page directory says otherwise.
+ */
+#define NVME_INTEL_QUIRK_WRITE_LATENCY 0x2
+
+/*
+ * The controller needs a delay before starts checking the device
+ * readiness, which is done by reading the NVME_CSTS_RDY bit.
+ */
+#define NVME_QUIRK_DELAY_BEFORE_CHK_RDY	0x4
+
+/*
+ * The controller performs best when I/O is split on particular
+ * LBA boundaries.
+ */
+#define NVME_INTEL_QUIRK_STRIPING 0x8
+
+/*
+ * The controller needs a delay after allocating an I/O queue pair
+ * before it is ready to accept I/O commands.
+ */
+#define NVME_QUIRK_DELAY_AFTER_QUEUE_ALLOC 0x10
+
+/*
+ * Earlier NVMe devices do not indicate whether unmapped blocks
+ * will read all zeroes or not. This define indicates that the
+ * device does in fact read all zeroes after an unmap event
+ */
+#define NVME_QUIRK_READ_ZERO_AFTER_DEALLOCATE 0x20
+
+/*
+ * The controller doesn't handle Identify value others than 0 or 1 correctly.
+ */
+#define NVME_QUIRK_IDENTIFY_CNS 0x40
+
+/*
+ * The controller supports Open Channel command set if matching additional
+ * condition, like the first byte (value 0x1) in the vendor specific
+ * bits of the namespace identify structure is set.
+ */
+#define NVME_QUIRK_OCSSD 0x80
+
+/*
+ * The controller has an Intel vendor ID but does not support Intel vendor-specific
+ * log pages.  This is primarily for QEMU emulated SSDs which report an Intel vendor
+ * ID but do not support these log pages.
+ */
+#define NVME_INTEL_QUIRK_NO_LOG_PAGES 0x100
+
+/*
+ * The controller does not set SHST_COMPLETE in a reasonable amount of time.  This
+ * is primarily seen in virtual VMWare NVMe SSDs.  This quirk merely adds an additional
+ * error message that on VMWare NVMe SSDs, the shutdown timeout may be expected.
+ */
+#define NVME_QUIRK_SHST_COMPLETE 0x200
+
+/*
+ * The controller requires an extra delay before starting the initialization process
+ * during attach.
+ */
+#define NVME_QUIRK_DELAY_BEFORE_INIT 0x400
+
+/*
+ * Some SSDs exhibit poor performance with the default SPDK NVMe IO queue size.
+ * This quirk will increase the default to 1024 which matches other operating
+ * systems, at the cost of some extra memory usage.  Users can still override
+ * the increased default by changing the spdk_nvme_io_qpair_opts when allocating
+ * a new queue pair.
+ */
+#define NVME_QUIRK_MINIMUM_IO_QUEUE_SIZE 0x800
+
+/**
+ * The maximum access width to PCI memory space is 8 Bytes, don't use AVX2 or
+ * SSE instructions to optimize the memory access(memcpy or memset) larger than
+ * 8 Bytes.
+ */
+#define NVME_QUIRK_MAXIMUM_PCI_ACCESS_WIDTH 0x1000
+
+/**
+ * The SSD does not support OPAL even through it sets the security bit in OACS.
+ */
+#define NVME_QUIRK_OACS_SECURITY 0x2000
+
+#define NVME_MAX_ASYNC_EVENTS	(8)
+
+#define NVME_MAX_ADMIN_TIMEOUT_IN_SECS	(30)
+
+/* Maximum log page size to fetch for AERs. */
+#define NVME_MAX_AER_LOG_SIZE		(4096)
+
+/*
+ * NVME_MAX_IO_QUEUES in nvme_spec.h defines the 64K spec-limit, but this
+ *  define specifies the maximum number of queues this driver will actually
+ *  try to configure, if available.
+ */
+#define DEFAULT_MAX_IO_QUEUES		(1024)
+#define DEFAULT_ADMIN_QUEUE_SIZE	(32)
+#define DEFAULT_IO_QUEUE_SIZE		(256)
+#define DEFAULT_IO_QUEUE_SIZE_FOR_QUIRK	(1024) /* Matches Linux kernel driver */
+
+#define DEFAULT_IO_QUEUE_REQUESTS	(512)
+
+#define SPDK_NVME_DEFAULT_RETRY_COUNT	(4)
+
+#define SPDK_NVME_TRANSPORT_ACK_TIMEOUT_DISABLED	(0)
+#define SPDK_NVME_DEFAULT_TRANSPORT_ACK_TIMEOUT	SPDK_NVME_TRANSPORT_ACK_TIMEOUT_DISABLED
+
+#define MIN_KEEP_ALIVE_TIMEOUT_IN_MS	(10000)
+
+/* We want to fit submission and completion rings each in a single 2MB
+ * hugepage to ensure physical address contiguity.
+ */
+#define MAX_IO_QUEUE_ENTRIES		(VALUE_2MB / spdk_max( \
+						sizeof(struct spdk_nvme_cmd), \
+						sizeof(struct spdk_nvme_cpl)))
+
+enum nvme_payload_type {
+	NVME_PAYLOAD_TYPE_INVALID = 0,
+
+	/** nvme_request::u.payload.contig_buffer is valid for this request */
+	NVME_PAYLOAD_TYPE_CONTIG,
+
+	/** nvme_request::u.sgl is valid for this request */
+	NVME_PAYLOAD_TYPE_SGL,
+};
+
+/**
+ * Descriptor for a request data payload.
+ */
+struct nvme_payload {
+	/**
+	 * Functions for retrieving physical addresses for scattered payloads.
+	 */
+	spdk_nvme_req_reset_sgl_cb reset_sgl_fn;
+	spdk_nvme_req_next_sge_cb next_sge_fn;
+
+	/**
+	 * If reset_sgl_fn == NULL, this is a contig payload, and contig_or_cb_arg contains the
+	 * virtual memory address of a single virtually contiguous buffer.
+	 *
+	 * If reset_sgl_fn != NULL, this is a SGL payload, and contig_or_cb_arg contains the
+	 * cb_arg that will be passed to the SGL callback functions.
+	 */
+	void *contig_or_cb_arg;
+
+	/** Virtual memory address of a single virtually contiguous metadata buffer */
+	void *md;
+};
+
+#define NVME_PAYLOAD_CONTIG(contig_, md_) \
+	(struct nvme_payload) { \
+		.reset_sgl_fn = NULL, \
+		.next_sge_fn = NULL, \
+		.contig_or_cb_arg = (contig_), \
+		.md = (md_), \
+	}
+
+#define NVME_PAYLOAD_SGL(reset_sgl_fn_, next_sge_fn_, cb_arg_, md_) \
+	(struct nvme_payload) { \
+		.reset_sgl_fn = (reset_sgl_fn_), \
+		.next_sge_fn = (next_sge_fn_), \
+		.contig_or_cb_arg = (cb_arg_), \
+		.md = (md_), \
+	}
+
+static inline enum nvme_payload_type
+nvme_payload_type(const struct nvme_payload *payload) {
+	return payload->reset_sgl_fn ? NVME_PAYLOAD_TYPE_SGL : NVME_PAYLOAD_TYPE_CONTIG;
+}
+
+struct nvme_error_cmd {
+	bool				do_not_submit;
+	uint64_t			timeout_tsc;
+	uint32_t			err_count;
+	uint8_t				opc;
+	struct spdk_nvme_status		status;
+	TAILQ_ENTRY(nvme_error_cmd)	link;
+};
+
+struct nvme_request {
+	struct spdk_nvme_cmd		cmd;
+
+	uint8_t				retries;
+
+	uint8_t				timed_out : 1;
+
+	/**
+	 * True if the request is in the queued_req list.
+	 */
+	uint8_t				queued : 1;
+	uint8_t				reserved : 6;
+
+	/**
+	 * Number of children requests still outstanding for this
+	 *  request which was split into multiple child requests.
+	 */
+	uint16_t			num_children;
+
+	/**
+	 * Offset in bytes from the beginning of payload for this request.
+	 * This is used for I/O commands that are split into multiple requests.
+	 */
+	uint32_t			payload_offset;
+	uint32_t			md_offset;
+
+	uint32_t			payload_size;
+
+	/**
+	 * Timeout ticks for error injection requests, can be extended in future
+	 * to support per-request timeout feature.
+	 */
+	uint64_t			timeout_tsc;
+
+	/**
+	 * Data payload for this request's command.
+	 */
+	struct nvme_payload		payload;
+
+	spdk_nvme_cmd_cb		cb_fn;
+	void				*cb_arg;
+	STAILQ_ENTRY(nvme_request)	stailq;
+
+	struct spdk_nvme_qpair		*qpair;
+
+	/*
+	 * The value of spdk_get_ticks() when the request was submitted to the hardware.
+	 * Only set if ctrlr->timeout_enabled is true.
+	 */
+	uint64_t			submit_tick;
+
+	/**
+	 * The active admin request can be moved to a per process pending
+	 *  list based on the saved pid to tell which process it belongs
+	 *  to. The cpl saves the original completion information which
+	 *  is used in the completion callback.
+	 * NOTE: these below two fields are only used for admin request.
+	 */
+	pid_t				pid;
+	struct spdk_nvme_cpl		cpl;
+
+	uint32_t			md_size;
+
+	/**
+	 * The following members should not be reordered with members
+	 *  above.  These members are only needed when splitting
+	 *  requests which is done rarely, and the driver is careful
+	 *  to not touch the following fields until a split operation is
+	 *  needed, to avoid touching an extra cacheline.
+	 */
+
+	/**
+	 * Points to the outstanding child requests for a parent request.
+	 *  Only valid if a request was split into multiple children
+	 *  requests, and is not initialized for non-split requests.
+	 */
+	TAILQ_HEAD(, nvme_request)	children;
+
+	/**
+	 * Linked-list pointers for a child request in its parent's list.
+	 */
+	TAILQ_ENTRY(nvme_request)	child_tailq;
+
+	/**
+	 * Points to a parent request if part of a split request,
+	 *   NULL otherwise.
+	 */
+	struct nvme_request		*parent;
+
+	/**
+	 * Completion status for a parent request.  Initialized to all 0's
+	 *  (SUCCESS) before child requests are submitted.  If a child
+	 *  request completes with error, the error status is copied here,
+	 *  to ensure that the parent request is also completed with error
+	 *  status once all child requests are completed.
+	 */
+	struct spdk_nvme_cpl		parent_status;
+
+	/**
+	 * The user_cb_fn and user_cb_arg fields are used for holding the original
+	 * callback data when using nvme_allocate_request_user_copy.
+	 */
+	spdk_nvme_cmd_cb		user_cb_fn;
+	void				*user_cb_arg;
+	void				*user_buffer;
+};
+
+struct nvme_completion_poll_status {
+	struct spdk_nvme_cpl	cpl;
+	bool			done;
+	/* This flag indicates that the request has been timed out and the memory
+	   must be freed in a completion callback */
+	bool			timed_out;
+};
+
+struct nvme_async_event_request {
+	struct spdk_nvme_ctrlr		*ctrlr;
+	struct nvme_request		*req;
+	struct spdk_nvme_cpl		cpl;
+};
+
+enum nvme_qpair_state {
+	NVME_QPAIR_DISCONNECTED,
+	NVME_QPAIR_DISCONNECTING,
+	NVME_QPAIR_CONNECTING,
+	NVME_QPAIR_CONNECTED,
+	NVME_QPAIR_ENABLING,
+	NVME_QPAIR_ENABLED,
+	NVME_QPAIR_DESTROYING,
+};
+
+struct spdk_nvme_qpair {
+	struct spdk_nvme_ctrlr			*ctrlr;
+
+	uint16_t				id;
+
+	uint8_t					qprio;
+
+	uint8_t					state : 3;
+
+	/*
+	 * Members for handling IO qpair deletion inside of a completion context.
+	 * These are specifically defined as single bits, so that they do not
+	 *  push this data structure out to another cacheline.
+	 */
+	uint8_t					in_completion_context : 1;
+	uint8_t					delete_after_completion_context: 1;
+
+	/*
+	 * Set when no deletion notification is needed. For example, the process
+	 * which allocated this qpair exited unexpectedly.
+	 */
+	uint8_t					no_deletion_notification_needed: 1;
+
+	uint8_t					first_fused_submitted: 1;
+
+	enum spdk_nvme_transport_type		trtype;
+
+	STAILQ_HEAD(, nvme_request)		free_req;
+	STAILQ_HEAD(, nvme_request)		queued_req;
+	STAILQ_HEAD(, nvme_request)		aborting_queued_req;
+
+	/* List entry for spdk_nvme_transport_poll_group::qpairs */
+	STAILQ_ENTRY(spdk_nvme_qpair)		poll_group_stailq;
+
+	/** Commands opcode in this list will return error */
+	TAILQ_HEAD(, nvme_error_cmd)		err_cmd_head;
+	/** Requests in this list will return error */
+	STAILQ_HEAD(, nvme_request)		err_req_head;
+
+	/* List entry for spdk_nvme_ctrlr::active_io_qpairs */
+	TAILQ_ENTRY(spdk_nvme_qpair)		tailq;
+
+	/* List entry for spdk_nvme_ctrlr_process::allocated_io_qpairs */
+	TAILQ_ENTRY(spdk_nvme_qpair)		per_process_tailq;
+
+	struct spdk_nvme_ctrlr_process		*active_proc;
+
+	struct spdk_nvme_transport_poll_group	*poll_group;
+
+	void					*poll_group_tailq_head;
+
+	void					*req_buf;
+
+	const struct spdk_nvme_transport	*transport;
+
+	uint8_t					transport_failure_reason: 2;
+};
+
+struct spdk_nvme_poll_group {
+	void						*ctx;
+	STAILQ_HEAD(, spdk_nvme_transport_poll_group)	tgroups;
+};
+
+struct spdk_nvme_transport_poll_group {
+	struct spdk_nvme_poll_group			*group;
+	const struct spdk_nvme_transport		*transport;
+	STAILQ_HEAD(, spdk_nvme_qpair)			connected_qpairs;
+	STAILQ_HEAD(, spdk_nvme_qpair)			disconnected_qpairs;
+	STAILQ_ENTRY(spdk_nvme_transport_poll_group)	link;
+	bool						in_completion_context;
+	uint64_t					num_qpairs_to_delete;
+};
+
+struct spdk_nvme_ns {
+	struct spdk_nvme_ctrlr		*ctrlr;
+	uint32_t			sector_size;
+
+	/*
+	 * Size of data transferred as part of each block,
+	 * including metadata if FLBAS indicates the metadata is transferred
+	 * as part of the data buffer at the end of each LBA.
+	 */
+	uint32_t			extended_lba_size;
+
+	uint32_t			md_size;
+	uint32_t			pi_type;
+	uint32_t			sectors_per_max_io;
+	uint32_t			sectors_per_stripe;
+	uint32_t			id;
+	uint16_t			flags;
+
+	/* Namespace Identification Descriptor List (CNS = 03h) */
+	uint8_t				id_desc_list[4096];
+};
+
+/**
+ * State of struct spdk_nvme_ctrlr (in particular, during initialization).
+ */
+enum nvme_ctrlr_state {
+	/**
+	 * Wait before initializing the controller.
+	 */
+	NVME_CTRLR_STATE_INIT_DELAY,
+
+	/**
+	 * Controller has not been initialized yet.
+	 */
+	NVME_CTRLR_STATE_INIT,
+
+	/**
+	 * Waiting for CSTS.RDY to transition from 0 to 1 so that CC.EN may be set to 0.
+	 */
+	NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1,
+
+	/**
+	 * Waiting for CSTS.RDY to transition from 1 to 0 so that CC.EN may be set to 1.
+	 */
+	NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0,
+
+	/**
+	 * Enable the controller by writing CC.EN to 1
+	 */
+	NVME_CTRLR_STATE_ENABLE,
+
+	/**
+	 * Waiting for CSTS.RDY to transition from 0 to 1 after enabling the controller.
+	 */
+	NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1,
+
+	/**
+	 * Reset the Admin queue of the controller.
+	 */
+	NVME_CTRLR_STATE_RESET_ADMIN_QUEUE,
+
+	/**
+	 * Identify Controller command will be sent to then controller.
+	 */
+	NVME_CTRLR_STATE_IDENTIFY,
+
+	/**
+	 * Waiting for Identify Controller command be completed.
+	 */
+	NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY,
+
+	/**
+	 * Set Number of Queues of the controller.
+	 */
+	NVME_CTRLR_STATE_SET_NUM_QUEUES,
+
+	/**
+	 * Waiting for Set Num of Queues command to be completed.
+	 */
+	NVME_CTRLR_STATE_WAIT_FOR_SET_NUM_QUEUES,
+
+	/**
+	 * Construct Namespace data structures of the controller.
+	 */
+	NVME_CTRLR_STATE_CONSTRUCT_NS,
+
+	/**
+	 * Get active Namespace list of the controller.
+	 */
+	NVME_CTRLR_STATE_IDENTIFY_ACTIVE_NS,
+
+	/**
+	 * Waiting for the Identify Active Namespace commands to be completed.
+	 */
+	NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ACTIVE_NS,
+
+	/**
+	 * Get Identify Namespace Data structure for each NS.
+	 */
+	NVME_CTRLR_STATE_IDENTIFY_NS,
+
+	/**
+	 * Waiting for the Identify Namespace commands to be completed.
+	 */
+	NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS,
+
+	/**
+	 * Get Identify Namespace Identification Descriptors.
+	 */
+	NVME_CTRLR_STATE_IDENTIFY_ID_DESCS,
+
+	/**
+	 * Waiting for the Identify Namespace Identification
+	 * Descriptors to be completed.
+	 */
+	NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ID_DESCS,
+
+	/**
+	 * Configure AER of the controller.
+	 */
+	NVME_CTRLR_STATE_CONFIGURE_AER,
+
+	/**
+	 * Waiting for the Configure AER to be completed.
+	 */
+	NVME_CTRLR_STATE_WAIT_FOR_CONFIGURE_AER,
+
+	/**
+	 * Set supported log pages of the controller.
+	 */
+	NVME_CTRLR_STATE_SET_SUPPORTED_LOG_PAGES,
+
+	/**
+	 * Set supported features of the controller.
+	 */
+	NVME_CTRLR_STATE_SET_SUPPORTED_FEATURES,
+
+	/**
+	 * Set Doorbell Buffer Config of the controller.
+	 */
+	NVME_CTRLR_STATE_SET_DB_BUF_CFG,
+
+	/**
+	 * Waiting for Doorbell Buffer Config to be completed.
+	 */
+	NVME_CTRLR_STATE_WAIT_FOR_DB_BUF_CFG,
+
+	/**
+	 * Set Keep Alive Timeout of the controller.
+	 */
+	NVME_CTRLR_STATE_SET_KEEP_ALIVE_TIMEOUT,
+
+	/**
+	 * Waiting for Set Keep Alive Timeout to be completed.
+	 */
+	NVME_CTRLR_STATE_WAIT_FOR_KEEP_ALIVE_TIMEOUT,
+
+	/**
+	 * Set Host ID of the controller.
+	 */
+	NVME_CTRLR_STATE_SET_HOST_ID,
+
+	/**
+	 * Waiting for Set Host ID to be completed.
+	 */
+	NVME_CTRLR_STATE_WAIT_FOR_HOST_ID,
+
+	/**
+	 * Controller initialization has completed and the controller is ready.
+	 */
+	NVME_CTRLR_STATE_READY,
+
+	/**
+	 * Controller inilialization has an error.
+	 */
+	NVME_CTRLR_STATE_ERROR
+};
+
+#define NVME_TIMEOUT_INFINITE	0
+
+/*
+ * Used to track properties for all processes accessing the controller.
+ */
+struct spdk_nvme_ctrlr_process {
+	/** Whether it is the primary process  */
+	bool						is_primary;
+
+	/** Process ID */
+	pid_t						pid;
+
+	/** Active admin requests to be completed */
+	STAILQ_HEAD(, nvme_request)			active_reqs;
+
+	TAILQ_ENTRY(spdk_nvme_ctrlr_process)		tailq;
+
+	/** Per process PCI device handle */
+	struct spdk_pci_device				*devhandle;
+
+	/** Reference to track the number of attachment to this controller. */
+	int						ref;
+
+	/** Allocated IO qpairs */
+	TAILQ_HEAD(, spdk_nvme_qpair)			allocated_io_qpairs;
+
+	spdk_nvme_aer_cb				aer_cb_fn;
+	void						*aer_cb_arg;
+
+	/**
+	 * A function pointer to timeout callback function
+	 */
+	spdk_nvme_timeout_cb		timeout_cb_fn;
+	void				*timeout_cb_arg;
+	uint64_t			timeout_ticks;
+};
+
+/*
+ * One of these per allocated PCI device.
+ */
+struct spdk_nvme_ctrlr {
+	/* Hot data (accessed in I/O path) starts here. */
+
+	/** Array of namespaces indexed by nsid - 1 */
+	struct spdk_nvme_ns		*ns;
+
+	uint32_t			num_ns;
+
+	bool				is_removed;
+
+	bool				is_resetting;
+
+	bool				is_failed;
+
+	bool				is_destructed;
+
+	bool				timeout_enabled;
+
+	uint16_t			max_sges;
+
+	uint16_t			cntlid;
+
+	/** Controller support flags */
+	uint64_t			flags;
+
+	/** NVMEoF in-capsule data size in bytes */
+	uint32_t			ioccsz_bytes;
+
+	/** NVMEoF in-capsule data offset in 16 byte units */
+	uint16_t			icdoff;
+
+	/* Cold data (not accessed in normal I/O path) is after this point. */
+
+	struct spdk_nvme_transport_id	trid;
+
+	union spdk_nvme_cap_register	cap;
+	union spdk_nvme_vs_register	vs;
+
+	enum nvme_ctrlr_state		state;
+	uint64_t			state_timeout_tsc;
+
+	uint64_t			next_keep_alive_tick;
+	uint64_t			keep_alive_interval_ticks;
+
+	TAILQ_ENTRY(spdk_nvme_ctrlr)	tailq;
+
+	/** All the log pages supported */
+	bool				log_page_supported[256];
+
+	/** All the features supported */
+	bool				feature_supported[256];
+
+	/** maximum i/o size in bytes */
+	uint32_t			max_xfer_size;
+
+	/** minimum page size supported by this controller in bytes */
+	uint32_t			min_page_size;
+
+	/** selected memory page size for this controller in bytes */
+	uint32_t			page_size;
+
+	uint32_t			num_aers;
+	struct nvme_async_event_request	aer[NVME_MAX_ASYNC_EVENTS];
+
+	/** guards access to the controller itself, including admin queues */
+	pthread_mutex_t			ctrlr_lock;
+
+	struct spdk_nvme_qpair		*adminq;
+
+	/** shadow doorbell buffer */
+	uint32_t			*shadow_doorbell;
+	/** eventidx buffer */
+	uint32_t			*eventidx;
+
+	/**
+	 * Identify Controller data.
+	 */
+	struct spdk_nvme_ctrlr_data	cdata;
+
+	/**
+	 * Keep track of active namespaces
+	 */
+	uint32_t			*active_ns_list;
+
+	/**
+	 * Array of Identify Namespace data.
+	 *
+	 * Stored separately from ns since nsdata should not normally be accessed during I/O.
+	 */
+	struct spdk_nvme_ns_data	*nsdata;
+
+	struct spdk_bit_array		*free_io_qids;
+	TAILQ_HEAD(, spdk_nvme_qpair)	active_io_qpairs;
+
+	struct spdk_nvme_ctrlr_opts	opts;
+
+	uint64_t			quirks;
+
+	/* Extra sleep time during controller initialization */
+	uint64_t			sleep_timeout_tsc;
+
+	/** Track all the processes manage this controller */
+	TAILQ_HEAD(, spdk_nvme_ctrlr_process)	active_procs;
+
+
+	STAILQ_HEAD(, nvme_request)	queued_aborts;
+	uint32_t			outstanding_aborts;
+
+	/* CB to notify the user when the ctrlr is removed/failed. */
+	spdk_nvme_remove_cb			remove_cb;
+	void					*cb_ctx;
+
+	struct spdk_nvme_qpair		*external_io_msgs_qpair;
+	pthread_mutex_t			external_io_msgs_lock;
+	struct spdk_ring		*external_io_msgs;
+
+	STAILQ_HEAD(, nvme_io_msg_producer) io_producers;
+};
+
+struct spdk_nvme_probe_ctx {
+	struct spdk_nvme_transport_id		trid;
+	void					*cb_ctx;
+	spdk_nvme_probe_cb			probe_cb;
+	spdk_nvme_attach_cb			attach_cb;
+	spdk_nvme_remove_cb			remove_cb;
+	TAILQ_HEAD(, spdk_nvme_ctrlr)		init_ctrlrs;
+};
+
+struct nvme_driver {
+	pthread_mutex_t			lock;
+
+	/** Multi-process shared attached controller list */
+	TAILQ_HEAD(, spdk_nvme_ctrlr)	shared_attached_ctrlrs;
+
+	bool				initialized;
+	struct spdk_uuid		default_extended_host_id;
+
+	/** netlink socket fd for hotplug messages */
+	int				hotplug_fd;
+};
+
+extern struct nvme_driver *g_spdk_nvme_driver;
+
+int nvme_driver_init(void);
+
+#define nvme_delay		usleep
+
+static inline bool
+nvme_qpair_is_admin_queue(struct spdk_nvme_qpair *qpair)
+{
+	return qpair->id == 0;
+}
+
+static inline bool
+nvme_qpair_is_io_queue(struct spdk_nvme_qpair *qpair)
+{
+	return qpair->id != 0;
+}
+
+static inline int
+nvme_robust_mutex_lock(pthread_mutex_t *mtx)
+{
+	int rc = pthread_mutex_lock(mtx);
+
+#ifndef __FreeBSD__
+	if (rc == EOWNERDEAD) {
+		rc = pthread_mutex_consistent(mtx);
+	}
+#endif
+
+	return rc;
+}
+
+static inline int
+nvme_robust_mutex_unlock(pthread_mutex_t *mtx)
+{
+	return pthread_mutex_unlock(mtx);
+}
+
+/* Poll group management functions. */
+int nvme_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair);
+int nvme_poll_group_disconnect_qpair(struct spdk_nvme_qpair *qpair);
+
+/* Admin functions */
+int	nvme_ctrlr_cmd_identify(struct spdk_nvme_ctrlr *ctrlr,
+				uint8_t cns, uint16_t cntid, uint32_t nsid,
+				void *payload, size_t payload_size,
+				spdk_nvme_cmd_cb cb_fn, void *cb_arg);
+int	nvme_ctrlr_cmd_set_num_queues(struct spdk_nvme_ctrlr *ctrlr,
+				      uint32_t num_queues, spdk_nvme_cmd_cb cb_fn,
+				      void *cb_arg);
+int	nvme_ctrlr_cmd_get_num_queues(struct spdk_nvme_ctrlr *ctrlr,
+				      spdk_nvme_cmd_cb cb_fn, void *cb_arg);
+int	nvme_ctrlr_cmd_set_async_event_config(struct spdk_nvme_ctrlr *ctrlr,
+		union spdk_nvme_feat_async_event_configuration config,
+		spdk_nvme_cmd_cb cb_fn, void *cb_arg);
+int	nvme_ctrlr_cmd_set_host_id(struct spdk_nvme_ctrlr *ctrlr, void *host_id, uint32_t host_id_size,
+				   spdk_nvme_cmd_cb cb_fn, void *cb_arg);
+int	nvme_ctrlr_cmd_attach_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid,
+				 struct spdk_nvme_ctrlr_list *payload, spdk_nvme_cmd_cb cb_fn, void *cb_arg);
+int	nvme_ctrlr_cmd_detach_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid,
+				 struct spdk_nvme_ctrlr_list *payload, spdk_nvme_cmd_cb cb_fn, void *cb_arg);
+int	nvme_ctrlr_cmd_create_ns(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns_data *payload,
+				 spdk_nvme_cmd_cb cb_fn, void *cb_arg);
+int	nvme_ctrlr_cmd_doorbell_buffer_config(struct spdk_nvme_ctrlr *ctrlr,
+		uint64_t prp1, uint64_t prp2,
+		spdk_nvme_cmd_cb cb_fn, void *cb_arg);
+int	nvme_ctrlr_cmd_delete_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, spdk_nvme_cmd_cb cb_fn,
+				 void *cb_arg);
+int	nvme_ctrlr_cmd_format(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid,
+			      struct spdk_nvme_format *format, spdk_nvme_cmd_cb cb_fn, void *cb_arg);
+int	nvme_ctrlr_cmd_fw_commit(struct spdk_nvme_ctrlr *ctrlr,
+				 const struct spdk_nvme_fw_commit *fw_commit,
+				 spdk_nvme_cmd_cb cb_fn, void *cb_arg);
+int	nvme_ctrlr_cmd_fw_image_download(struct spdk_nvme_ctrlr *ctrlr,
+		uint32_t size, uint32_t offset, void *payload,
+		spdk_nvme_cmd_cb cb_fn, void *cb_arg);
+int	nvme_ctrlr_cmd_sanitize(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid,
+				struct spdk_nvme_sanitize *sanitize, uint32_t cdw11,
+				spdk_nvme_cmd_cb cb_fn, void *cb_arg);
+void	nvme_completion_poll_cb(void *arg, const struct spdk_nvme_cpl *cpl);
+int	nvme_wait_for_completion(struct spdk_nvme_qpair *qpair,
+				 struct nvme_completion_poll_status *status);
+int	nvme_wait_for_completion_robust_lock(struct spdk_nvme_qpair *qpair,
+		struct nvme_completion_poll_status *status,
+		pthread_mutex_t *robust_mutex);
+int	nvme_wait_for_completion_timeout(struct spdk_nvme_qpair *qpair,
+		struct nvme_completion_poll_status *status,
+		uint64_t timeout_in_secs);
+
+struct spdk_nvme_ctrlr_process *nvme_ctrlr_get_process(struct spdk_nvme_ctrlr *ctrlr,
+		pid_t pid);
+struct spdk_nvme_ctrlr_process *nvme_ctrlr_get_current_process(struct spdk_nvme_ctrlr *ctrlr);
+int	nvme_ctrlr_add_process(struct spdk_nvme_ctrlr *ctrlr, void *devhandle);
+void	nvme_ctrlr_free_processes(struct spdk_nvme_ctrlr *ctrlr);
+struct spdk_pci_device *nvme_ctrlr_proc_get_devhandle(struct spdk_nvme_ctrlr *ctrlr);
+
+int	nvme_ctrlr_probe(const struct spdk_nvme_transport_id *trid,
+			 struct spdk_nvme_probe_ctx *probe_ctx, void *devhandle);
+
+int	nvme_ctrlr_construct(struct spdk_nvme_ctrlr *ctrlr);
+void	nvme_ctrlr_destruct_finish(struct spdk_nvme_ctrlr *ctrlr);
+void	nvme_ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr);
+void	nvme_ctrlr_fail(struct spdk_nvme_ctrlr *ctrlr, bool hot_remove);
+int	nvme_ctrlr_reset(struct spdk_nvme_ctrlr *ctrlr);
+int	nvme_ctrlr_process_init(struct spdk_nvme_ctrlr *ctrlr);
+void	nvme_ctrlr_connected(struct spdk_nvme_probe_ctx *probe_ctx,
+			     struct spdk_nvme_ctrlr *ctrlr);
+
+int	nvme_ctrlr_submit_admin_request(struct spdk_nvme_ctrlr *ctrlr,
+					struct nvme_request *req);
+int	nvme_ctrlr_get_cap(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_cap_register *cap);
+int	nvme_ctrlr_get_vs(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_vs_register *vs);
+int	nvme_ctrlr_get_cmbsz(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_cmbsz_register *cmbsz);
+void	nvme_ctrlr_init_cap(struct spdk_nvme_ctrlr *ctrlr, const union spdk_nvme_cap_register *cap,
+			    const union spdk_nvme_vs_register *vs);
+void nvme_ctrlr_disconnect_qpair(struct spdk_nvme_qpair *qpair);
+int nvme_qpair_init(struct spdk_nvme_qpair *qpair, uint16_t id,
+		    struct spdk_nvme_ctrlr *ctrlr,
+		    enum spdk_nvme_qprio qprio,
+		    uint32_t num_requests);
+void	nvme_qpair_deinit(struct spdk_nvme_qpair *qpair);
+void	nvme_qpair_complete_error_reqs(struct spdk_nvme_qpair *qpair);
+int	nvme_qpair_submit_request(struct spdk_nvme_qpair *qpair,
+				  struct nvme_request *req);
+void	nvme_qpair_abort_reqs(struct spdk_nvme_qpair *qpair, uint32_t dnr);
+uint32_t nvme_qpair_abort_queued_reqs(struct spdk_nvme_qpair *qpair, void *cmd_cb_arg);
+void	nvme_qpair_resubmit_requests(struct spdk_nvme_qpair *qpair, uint32_t num_requests);
+
+int	nvme_ctrlr_identify_active_ns(struct spdk_nvme_ctrlr *ctrlr);
+void	nvme_ns_set_identify_data(struct spdk_nvme_ns *ns);
+int	nvme_ns_construct(struct spdk_nvme_ns *ns, uint32_t id,
+			  struct spdk_nvme_ctrlr *ctrlr);
+void	nvme_ns_destruct(struct spdk_nvme_ns *ns);
+int	nvme_ns_update(struct spdk_nvme_ns *ns);
+
+int	nvme_fabric_ctrlr_set_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t value);
+int	nvme_fabric_ctrlr_set_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t value);
+int	nvme_fabric_ctrlr_get_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t *value);
+int	nvme_fabric_ctrlr_scan(struct spdk_nvme_probe_ctx *probe_ctx, bool direct_connect);
+int	nvme_fabric_ctrlr_get_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t *value);
+int	nvme_fabric_ctrlr_discover(struct spdk_nvme_ctrlr *ctrlr,
+				   struct spdk_nvme_probe_ctx *probe_ctx);
+int	nvme_fabric_qpair_connect(struct spdk_nvme_qpair *qpair, uint32_t num_entries);
+
+static inline struct nvme_request *
+nvme_allocate_request(struct spdk_nvme_qpair *qpair,
+		      const struct nvme_payload *payload, uint32_t payload_size, uint32_t md_size,
+		      spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	struct nvme_request *req;
+
+	req = STAILQ_FIRST(&qpair->free_req);
+	if (req == NULL) {
+		return req;
+	}
+
+	STAILQ_REMOVE_HEAD(&qpair->free_req, stailq);
+
+	/*
+	 * Only memset/zero fields that need it.  All other fields
+	 *  will be initialized appropriately either later in this
+	 *  function, or before they are needed later in the
+	 *  submission patch.  For example, the children
+	 *  TAILQ_ENTRY and following members are
+	 *  only used as part of I/O splitting so we avoid
+	 *  memsetting them until it is actually needed.
+	 *  They will be initialized in nvme_request_add_child()
+	 *  if the request is split.
+	 */
+	memset(req, 0, offsetof(struct nvme_request, payload_size));
+
+	req->cb_fn = cb_fn;
+	req->cb_arg = cb_arg;
+	req->payload = *payload;
+	req->payload_size = payload_size;
+	req->md_size = md_size;
+	req->pid = g_spdk_nvme_pid;
+	req->submit_tick = 0;
+
+	return req;
+}
+
+static inline struct nvme_request *
+nvme_allocate_request_contig(struct spdk_nvme_qpair *qpair,
+			     void *buffer, uint32_t payload_size,
+			     spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	struct nvme_payload payload;
+
+	payload = NVME_PAYLOAD_CONTIG(buffer, NULL);
+
+	return nvme_allocate_request(qpair, &payload, payload_size, 0, cb_fn, cb_arg);
+}
+
+static inline struct nvme_request *
+nvme_allocate_request_null(struct spdk_nvme_qpair *qpair, spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	return nvme_allocate_request_contig(qpair, NULL, 0, cb_fn, cb_arg);
+}
+
+struct nvme_request *nvme_allocate_request_user_copy(struct spdk_nvme_qpair *qpair,
+		void *buffer, uint32_t payload_size,
+		spdk_nvme_cmd_cb cb_fn, void *cb_arg, bool host_to_controller);
+
+static inline void
+nvme_complete_request(spdk_nvme_cmd_cb cb_fn, void *cb_arg, struct spdk_nvme_qpair *qpair,
+		      struct nvme_request *req, struct spdk_nvme_cpl *cpl)
+{
+	struct spdk_nvme_cpl            err_cpl;
+	struct nvme_error_cmd           *cmd;
+
+	/* error injection at completion path,
+	 * only inject for successful completed commands
+	 */
+	if (spdk_unlikely(!TAILQ_EMPTY(&qpair->err_cmd_head) &&
+			  !spdk_nvme_cpl_is_error(cpl))) {
+		TAILQ_FOREACH(cmd, &qpair->err_cmd_head, link) {
+
+			if (cmd->do_not_submit) {
+				continue;
+			}
+
+			if ((cmd->opc == req->cmd.opc) && cmd->err_count) {
+
+				err_cpl = *cpl;
+				err_cpl.status.sct = cmd->status.sct;
+				err_cpl.status.sc = cmd->status.sc;
+
+				cpl = &err_cpl;
+				cmd->err_count--;
+				break;
+			}
+		}
+	}
+
+	if (cb_fn) {
+		cb_fn(cb_arg, cpl);
+	}
+}
+
+static inline void
+nvme_free_request(struct nvme_request *req)
+{
+	assert(req != NULL);
+	assert(req->num_children == 0);
+	assert(req->qpair != NULL);
+
+	STAILQ_INSERT_HEAD(&req->qpair->free_req, req, stailq);
+}
+
+static inline void
+nvme_qpair_set_state(struct spdk_nvme_qpair *qpair, enum nvme_qpair_state state)
+{
+	qpair->state = state;
+}
+
+static inline enum nvme_qpair_state
+nvme_qpair_get_state(struct spdk_nvme_qpair *qpair) {
+	return qpair->state;
+}
+
+static inline void
+nvme_qpair_free_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req)
+{
+	assert(req != NULL);
+	assert(req->num_children == 0);
+
+	STAILQ_INSERT_HEAD(&qpair->free_req, req, stailq);
+}
+
+static inline void
+nvme_request_remove_child(struct nvme_request *parent, struct nvme_request *child)
+{
+	assert(parent != NULL);
+	assert(child != NULL);
+	assert(child->parent == parent);
+	assert(parent->num_children != 0);
+
+	parent->num_children--;
+	child->parent = NULL;
+	TAILQ_REMOVE(&parent->children, child, child_tailq);
+}
+
+static inline void
+nvme_cb_complete_child(void *child_arg, const struct spdk_nvme_cpl *cpl)
+{
+	struct nvme_request *child = child_arg;
+	struct nvme_request *parent = child->parent;
+
+	nvme_request_remove_child(parent, child);
+
+	if (spdk_nvme_cpl_is_error(cpl)) {
+		memcpy(&parent->parent_status, cpl, sizeof(*cpl));
+	}
+
+	if (parent->num_children == 0) {
+		nvme_complete_request(parent->cb_fn, parent->cb_arg, parent->qpair,
+				      parent, &parent->parent_status);
+		nvme_free_request(parent);
+	}
+}
+
+static inline void
+nvme_request_add_child(struct nvme_request *parent, struct nvme_request *child)
+{
+	assert(parent->num_children != UINT16_MAX);
+
+	if (parent->num_children == 0) {
+		/*
+		 * Defer initialization of the children TAILQ since it falls
+		 *  on a separate cacheline.  This ensures we do not touch this
+		 *  cacheline except on request splitting cases, which are
+		 *  relatively rare.
+		 */
+		TAILQ_INIT(&parent->children);
+		parent->parent = NULL;
+		memset(&parent->parent_status, 0, sizeof(struct spdk_nvme_cpl));
+	}
+
+	parent->num_children++;
+	TAILQ_INSERT_TAIL(&parent->children, child, child_tailq);
+	child->parent = parent;
+	child->cb_fn = nvme_cb_complete_child;
+	child->cb_arg = child;
+}
+
+static inline void
+nvme_request_free_children(struct nvme_request *req)
+{
+	struct nvme_request *child, *tmp;
+
+	if (req->num_children == 0) {
+		return;
+	}
+
+	/* free all child nvme_request */
+	TAILQ_FOREACH_SAFE(child, &req->children, child_tailq, tmp) {
+		nvme_request_remove_child(req, child);
+		nvme_request_free_children(child);
+		nvme_free_request(child);
+	}
+}
+
+int	nvme_request_check_timeout(struct nvme_request *req, uint16_t cid,
+				   struct spdk_nvme_ctrlr_process *active_proc, uint64_t now_tick);
+uint64_t nvme_get_quirks(const struct spdk_pci_id *id);
+
+int	nvme_robust_mutex_init_shared(pthread_mutex_t *mtx);
+int	nvme_robust_mutex_init_recursive_shared(pthread_mutex_t *mtx);
+
+bool	nvme_completion_is_retry(const struct spdk_nvme_cpl *cpl);
+
+struct spdk_nvme_ctrlr *nvme_get_ctrlr_by_trid_unsafe(
+	const struct spdk_nvme_transport_id *trid);
+
+const struct spdk_nvme_transport *nvme_get_transport(const char *transport_name);
+const struct spdk_nvme_transport *nvme_get_first_transport(void);
+const struct spdk_nvme_transport *nvme_get_next_transport(const struct spdk_nvme_transport
+		*transport);
+
+/* Transport specific functions */
+struct spdk_nvme_ctrlr *nvme_transport_ctrlr_construct(const struct spdk_nvme_transport_id *trid,
+		const struct spdk_nvme_ctrlr_opts *opts,
+		void *devhandle);
+int nvme_transport_ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr);
+int nvme_transport_ctrlr_scan(struct spdk_nvme_probe_ctx *probe_ctx, bool direct_connect);
+int nvme_transport_ctrlr_enable(struct spdk_nvme_ctrlr *ctrlr);
+int nvme_transport_ctrlr_set_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t value);
+int nvme_transport_ctrlr_set_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t value);
+int nvme_transport_ctrlr_get_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t *value);
+int nvme_transport_ctrlr_get_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t *value);
+uint32_t nvme_transport_ctrlr_get_max_xfer_size(struct spdk_nvme_ctrlr *ctrlr);
+uint16_t nvme_transport_ctrlr_get_max_sges(struct spdk_nvme_ctrlr *ctrlr);
+struct spdk_nvme_qpair *nvme_transport_ctrlr_create_io_qpair(struct spdk_nvme_ctrlr *ctrlr,
+		uint16_t qid, const struct spdk_nvme_io_qpair_opts *opts);
+int nvme_transport_ctrlr_reserve_cmb(struct spdk_nvme_ctrlr *ctrlr);
+void *nvme_transport_ctrlr_map_cmb(struct spdk_nvme_ctrlr *ctrlr, size_t *size);
+int nvme_transport_ctrlr_unmap_cmb(struct spdk_nvme_ctrlr *ctrlr);
+int nvme_transport_ctrlr_delete_io_qpair(struct spdk_nvme_ctrlr *ctrlr,
+		struct spdk_nvme_qpair *qpair);
+int nvme_transport_ctrlr_connect_qpair(struct spdk_nvme_ctrlr *ctrlr,
+				       struct spdk_nvme_qpair *qpair);
+void nvme_transport_ctrlr_disconnect_qpair(struct spdk_nvme_ctrlr *ctrlr,
+		struct spdk_nvme_qpair *qpair);
+void nvme_transport_qpair_abort_reqs(struct spdk_nvme_qpair *qpair, uint32_t dnr);
+int nvme_transport_qpair_reset(struct spdk_nvme_qpair *qpair);
+int nvme_transport_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req);
+int32_t nvme_transport_qpair_process_completions(struct spdk_nvme_qpair *qpair,
+		uint32_t max_completions);
+void nvme_transport_admin_qpair_abort_aers(struct spdk_nvme_qpair *qpair);
+int nvme_transport_qpair_iterate_requests(struct spdk_nvme_qpair *qpair,
+		int (*iter_fn)(struct nvme_request *req, void *arg),
+		void *arg);
+
+struct spdk_nvme_transport_poll_group *nvme_transport_poll_group_create(
+	const struct spdk_nvme_transport *transport);
+int nvme_transport_poll_group_add(struct spdk_nvme_transport_poll_group *tgroup,
+				  struct spdk_nvme_qpair *qpair);
+int nvme_transport_poll_group_remove(struct spdk_nvme_transport_poll_group *tgroup,
+				     struct spdk_nvme_qpair *qpair);
+int nvme_transport_poll_group_disconnect_qpair(struct spdk_nvme_qpair *qpair);
+int nvme_transport_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair);
+int64_t nvme_transport_poll_group_process_completions(struct spdk_nvme_transport_poll_group *tgroup,
+		uint32_t completions_per_qpair, spdk_nvme_disconnected_qpair_cb disconnected_qpair_cb);
+int nvme_transport_poll_group_destroy(struct spdk_nvme_transport_poll_group *tgroup);
+/*
+ * Below ref related functions must be called with the global
+ *  driver lock held for the multi-process condition.
+ *  Within these functions, the per ctrlr ctrlr_lock is also
+ *  acquired for the multi-thread condition.
+ */
+void	nvme_ctrlr_proc_get_ref(struct spdk_nvme_ctrlr *ctrlr);
+void	nvme_ctrlr_proc_put_ref(struct spdk_nvme_ctrlr *ctrlr);
+int	nvme_ctrlr_get_ref_count(struct spdk_nvme_ctrlr *ctrlr);
+
+static inline bool
+_is_page_aligned(uint64_t address, uint64_t page_size)
+{
+	return (address & (page_size - 1)) == 0;
+}
+
+#endif /* __NVME_INTERNAL_H__ */
diff --git a/src/spdk/lib/nvme/nvme_io_msg.c b/src/spdk/lib/nvme/nvme_io_msg.c
new file mode 100644
index 000000000..fb5aec3d4
--- /dev/null
+++ b/src/spdk/lib/nvme/nvme_io_msg.c
@@ -0,0 +1,216 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "nvme_internal.h"
+#include "nvme_io_msg.h"
+
+#define SPDK_NVME_MSG_IO_PROCESS_SIZE 8
+
+/**
+ * Send message to IO queue.
+ */
+int
+nvme_io_msg_send(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, spdk_nvme_io_msg_fn fn,
+		 void *arg)
+{
+	int rc;
+	struct spdk_nvme_io_msg *io;
+
+	/* Protect requests ring against preemptive producers */
+	pthread_mutex_lock(&ctrlr->external_io_msgs_lock);
+
+	io = (struct spdk_nvme_io_msg *)calloc(1, sizeof(struct spdk_nvme_io_msg));
+	if (!io) {
+		SPDK_ERRLOG("IO msg allocation failed.");
+		pthread_mutex_unlock(&ctrlr->external_io_msgs_lock);
+		return -ENOMEM;
+	}
+
+	io->ctrlr = ctrlr;
+	io->nsid = nsid;
+	io->fn = fn;
+	io->arg = arg;
+
+	rc = spdk_ring_enqueue(ctrlr->external_io_msgs, (void **)&io, 1, NULL);
+	if (rc != 1) {
+		assert(false);
+		free(io);
+		pthread_mutex_unlock(&ctrlr->external_io_msgs_lock);
+		return -ENOMEM;
+	}
+
+	pthread_mutex_unlock(&ctrlr->external_io_msgs_lock);
+
+	return 0;
+}
+
+int
+nvme_io_msg_process(struct spdk_nvme_ctrlr *ctrlr)
+{
+	int i;
+	int count;
+	struct spdk_nvme_io_msg *io;
+	void *requests[SPDK_NVME_MSG_IO_PROCESS_SIZE];
+
+	if (!ctrlr->external_io_msgs || !ctrlr->external_io_msgs_qpair) {
+		/* Not ready or pending reset */
+		return 0;
+	}
+
+	spdk_nvme_qpair_process_completions(ctrlr->external_io_msgs_qpair, 0);
+
+	count = spdk_ring_dequeue(ctrlr->external_io_msgs, requests,
+				  SPDK_NVME_MSG_IO_PROCESS_SIZE);
+	if (count == 0) {
+		return 0;
+	}
+
+	for (i = 0; i < count; i++) {
+		io = requests[i];
+
+		assert(io != NULL);
+
+		io->fn(io->ctrlr, io->nsid, io->arg);
+		free(io);
+	}
+
+	return count;
+}
+
+static bool
+nvme_io_msg_is_producer_registered(struct spdk_nvme_ctrlr *ctrlr,
+				   struct nvme_io_msg_producer *io_msg_producer)
+{
+	struct nvme_io_msg_producer *tmp;
+
+	STAILQ_FOREACH(tmp, &ctrlr->io_producers, link) {
+		if (tmp == io_msg_producer) {
+			return true;
+		}
+	}
+	return false;
+}
+
+int
+nvme_io_msg_ctrlr_register(struct spdk_nvme_ctrlr *ctrlr,
+			   struct nvme_io_msg_producer *io_msg_producer)
+{
+	if (io_msg_producer == NULL) {
+		SPDK_ERRLOG("io_msg_producer cannot be NULL\n");
+		return -EINVAL;
+	}
+
+	if (nvme_io_msg_is_producer_registered(ctrlr, io_msg_producer)) {
+		return -EEXIST;
+	}
+
+	if (!STAILQ_EMPTY(&ctrlr->io_producers) || ctrlr->is_resetting) {
+		/* There are registered producers - IO messaging already started */
+		STAILQ_INSERT_TAIL(&ctrlr->io_producers, io_msg_producer, link);
+		return 0;
+	}
+
+	pthread_mutex_init(&ctrlr->external_io_msgs_lock, NULL);
+
+	/**
+	 * Initialize ring and qpair for controller
+	 */
+	ctrlr->external_io_msgs = spdk_ring_create(SPDK_RING_TYPE_MP_SC, 65536, SPDK_ENV_SOCKET_ID_ANY);
+	if (!ctrlr->external_io_msgs) {
+		SPDK_ERRLOG("Unable to allocate memory for message ring\n");
+		return -ENOMEM;
+	}
+
+	ctrlr->external_io_msgs_qpair = spdk_nvme_ctrlr_alloc_io_qpair(ctrlr, NULL, 0);
+	if (ctrlr->external_io_msgs_qpair == NULL) {
+		SPDK_ERRLOG("spdk_nvme_ctrlr_alloc_io_qpair() failed\n");
+		spdk_ring_free(ctrlr->external_io_msgs);
+		ctrlr->external_io_msgs = NULL;
+		return -ENOMEM;
+	}
+
+	STAILQ_INSERT_TAIL(&ctrlr->io_producers, io_msg_producer, link);
+
+	return 0;
+}
+
+void
+nvme_io_msg_ctrlr_update(struct spdk_nvme_ctrlr *ctrlr)
+{
+	struct nvme_io_msg_producer *io_msg_producer;
+
+	/* Update all producers */
+	STAILQ_FOREACH(io_msg_producer, &ctrlr->io_producers, link) {
+		io_msg_producer->update(ctrlr);
+	}
+}
+
+void
+nvme_io_msg_ctrlr_detach(struct spdk_nvme_ctrlr *ctrlr)
+{
+	struct nvme_io_msg_producer *io_msg_producer, *tmp;
+
+	/* Stop all producers */
+	STAILQ_FOREACH_SAFE(io_msg_producer, &ctrlr->io_producers, link, tmp) {
+		io_msg_producer->stop(ctrlr);
+		STAILQ_REMOVE(&ctrlr->io_producers, io_msg_producer, nvme_io_msg_producer, link);
+	}
+
+	if (ctrlr->external_io_msgs) {
+		spdk_ring_free(ctrlr->external_io_msgs);
+		ctrlr->external_io_msgs = NULL;
+	}
+
+	if (ctrlr->external_io_msgs_qpair) {
+		spdk_nvme_ctrlr_free_io_qpair(ctrlr->external_io_msgs_qpair);
+		ctrlr->external_io_msgs_qpair = NULL;
+	}
+
+	pthread_mutex_destroy(&ctrlr->external_io_msgs_lock);
+}
+
+void
+nvme_io_msg_ctrlr_unregister(struct spdk_nvme_ctrlr *ctrlr,
+			     struct nvme_io_msg_producer *io_msg_producer)
+{
+	assert(io_msg_producer != NULL);
+
+	if (!nvme_io_msg_is_producer_registered(ctrlr, io_msg_producer)) {
+		return;
+	}
+
+	STAILQ_REMOVE(&ctrlr->io_producers, io_msg_producer, nvme_io_msg_producer, link);
+	if (STAILQ_EMPTY(&ctrlr->io_producers)) {
+		nvme_io_msg_ctrlr_detach(ctrlr);
+	}
+}
diff --git a/src/spdk/lib/nvme/nvme_io_msg.h b/src/spdk/lib/nvme/nvme_io_msg.h
new file mode 100644
index 000000000..9c18261d5
--- /dev/null
+++ b/src/spdk/lib/nvme/nvme_io_msg.h
@@ -0,0 +1,90 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * SPDK cuse
+ */
+
+
+#ifndef SPDK_NVME_IO_MSG_H_
+#define SPDK_NVME_IO_MSG_H_
+
+typedef void (*spdk_nvme_io_msg_fn)(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid,
+				    void *arg);
+
+struct spdk_nvme_io_msg {
+	struct spdk_nvme_ctrlr	*ctrlr;
+	uint32_t		nsid;
+
+	spdk_nvme_io_msg_fn	fn;
+	void			*arg;
+};
+
+struct nvme_io_msg_producer {
+	const char *name;
+	void (*update)(struct spdk_nvme_ctrlr *ctrlr);
+	void (*stop)(struct spdk_nvme_ctrlr *ctrlr);
+	STAILQ_ENTRY(nvme_io_msg_producer) link;
+};
+
+int nvme_io_msg_send(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, spdk_nvme_io_msg_fn fn,
+		     void *arg);
+
+/**
+ * Process IO message sent to controller from external module.
+ *
+ * This call process requests from the ring, send IO to an allocated qpair or
+ * admin commands in its context. This call is non-blocking and intended to be
+ * polled by SPDK thread to provide safe environment for NVMe request
+ * completition sent by external module to controller.
+ *
+ * The caller must ensure that each controller is polled by only one thread at
+ * a time.
+ *
+ * This function may be called at any point while the controller is attached to
+ * the SPDK NVMe driver.
+ *
+ * \param ctrlr Opaque handle to NVMe controller.
+ *
+ * \return number of processed external IO messages.
+ */
+int nvme_io_msg_process(struct spdk_nvme_ctrlr *ctrlr);
+
+int nvme_io_msg_ctrlr_register(struct spdk_nvme_ctrlr *ctrlr,
+			       struct nvme_io_msg_producer *io_msg_producer);
+void nvme_io_msg_ctrlr_unregister(struct spdk_nvme_ctrlr *ctrlr,
+				  struct nvme_io_msg_producer *io_msg_producer);
+void nvme_io_msg_ctrlr_detach(struct spdk_nvme_ctrlr *ctrlr);
+void nvme_io_msg_ctrlr_update(struct spdk_nvme_ctrlr *ctrlr);
+
+#endif /* SPDK_NVME_IO_MSG_H_ */
diff --git a/src/spdk/lib/nvme/nvme_ns.c b/src/spdk/lib/nvme/nvme_ns.c
new file mode 100644
index 000000000..5d424e5c7
--- /dev/null
+++ b/src/spdk/lib/nvme/nvme_ns.c
@@ -0,0 +1,401 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation. All rights reserved.
+ *   Copyright (c) 2020 Mellanox Technologies LTD. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "nvme_internal.h"
+
+static inline struct spdk_nvme_ns_data *
+_nvme_ns_get_data(struct spdk_nvme_ns *ns)
+{
+	return &ns->ctrlr->nsdata[ns->id - 1];
+}
+
+/**
+ * Update Namespace flags based on Identify Controller
+ * and Identify Namespace.  This can be also used for
+ * Namespace Attribute Notice events and Namespace
+ * operations such as Attach/Detach.
+ */
+void
+nvme_ns_set_identify_data(struct spdk_nvme_ns *ns)
+{
+	struct spdk_nvme_ns_data	*nsdata;
+
+	nsdata = _nvme_ns_get_data(ns);
+
+	ns->flags = 0x0000;
+
+	ns->sector_size = 1 << nsdata->lbaf[nsdata->flbas.format].lbads;
+	ns->extended_lba_size = ns->sector_size;
+
+	ns->md_size = nsdata->lbaf[nsdata->flbas.format].ms;
+	if (nsdata->flbas.extended) {
+		ns->flags |= SPDK_NVME_NS_EXTENDED_LBA_SUPPORTED;
+		ns->extended_lba_size += ns->md_size;
+	}
+
+	ns->sectors_per_max_io = spdk_nvme_ns_get_max_io_xfer_size(ns) / ns->extended_lba_size;
+
+	if (nsdata->noiob) {
+		ns->sectors_per_stripe = nsdata->noiob;
+		SPDK_DEBUGLOG(SPDK_LOG_NVME, "ns %u optimal IO boundary %" PRIu32 " blocks\n",
+			      ns->id, ns->sectors_per_stripe);
+	} else if (ns->ctrlr->quirks & NVME_INTEL_QUIRK_STRIPING &&
+		   ns->ctrlr->cdata.vs[3] != 0) {
+		ns->sectors_per_stripe = (1ULL << ns->ctrlr->cdata.vs[3]) * ns->ctrlr->min_page_size /
+					 ns->sector_size;
+		SPDK_DEBUGLOG(SPDK_LOG_NVME, "ns %u stripe size quirk %" PRIu32 " blocks\n",
+			      ns->id, ns->sectors_per_stripe);
+	} else {
+		ns->sectors_per_stripe = 0;
+	}
+
+	if (ns->ctrlr->cdata.oncs.dsm) {
+		ns->flags |= SPDK_NVME_NS_DEALLOCATE_SUPPORTED;
+	}
+
+	if (ns->ctrlr->cdata.oncs.compare) {
+		ns->flags |= SPDK_NVME_NS_COMPARE_SUPPORTED;
+	}
+
+	if (ns->ctrlr->cdata.vwc.present) {
+		ns->flags |= SPDK_NVME_NS_FLUSH_SUPPORTED;
+	}
+
+	if (ns->ctrlr->cdata.oncs.write_zeroes) {
+		ns->flags |= SPDK_NVME_NS_WRITE_ZEROES_SUPPORTED;
+	}
+
+	if (ns->ctrlr->cdata.oncs.write_unc) {
+		ns->flags |= SPDK_NVME_NS_WRITE_UNCORRECTABLE_SUPPORTED;
+	}
+
+	if (nsdata->nsrescap.raw) {
+		ns->flags |= SPDK_NVME_NS_RESERVATION_SUPPORTED;
+	}
+
+	ns->pi_type = SPDK_NVME_FMT_NVM_PROTECTION_DISABLE;
+	if (nsdata->lbaf[nsdata->flbas.format].ms && nsdata->dps.pit) {
+		ns->flags |= SPDK_NVME_NS_DPS_PI_SUPPORTED;
+		ns->pi_type = nsdata->dps.pit;
+	}
+}
+
+static int
+nvme_ctrlr_identify_ns(struct spdk_nvme_ns *ns)
+{
+	struct nvme_completion_poll_status	*status;
+	struct spdk_nvme_ns_data		*nsdata;
+	int					rc;
+
+	status = calloc(1, sizeof(*status));
+	if (!status) {
+		SPDK_ERRLOG("Failed to allocate status tracker\n");
+		return -ENOMEM;
+	}
+
+	nsdata = _nvme_ns_get_data(ns);
+	rc = nvme_ctrlr_cmd_identify(ns->ctrlr, SPDK_NVME_IDENTIFY_NS, 0, ns->id,
+				     nsdata, sizeof(*nsdata),
+				     nvme_completion_poll_cb, status);
+	if (rc != 0) {
+		free(status);
+		return rc;
+	}
+
+	if (nvme_wait_for_completion_robust_lock(ns->ctrlr->adminq, status,
+			&ns->ctrlr->ctrlr_lock)) {
+		if (!status->timed_out) {
+			free(status);
+		}
+		/* This can occur if the namespace is not active. Simply zero the
+		 * namespace data and continue. */
+		nvme_ns_destruct(ns);
+		return 0;
+	}
+	free(status);
+
+	nvme_ns_set_identify_data(ns);
+
+	return 0;
+}
+
+static int
+nvme_ctrlr_identify_id_desc(struct spdk_nvme_ns *ns)
+{
+	struct nvme_completion_poll_status      *status;
+	int                                     rc;
+
+	memset(ns->id_desc_list, 0, sizeof(ns->id_desc_list));
+
+	if (ns->ctrlr->vs.raw < SPDK_NVME_VERSION(1, 3, 0) ||
+	    (ns->ctrlr->quirks & NVME_QUIRK_IDENTIFY_CNS)) {
+		SPDK_DEBUGLOG(SPDK_LOG_NVME, "Version < 1.3; not attempting to retrieve NS ID Descriptor List\n");
+		return 0;
+	}
+
+	status = calloc(1, sizeof(*status));
+	if (!status) {
+		SPDK_ERRLOG("Failed to allocate status tracker\n");
+		return -ENOMEM;
+	}
+
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "Attempting to retrieve NS ID Descriptor List\n");
+	rc = nvme_ctrlr_cmd_identify(ns->ctrlr, SPDK_NVME_IDENTIFY_NS_ID_DESCRIPTOR_LIST, 0, ns->id,
+				     ns->id_desc_list, sizeof(ns->id_desc_list),
+				     nvme_completion_poll_cb, status);
+	if (rc < 0) {
+		free(status);
+		return rc;
+	}
+
+	rc = nvme_wait_for_completion_robust_lock(ns->ctrlr->adminq, status, &ns->ctrlr->ctrlr_lock);
+	if (rc != 0) {
+		SPDK_WARNLOG("Failed to retrieve NS ID Descriptor List\n");
+		memset(ns->id_desc_list, 0, sizeof(ns->id_desc_list));
+	}
+
+	if (!status->timed_out) {
+		free(status);
+	}
+
+	return rc;
+}
+
+uint32_t
+spdk_nvme_ns_get_id(struct spdk_nvme_ns *ns)
+{
+	return ns->id;
+}
+
+bool
+spdk_nvme_ns_is_active(struct spdk_nvme_ns *ns)
+{
+	const struct spdk_nvme_ns_data *nsdata = NULL;
+
+	/*
+	 * According to the spec, valid NS has non-zero id.
+	 */
+	if (ns->id == 0) {
+		return false;
+	}
+
+	nsdata = _nvme_ns_get_data(ns);
+
+	/*
+	 * According to the spec, Identify Namespace will return a zero-filled structure for
+	 *  inactive namespace IDs.
+	 * Check NCAP since it must be nonzero for an active namespace.
+	 */
+	return nsdata->ncap != 0;
+}
+
+struct spdk_nvme_ctrlr *
+spdk_nvme_ns_get_ctrlr(struct spdk_nvme_ns *ns)
+{
+	return ns->ctrlr;
+}
+
+uint32_t
+spdk_nvme_ns_get_max_io_xfer_size(struct spdk_nvme_ns *ns)
+{
+	return ns->ctrlr->max_xfer_size;
+}
+
+uint32_t
+spdk_nvme_ns_get_sector_size(struct spdk_nvme_ns *ns)
+{
+	return ns->sector_size;
+}
+
+uint32_t
+spdk_nvme_ns_get_extended_sector_size(struct spdk_nvme_ns *ns)
+{
+	return ns->extended_lba_size;
+}
+
+uint64_t
+spdk_nvme_ns_get_num_sectors(struct spdk_nvme_ns *ns)
+{
+	return _nvme_ns_get_data(ns)->nsze;
+}
+
+uint64_t
+spdk_nvme_ns_get_size(struct spdk_nvme_ns *ns)
+{
+	return spdk_nvme_ns_get_num_sectors(ns) * spdk_nvme_ns_get_sector_size(ns);
+}
+
+uint32_t
+spdk_nvme_ns_get_flags(struct spdk_nvme_ns *ns)
+{
+	return ns->flags;
+}
+
+enum spdk_nvme_pi_type
+spdk_nvme_ns_get_pi_type(struct spdk_nvme_ns *ns) {
+	return ns->pi_type;
+}
+
+bool
+spdk_nvme_ns_supports_extended_lba(struct spdk_nvme_ns *ns)
+{
+	return (ns->flags & SPDK_NVME_NS_EXTENDED_LBA_SUPPORTED) ? true : false;
+}
+
+bool
+spdk_nvme_ns_supports_compare(struct spdk_nvme_ns *ns)
+{
+	return (ns->flags & SPDK_NVME_NS_COMPARE_SUPPORTED) ? true : false;
+}
+
+uint32_t
+spdk_nvme_ns_get_md_size(struct spdk_nvme_ns *ns)
+{
+	return ns->md_size;
+}
+
+const struct spdk_nvme_ns_data *
+spdk_nvme_ns_get_data(struct spdk_nvme_ns *ns)
+{
+	return _nvme_ns_get_data(ns);
+}
+
+enum spdk_nvme_dealloc_logical_block_read_value spdk_nvme_ns_get_dealloc_logical_block_read_value(
+	struct spdk_nvme_ns *ns)
+{
+	struct spdk_nvme_ctrlr *ctrlr = ns->ctrlr;
+	const struct spdk_nvme_ns_data *data = spdk_nvme_ns_get_data(ns);
+
+	if (ctrlr->quirks & NVME_QUIRK_READ_ZERO_AFTER_DEALLOCATE) {
+		return SPDK_NVME_DEALLOC_READ_00;
+	} else {
+		return data->dlfeat.bits.read_value;
+	}
+}
+
+uint32_t
+spdk_nvme_ns_get_optimal_io_boundary(struct spdk_nvme_ns *ns)
+{
+	return ns->sectors_per_stripe;
+}
+
+static const void *
+nvme_ns_find_id_desc(const struct spdk_nvme_ns *ns, enum spdk_nvme_nidt type, size_t *length)
+{
+	const struct spdk_nvme_ns_id_desc *desc;
+	size_t offset;
+
+	offset = 0;
+	while (offset + 4 < sizeof(ns->id_desc_list)) {
+		desc = (const struct spdk_nvme_ns_id_desc *)&ns->id_desc_list[offset];
+
+		if (desc->nidl == 0) {
+			/* End of list */
+			return NULL;
+		}
+
+		/*
+		 * Check if this descriptor fits within the list.
+		 * 4 is the fixed-size descriptor header (not counted in NIDL).
+		 */
+		if (offset + desc->nidl + 4 > sizeof(ns->id_desc_list)) {
+			/* Descriptor longer than remaining space in list (invalid) */
+			return NULL;
+		}
+
+		if (desc->nidt == type) {
+			*length = desc->nidl;
+			return &desc->nid[0];
+		}
+
+		offset += 4 + desc->nidl;
+	}
+
+	return NULL;
+}
+
+const struct spdk_uuid *
+spdk_nvme_ns_get_uuid(const struct spdk_nvme_ns *ns)
+{
+	const struct spdk_uuid *uuid;
+	size_t uuid_size;
+
+	uuid = nvme_ns_find_id_desc(ns, SPDK_NVME_NIDT_UUID, &uuid_size);
+	if (uuid == NULL || uuid_size != sizeof(*uuid)) {
+		return NULL;
+	}
+
+	return uuid;
+}
+
+int nvme_ns_construct(struct spdk_nvme_ns *ns, uint32_t id,
+		      struct spdk_nvme_ctrlr *ctrlr)
+{
+	int	rc;
+
+	assert(id > 0);
+
+	ns->ctrlr = ctrlr;
+	ns->id = id;
+
+	rc = nvme_ctrlr_identify_ns(ns);
+	if (rc != 0) {
+		return rc;
+	}
+
+	return nvme_ctrlr_identify_id_desc(ns);
+}
+
+void nvme_ns_destruct(struct spdk_nvme_ns *ns)
+{
+	struct spdk_nvme_ns_data *nsdata;
+
+	if (!ns->id) {
+		return;
+	}
+
+	nsdata = _nvme_ns_get_data(ns);
+	memset(nsdata, 0, sizeof(*nsdata));
+	ns->sector_size = 0;
+	ns->extended_lba_size = 0;
+	ns->md_size = 0;
+	ns->pi_type = 0;
+	ns->sectors_per_max_io = 0;
+	ns->sectors_per_stripe = 0;
+	ns->flags = 0;
+}
+
+int nvme_ns_update(struct spdk_nvme_ns *ns)
+{
+	return nvme_ctrlr_identify_ns(ns);
+}
diff --git a/src/spdk/lib/nvme/nvme_ns_cmd.c b/src/spdk/lib/nvme/nvme_ns_cmd.c
new file mode 100644
index 000000000..eaa825fa8
--- /dev/null
+++ b/src/spdk/lib/nvme/nvme_ns_cmd.c
@@ -0,0 +1,1074 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "nvme_internal.h"
+
+static inline struct nvme_request *_nvme_ns_cmd_rw(struct spdk_nvme_ns *ns,
+		struct spdk_nvme_qpair *qpair,
+		const struct nvme_payload *payload, uint32_t payload_offset, uint32_t md_offset,
+		uint64_t lba, uint32_t lba_count, spdk_nvme_cmd_cb cb_fn,
+		void *cb_arg, uint32_t opc, uint32_t io_flags,
+		uint16_t apptag_mask, uint16_t apptag, bool check_sgl);
+
+
+static bool
+nvme_ns_check_request_length(uint32_t lba_count, uint32_t sectors_per_max_io,
+			     uint32_t sectors_per_stripe, uint32_t qdepth)
+{
+	uint32_t child_per_io = UINT32_MAX;
+
+	/* After a namespace is destroyed(e.g. hotplug), all the fields associated with the
+	 * namespace will be cleared to zero, the function will return TRUE for this case,
+	 * and -EINVAL will be returned to caller.
+	 */
+	if (sectors_per_stripe > 0) {
+		child_per_io = (lba_count + sectors_per_stripe - 1) / sectors_per_stripe;
+	} else if (sectors_per_max_io > 0) {
+		child_per_io = (lba_count + sectors_per_max_io - 1) / sectors_per_max_io;
+	}
+
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "checking maximum i/o length %d\n", child_per_io);
+
+	return child_per_io >= qdepth;
+}
+
+static struct nvme_request *
+_nvme_add_child_request(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
+			const struct nvme_payload *payload,
+			uint32_t payload_offset, uint32_t md_offset,
+			uint64_t lba, uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t opc,
+			uint32_t io_flags, uint16_t apptag_mask, uint16_t apptag,
+			struct nvme_request *parent, bool check_sgl)
+{
+	struct nvme_request	*child;
+
+	child = _nvme_ns_cmd_rw(ns, qpair, payload, payload_offset, md_offset, lba, lba_count, cb_fn,
+				cb_arg, opc, io_flags, apptag_mask, apptag, check_sgl);
+	if (child == NULL) {
+		nvme_request_free_children(parent);
+		nvme_free_request(parent);
+		return NULL;
+	}
+
+	nvme_request_add_child(parent, child);
+	return child;
+}
+
+static struct nvme_request *
+_nvme_ns_cmd_split_request(struct spdk_nvme_ns *ns,
+			   struct spdk_nvme_qpair *qpair,
+			   const struct nvme_payload *payload,
+			   uint32_t payload_offset, uint32_t md_offset,
+			   uint64_t lba, uint32_t lba_count,
+			   spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t opc,
+			   uint32_t io_flags, struct nvme_request *req,
+			   uint32_t sectors_per_max_io, uint32_t sector_mask,
+			   uint16_t apptag_mask, uint16_t apptag)
+{
+	uint32_t		sector_size;
+	uint32_t		md_size = ns->md_size;
+	uint32_t		remaining_lba_count = lba_count;
+	struct nvme_request	*child;
+
+	sector_size = ns->extended_lba_size;
+
+	if ((io_flags & SPDK_NVME_IO_FLAGS_PRACT) &&
+	    (ns->flags & SPDK_NVME_NS_EXTENDED_LBA_SUPPORTED) &&
+	    (ns->flags & SPDK_NVME_NS_DPS_PI_SUPPORTED) &&
+	    (md_size == 8)) {
+		sector_size -= 8;
+	}
+
+	while (remaining_lba_count > 0) {
+		lba_count = sectors_per_max_io - (lba & sector_mask);
+		lba_count = spdk_min(remaining_lba_count, lba_count);
+
+		child = _nvme_add_child_request(ns, qpair, payload, payload_offset, md_offset,
+						lba, lba_count, cb_fn, cb_arg, opc,
+						io_flags, apptag_mask, apptag, req, true);
+		if (child == NULL) {
+			return NULL;
+		}
+
+		remaining_lba_count -= lba_count;
+		lba += lba_count;
+		payload_offset += lba_count * sector_size;
+		md_offset += lba_count * md_size;
+	}
+
+	return req;
+}
+
+static inline bool
+_is_io_flags_valid(uint32_t io_flags)
+{
+	if (io_flags & ~SPDK_NVME_IO_FLAGS_VALID_MASK) {
+		/* Invalid io_flags */
+		SPDK_ERRLOG("Invalid io_flags 0x%x\n", io_flags);
+		return false;
+	}
+
+	return true;
+}
+
+static void
+_nvme_ns_cmd_setup_request(struct spdk_nvme_ns *ns, struct nvme_request *req,
+			   uint32_t opc, uint64_t lba, uint32_t lba_count,
+			   uint32_t io_flags, uint16_t apptag_mask, uint16_t apptag)
+{
+	struct spdk_nvme_cmd	*cmd;
+
+	assert(_is_io_flags_valid(io_flags));
+
+	cmd = &req->cmd;
+	cmd->opc = opc;
+	cmd->nsid = ns->id;
+
+	*(uint64_t *)&cmd->cdw10 = lba;
+
+	if (ns->flags & SPDK_NVME_NS_DPS_PI_SUPPORTED) {
+		switch (ns->pi_type) {
+		case SPDK_NVME_FMT_NVM_PROTECTION_TYPE1:
+		case SPDK_NVME_FMT_NVM_PROTECTION_TYPE2:
+			cmd->cdw14 = (uint32_t)lba;
+			break;
+		}
+	}
+
+	cmd->fuse = (io_flags & SPDK_NVME_IO_FLAGS_FUSE_MASK);
+
+	cmd->cdw12 = lba_count - 1;
+	cmd->cdw12 |= (io_flags & SPDK_NVME_IO_FLAGS_CDW12_MASK);
+
+	cmd->cdw15 = apptag_mask;
+	cmd->cdw15 = (cmd->cdw15 << 16 | apptag);
+}
+
+static struct nvme_request *
+_nvme_ns_cmd_split_request_prp(struct spdk_nvme_ns *ns,
+			       struct spdk_nvme_qpair *qpair,
+			       const struct nvme_payload *payload,
+			       uint32_t payload_offset, uint32_t md_offset,
+			       uint64_t lba, uint32_t lba_count,
+			       spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t opc,
+			       uint32_t io_flags, struct nvme_request *req,
+			       uint16_t apptag_mask, uint16_t apptag)
+{
+	spdk_nvme_req_reset_sgl_cb reset_sgl_fn = req->payload.reset_sgl_fn;
+	spdk_nvme_req_next_sge_cb next_sge_fn = req->payload.next_sge_fn;
+	void *sgl_cb_arg = req->payload.contig_or_cb_arg;
+	bool start_valid, end_valid, last_sge, child_equals_parent;
+	uint64_t child_lba = lba;
+	uint32_t req_current_length = 0;
+	uint32_t child_length = 0;
+	uint32_t sge_length;
+	uint32_t page_size = qpair->ctrlr->page_size;
+	uintptr_t address;
+
+	reset_sgl_fn(sgl_cb_arg, payload_offset);
+	next_sge_fn(sgl_cb_arg, (void **)&address, &sge_length);
+	while (req_current_length < req->payload_size) {
+
+		if (sge_length == 0) {
+			continue;
+		} else if (req_current_length + sge_length > req->payload_size) {
+			sge_length = req->payload_size - req_current_length;
+		}
+
+		/*
+		 * The start of the SGE is invalid if the start address is not page aligned,
+		 *  unless it is the first SGE in the child request.
+		 */
+		start_valid = child_length == 0 || _is_page_aligned(address, page_size);
+
+		/* Boolean for whether this is the last SGE in the parent request. */
+		last_sge = (req_current_length + sge_length == req->payload_size);
+
+		/*
+		 * The end of the SGE is invalid if the end address is not page aligned,
+		 *  unless it is the last SGE in the parent request.
+		 */
+		end_valid = last_sge || _is_page_aligned(address + sge_length, page_size);
+
+		/*
+		 * This child request equals the parent request, meaning that no splitting
+		 *  was required for the parent request (the one passed into this function).
+		 *  In this case, we do not create a child request at all - we just send
+		 *  the original request as a single request at the end of this function.
+		 */
+		child_equals_parent = (child_length + sge_length == req->payload_size);
+
+		if (start_valid) {
+			/*
+			 * The start of the SGE is valid, so advance the length parameters,
+			 *  to include this SGE with previous SGEs for this child request
+			 *  (if any).  If it is not valid, we do not advance the length
+			 *  parameters nor get the next SGE, because we must send what has
+			 *  been collected before this SGE as a child request.
+			 */
+			child_length += sge_length;
+			req_current_length += sge_length;
+			if (req_current_length < req->payload_size) {
+				next_sge_fn(sgl_cb_arg, (void **)&address, &sge_length);
+			}
+			/*
+			 * If the next SGE is not page aligned, we will need to create a child
+			 *  request for what we have so far, and then start a new child request for
+			 *  the next SGE.
+			 */
+			start_valid = _is_page_aligned(address, page_size);
+		}
+
+		if (start_valid && end_valid && !last_sge) {
+			continue;
+		}
+
+		/*
+		 * We need to create a split here.  Send what we have accumulated so far as a child
+		 *  request.  Checking if child_equals_parent allows us to *not* create a child request
+		 *  when no splitting is required - in that case we will fall-through and just create
+		 *  a single request with no children for the entire I/O.
+		 */
+		if (!child_equals_parent) {
+			struct nvme_request *child;
+			uint32_t child_lba_count;
+
+			if ((child_length % ns->extended_lba_size) != 0) {
+				SPDK_ERRLOG("child_length %u not even multiple of lba_size %u\n",
+					    child_length, ns->extended_lba_size);
+				return NULL;
+			}
+			child_lba_count = child_length / ns->extended_lba_size;
+			/*
+			 * Note the last parameter is set to "false" - this tells the recursive
+			 *  call to _nvme_ns_cmd_rw() to not bother with checking for SGL splitting
+			 *  since we have already verified it here.
+			 */
+			child = _nvme_add_child_request(ns, qpair, payload, payload_offset, md_offset,
+							child_lba, child_lba_count,
+							cb_fn, cb_arg, opc, io_flags,
+							apptag_mask, apptag, req, false);
+			if (child == NULL) {
+				return NULL;
+			}
+			payload_offset += child_length;
+			md_offset += child_lba_count * ns->md_size;
+			child_lba += child_lba_count;
+			child_length = 0;
+		}
+	}
+
+	if (child_length == req->payload_size) {
+		/* No splitting was required, so setup the whole payload as one request. */
+		_nvme_ns_cmd_setup_request(ns, req, opc, lba, lba_count, io_flags, apptag_mask, apptag);
+	}
+
+	return req;
+}
+
+static struct nvme_request *
+_nvme_ns_cmd_split_request_sgl(struct spdk_nvme_ns *ns,
+			       struct spdk_nvme_qpair *qpair,
+			       const struct nvme_payload *payload,
+			       uint32_t payload_offset, uint32_t md_offset,
+			       uint64_t lba, uint32_t lba_count,
+			       spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t opc,
+			       uint32_t io_flags, struct nvme_request *req,
+			       uint16_t apptag_mask, uint16_t apptag)
+{
+	spdk_nvme_req_reset_sgl_cb reset_sgl_fn = req->payload.reset_sgl_fn;
+	spdk_nvme_req_next_sge_cb next_sge_fn = req->payload.next_sge_fn;
+	void *sgl_cb_arg = req->payload.contig_or_cb_arg;
+	uint64_t child_lba = lba;
+	uint32_t req_current_length = 0;
+	uint32_t child_length = 0;
+	uint32_t sge_length;
+	uint16_t max_sges, num_sges;
+	uintptr_t address;
+
+	max_sges = ns->ctrlr->max_sges;
+
+	reset_sgl_fn(sgl_cb_arg, payload_offset);
+	num_sges = 0;
+
+	while (req_current_length < req->payload_size) {
+		next_sge_fn(sgl_cb_arg, (void **)&address, &sge_length);
+
+		if (req_current_length + sge_length > req->payload_size) {
+			sge_length = req->payload_size - req_current_length;
+		}
+
+		child_length += sge_length;
+		req_current_length += sge_length;
+		num_sges++;
+
+		if (num_sges < max_sges && req_current_length < req->payload_size) {
+			continue;
+		}
+
+		/*
+		 * We need to create a split here.  Send what we have accumulated so far as a child
+		 *  request.  Checking if the child equals the full payload allows us to *not*
+		 *  create a child request when no splitting is required - in that case we will
+		 *  fall-through and just create a single request with no children for the entire I/O.
+		 */
+		if (child_length != req->payload_size) {
+			struct nvme_request *child;
+			uint32_t child_lba_count;
+
+			if ((child_length % ns->extended_lba_size) != 0) {
+				SPDK_ERRLOG("child_length %u not even multiple of lba_size %u\n",
+					    child_length, ns->extended_lba_size);
+				return NULL;
+			}
+			child_lba_count = child_length / ns->extended_lba_size;
+			/*
+			 * Note the last parameter is set to "false" - this tells the recursive
+			 *  call to _nvme_ns_cmd_rw() to not bother with checking for SGL splitting
+			 *  since we have already verified it here.
+			 */
+			child = _nvme_add_child_request(ns, qpair, payload, payload_offset, md_offset,
+							child_lba, child_lba_count,
+							cb_fn, cb_arg, opc, io_flags,
+							apptag_mask, apptag, req, false);
+			if (child == NULL) {
+				return NULL;
+			}
+			payload_offset += child_length;
+			md_offset += child_lba_count * ns->md_size;
+			child_lba += child_lba_count;
+			child_length = 0;
+			num_sges = 0;
+		}
+	}
+
+	if (child_length == req->payload_size) {
+		/* No splitting was required, so setup the whole payload as one request. */
+		_nvme_ns_cmd_setup_request(ns, req, opc, lba, lba_count, io_flags, apptag_mask, apptag);
+	}
+
+	return req;
+}
+
+static inline struct nvme_request *
+_nvme_ns_cmd_rw(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
+		const struct nvme_payload *payload, uint32_t payload_offset, uint32_t md_offset,
+		uint64_t lba, uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t opc,
+		uint32_t io_flags, uint16_t apptag_mask, uint16_t apptag, bool check_sgl)
+{
+	struct nvme_request	*req;
+	uint32_t		sector_size;
+	uint32_t		sectors_per_max_io;
+	uint32_t		sectors_per_stripe;
+
+	sector_size = ns->extended_lba_size;
+	sectors_per_max_io = ns->sectors_per_max_io;
+	sectors_per_stripe = ns->sectors_per_stripe;
+
+	if ((io_flags & SPDK_NVME_IO_FLAGS_PRACT) &&
+	    (ns->flags & SPDK_NVME_NS_EXTENDED_LBA_SUPPORTED) &&
+	    (ns->flags & SPDK_NVME_NS_DPS_PI_SUPPORTED) &&
+	    (ns->md_size == 8)) {
+		sector_size -= 8;
+	}
+
+	req = nvme_allocate_request(qpair, payload, lba_count * sector_size, lba_count * ns->md_size,
+				    cb_fn, cb_arg);
+	if (req == NULL) {
+		return NULL;
+	}
+
+	req->payload_offset = payload_offset;
+	req->md_offset = md_offset;
+
+	/*
+	 * Intel DC P3*00 NVMe controllers benefit from driver-assisted striping.
+	 * If this controller defines a stripe boundary and this I/O spans a stripe
+	 *  boundary, split the request into multiple requests and submit each
+	 *  separately to hardware.
+	 */
+	if (sectors_per_stripe > 0 &&
+	    (((lba & (sectors_per_stripe - 1)) + lba_count) > sectors_per_stripe)) {
+
+		return _nvme_ns_cmd_split_request(ns, qpair, payload, payload_offset, md_offset, lba, lba_count,
+						  cb_fn,
+						  cb_arg, opc,
+						  io_flags, req, sectors_per_stripe, sectors_per_stripe - 1, apptag_mask, apptag);
+	} else if (lba_count > sectors_per_max_io) {
+		return _nvme_ns_cmd_split_request(ns, qpair, payload, payload_offset, md_offset, lba, lba_count,
+						  cb_fn,
+						  cb_arg, opc,
+						  io_flags, req, sectors_per_max_io, 0, apptag_mask, apptag);
+	} else if (nvme_payload_type(&req->payload) == NVME_PAYLOAD_TYPE_SGL && check_sgl) {
+		if (ns->ctrlr->flags & SPDK_NVME_CTRLR_SGL_SUPPORTED) {
+			return _nvme_ns_cmd_split_request_sgl(ns, qpair, payload, payload_offset, md_offset,
+							      lba, lba_count, cb_fn, cb_arg, opc, io_flags,
+							      req, apptag_mask, apptag);
+		} else {
+			return _nvme_ns_cmd_split_request_prp(ns, qpair, payload, payload_offset, md_offset,
+							      lba, lba_count, cb_fn, cb_arg, opc, io_flags,
+							      req, apptag_mask, apptag);
+		}
+	}
+
+	_nvme_ns_cmd_setup_request(ns, req, opc, lba, lba_count, io_flags, apptag_mask, apptag);
+	return req;
+}
+
+int
+spdk_nvme_ns_cmd_compare(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, void *buffer,
+			 uint64_t lba,
+			 uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, void *cb_arg,
+			 uint32_t io_flags)
+{
+	struct nvme_request *req;
+	struct nvme_payload payload;
+
+	if (!_is_io_flags_valid(io_flags)) {
+		return -EINVAL;
+	}
+
+	payload = NVME_PAYLOAD_CONTIG(buffer, NULL);
+
+	req = _nvme_ns_cmd_rw(ns, qpair, &payload, 0, 0, lba, lba_count, cb_fn, cb_arg,
+			      SPDK_NVME_OPC_COMPARE,
+			      io_flags, 0,
+			      0, true);
+	if (req != NULL) {
+		return nvme_qpair_submit_request(qpair, req);
+	} else if (nvme_ns_check_request_length(lba_count,
+						ns->sectors_per_max_io,
+						ns->sectors_per_stripe,
+						qpair->ctrlr->opts.io_queue_requests)) {
+		return -EINVAL;
+	} else {
+		return -ENOMEM;
+	}
+}
+
+int
+spdk_nvme_ns_cmd_compare_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
+				 void *buffer,
+				 void *metadata,
+				 uint64_t lba,
+				 uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, void *cb_arg,
+				 uint32_t io_flags, uint16_t apptag_mask, uint16_t apptag)
+{
+	struct nvme_request *req;
+	struct nvme_payload payload;
+
+	if (!_is_io_flags_valid(io_flags)) {
+		return -EINVAL;
+	}
+
+	payload = NVME_PAYLOAD_CONTIG(buffer, metadata);
+
+	req = _nvme_ns_cmd_rw(ns, qpair, &payload, 0, 0, lba, lba_count, cb_fn, cb_arg,
+			      SPDK_NVME_OPC_COMPARE,
+			      io_flags,
+			      apptag_mask, apptag, true);
+	if (req != NULL) {
+		return nvme_qpair_submit_request(qpair, req);
+	} else if (nvme_ns_check_request_length(lba_count,
+						ns->sectors_per_max_io,
+						ns->sectors_per_stripe,
+						qpair->ctrlr->opts.io_queue_requests)) {
+		return -EINVAL;
+	} else {
+		return -ENOMEM;
+	}
+}
+
+int
+spdk_nvme_ns_cmd_comparev(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
+			  uint64_t lba, uint32_t lba_count,
+			  spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t io_flags,
+			  spdk_nvme_req_reset_sgl_cb reset_sgl_fn,
+			  spdk_nvme_req_next_sge_cb next_sge_fn)
+{
+	struct nvme_request *req;
+	struct nvme_payload payload;
+
+	if (!_is_io_flags_valid(io_flags)) {
+		return -EINVAL;
+	}
+
+	if (reset_sgl_fn == NULL || next_sge_fn == NULL) {
+		return -EINVAL;
+	}
+
+	payload = NVME_PAYLOAD_SGL(reset_sgl_fn, next_sge_fn, cb_arg, NULL);
+
+	req = _nvme_ns_cmd_rw(ns, qpair, &payload, 0, 0, lba, lba_count, cb_fn, cb_arg,
+			      SPDK_NVME_OPC_COMPARE,
+			      io_flags, 0, 0, true);
+	if (req != NULL) {
+		return nvme_qpair_submit_request(qpair, req);
+	} else if (nvme_ns_check_request_length(lba_count,
+						ns->sectors_per_max_io,
+						ns->sectors_per_stripe,
+						qpair->ctrlr->opts.io_queue_requests)) {
+		return -EINVAL;
+	} else {
+		return -ENOMEM;
+	}
+}
+
+int
+spdk_nvme_ns_cmd_comparev_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
+				  uint64_t lba, uint32_t lba_count,
+				  spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t io_flags,
+				  spdk_nvme_req_reset_sgl_cb reset_sgl_fn,
+				  spdk_nvme_req_next_sge_cb next_sge_fn, void *metadata,
+				  uint16_t apptag_mask, uint16_t apptag)
+{
+	struct nvme_request *req;
+	struct nvme_payload payload;
+
+	if (!_is_io_flags_valid(io_flags)) {
+		return -EINVAL;
+	}
+
+	if (reset_sgl_fn == NULL || next_sge_fn == NULL) {
+		return -EINVAL;
+	}
+
+	payload = NVME_PAYLOAD_SGL(reset_sgl_fn, next_sge_fn, cb_arg, metadata);
+
+	req = _nvme_ns_cmd_rw(ns, qpair, &payload, 0, 0, lba, lba_count, cb_fn, cb_arg,
+			      SPDK_NVME_OPC_COMPARE, io_flags, apptag_mask, apptag, true);
+	if (req != NULL) {
+		return nvme_qpair_submit_request(qpair, req);
+	} else if (nvme_ns_check_request_length(lba_count,
+						ns->sectors_per_max_io,
+						ns->sectors_per_stripe,
+						qpair->ctrlr->opts.io_queue_requests)) {
+		return -EINVAL;
+	} else {
+		return -ENOMEM;
+	}
+}
+
+int
+spdk_nvme_ns_cmd_read(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, void *buffer,
+		      uint64_t lba,
+		      uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, void *cb_arg,
+		      uint32_t io_flags)
+{
+	struct nvme_request *req;
+	struct nvme_payload payload;
+
+	if (!_is_io_flags_valid(io_flags)) {
+		return -EINVAL;
+	}
+
+	payload = NVME_PAYLOAD_CONTIG(buffer, NULL);
+
+	req = _nvme_ns_cmd_rw(ns, qpair, &payload, 0, 0, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_READ,
+			      io_flags, 0,
+			      0, true);
+	if (req != NULL) {
+		return nvme_qpair_submit_request(qpair, req);
+	} else if (nvme_ns_check_request_length(lba_count,
+						ns->sectors_per_max_io,
+						ns->sectors_per_stripe,
+						qpair->ctrlr->opts.io_queue_requests)) {
+		return -EINVAL;
+	} else {
+		return -ENOMEM;
+	}
+}
+
+int
+spdk_nvme_ns_cmd_read_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, void *buffer,
+			      void *metadata,
+			      uint64_t lba,
+			      uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, void *cb_arg,
+			      uint32_t io_flags, uint16_t apptag_mask, uint16_t apptag)
+{
+	struct nvme_request *req;
+	struct nvme_payload payload;
+
+	if (!_is_io_flags_valid(io_flags)) {
+		return -EINVAL;
+	}
+
+	payload = NVME_PAYLOAD_CONTIG(buffer, metadata);
+
+	req = _nvme_ns_cmd_rw(ns, qpair, &payload, 0, 0, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_READ,
+			      io_flags,
+			      apptag_mask, apptag, true);
+	if (req != NULL) {
+		return nvme_qpair_submit_request(qpair, req);
+	} else if (nvme_ns_check_request_length(lba_count,
+						ns->sectors_per_max_io,
+						ns->sectors_per_stripe,
+						qpair->ctrlr->opts.io_queue_requests)) {
+		return -EINVAL;
+	} else {
+		return -ENOMEM;
+	}
+}
+
+int
+spdk_nvme_ns_cmd_readv(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
+		       uint64_t lba, uint32_t lba_count,
+		       spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t io_flags,
+		       spdk_nvme_req_reset_sgl_cb reset_sgl_fn,
+		       spdk_nvme_req_next_sge_cb next_sge_fn)
+{
+	struct nvme_request *req;
+	struct nvme_payload payload;
+
+	if (!_is_io_flags_valid(io_flags)) {
+		return -EINVAL;
+	}
+
+	if (reset_sgl_fn == NULL || next_sge_fn == NULL) {
+		return -EINVAL;
+	}
+
+	payload = NVME_PAYLOAD_SGL(reset_sgl_fn, next_sge_fn, cb_arg, NULL);
+
+	req = _nvme_ns_cmd_rw(ns, qpair, &payload, 0, 0, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_READ,
+			      io_flags, 0, 0, true);
+	if (req != NULL) {
+		return nvme_qpair_submit_request(qpair, req);
+	} else if (nvme_ns_check_request_length(lba_count,
+						ns->sectors_per_max_io,
+						ns->sectors_per_stripe,
+						qpair->ctrlr->opts.io_queue_requests)) {
+		return -EINVAL;
+	} else {
+		return -ENOMEM;
+	}
+}
+
+int
+spdk_nvme_ns_cmd_readv_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
+			       uint64_t lba, uint32_t lba_count,
+			       spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t io_flags,
+			       spdk_nvme_req_reset_sgl_cb reset_sgl_fn,
+			       spdk_nvme_req_next_sge_cb next_sge_fn, void *metadata,
+			       uint16_t apptag_mask, uint16_t apptag)
+{
+	struct nvme_request *req;
+	struct nvme_payload payload;
+
+	if (!_is_io_flags_valid(io_flags)) {
+		return -EINVAL;
+	}
+
+	if (reset_sgl_fn == NULL || next_sge_fn == NULL) {
+		return -EINVAL;
+	}
+
+	payload = NVME_PAYLOAD_SGL(reset_sgl_fn, next_sge_fn, cb_arg, metadata);
+
+	req = _nvme_ns_cmd_rw(ns, qpair, &payload, 0, 0, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_READ,
+			      io_flags, apptag_mask, apptag, true);
+	if (req != NULL) {
+		return nvme_qpair_submit_request(qpair, req);
+	} else if (nvme_ns_check_request_length(lba_count,
+						ns->sectors_per_max_io,
+						ns->sectors_per_stripe,
+						qpair->ctrlr->opts.io_queue_requests)) {
+		return -EINVAL;
+	} else {
+		return -ENOMEM;
+	}
+}
+
+int
+spdk_nvme_ns_cmd_write(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
+		       void *buffer, uint64_t lba,
+		       uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, void *cb_arg,
+		       uint32_t io_flags)
+{
+	struct nvme_request *req;
+	struct nvme_payload payload;
+
+	if (!_is_io_flags_valid(io_flags)) {
+		return -EINVAL;
+	}
+
+	payload = NVME_PAYLOAD_CONTIG(buffer, NULL);
+
+	req = _nvme_ns_cmd_rw(ns, qpair, &payload, 0, 0, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_WRITE,
+			      io_flags, 0, 0, true);
+	if (req != NULL) {
+		return nvme_qpair_submit_request(qpair, req);
+	} else if (nvme_ns_check_request_length(lba_count,
+						ns->sectors_per_max_io,
+						ns->sectors_per_stripe,
+						qpair->ctrlr->opts.io_queue_requests)) {
+		return -EINVAL;
+	} else {
+		return -ENOMEM;
+	}
+}
+
+int
+spdk_nvme_ns_cmd_write_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
+			       void *buffer, void *metadata, uint64_t lba,
+			       uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, void *cb_arg,
+			       uint32_t io_flags, uint16_t apptag_mask, uint16_t apptag)
+{
+	struct nvme_request *req;
+	struct nvme_payload payload;
+
+	if (!_is_io_flags_valid(io_flags)) {
+		return -EINVAL;
+	}
+
+	payload = NVME_PAYLOAD_CONTIG(buffer, metadata);
+
+	req = _nvme_ns_cmd_rw(ns, qpair, &payload, 0, 0, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_WRITE,
+			      io_flags, apptag_mask, apptag, true);
+	if (req != NULL) {
+		return nvme_qpair_submit_request(qpair, req);
+	} else if (nvme_ns_check_request_length(lba_count,
+						ns->sectors_per_max_io,
+						ns->sectors_per_stripe,
+						qpair->ctrlr->opts.io_queue_requests)) {
+		return -EINVAL;
+	} else {
+		return -ENOMEM;
+	}
+}
+
+int
+spdk_nvme_ns_cmd_writev(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
+			uint64_t lba, uint32_t lba_count,
+			spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t io_flags,
+			spdk_nvme_req_reset_sgl_cb reset_sgl_fn,
+			spdk_nvme_req_next_sge_cb next_sge_fn)
+{
+	struct nvme_request *req;
+	struct nvme_payload payload;
+
+	if (!_is_io_flags_valid(io_flags)) {
+		return -EINVAL;
+	}
+
+	if (reset_sgl_fn == NULL || next_sge_fn == NULL) {
+		return -EINVAL;
+	}
+
+	payload = NVME_PAYLOAD_SGL(reset_sgl_fn, next_sge_fn, cb_arg, NULL);
+
+	req = _nvme_ns_cmd_rw(ns, qpair, &payload, 0, 0, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_WRITE,
+			      io_flags, 0, 0, true);
+	if (req != NULL) {
+		return nvme_qpair_submit_request(qpair, req);
+	} else if (nvme_ns_check_request_length(lba_count,
+						ns->sectors_per_max_io,
+						ns->sectors_per_stripe,
+						qpair->ctrlr->opts.io_queue_requests)) {
+		return -EINVAL;
+	} else {
+		return -ENOMEM;
+	}
+}
+
+int
+spdk_nvme_ns_cmd_writev_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
+				uint64_t lba, uint32_t lba_count,
+				spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t io_flags,
+				spdk_nvme_req_reset_sgl_cb reset_sgl_fn,
+				spdk_nvme_req_next_sge_cb next_sge_fn, void *metadata,
+				uint16_t apptag_mask, uint16_t apptag)
+{
+	struct nvme_request *req;
+	struct nvme_payload payload;
+
+	if (!_is_io_flags_valid(io_flags)) {
+		return -EINVAL;
+	}
+
+	if (reset_sgl_fn == NULL || next_sge_fn == NULL) {
+		return -EINVAL;
+	}
+
+	payload = NVME_PAYLOAD_SGL(reset_sgl_fn, next_sge_fn, cb_arg, metadata);
+
+	req = _nvme_ns_cmd_rw(ns, qpair, &payload, 0, 0, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_WRITE,
+			      io_flags, apptag_mask, apptag, true);
+	if (req != NULL) {
+		return nvme_qpair_submit_request(qpair, req);
+	} else if (nvme_ns_check_request_length(lba_count,
+						ns->sectors_per_max_io,
+						ns->sectors_per_stripe,
+						qpair->ctrlr->opts.io_queue_requests)) {
+		return -EINVAL;
+	} else {
+		return -ENOMEM;
+	}
+}
+
+int
+spdk_nvme_ns_cmd_write_zeroes(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
+			      uint64_t lba, uint32_t lba_count,
+			      spdk_nvme_cmd_cb cb_fn, void *cb_arg,
+			      uint32_t io_flags)
+{
+	struct nvme_request	*req;
+	struct spdk_nvme_cmd	*cmd;
+	uint64_t		*tmp_lba;
+
+	if (!_is_io_flags_valid(io_flags)) {
+		return -EINVAL;
+	}
+
+	if (lba_count == 0 || lba_count > UINT16_MAX + 1) {
+		return -EINVAL;
+	}
+
+	req = nvme_allocate_request_null(qpair, cb_fn, cb_arg);
+	if (req == NULL) {
+		return -ENOMEM;
+	}
+
+	cmd = &req->cmd;
+	cmd->opc = SPDK_NVME_OPC_WRITE_ZEROES;
+	cmd->nsid = ns->id;
+
+	tmp_lba = (uint64_t *)&cmd->cdw10;
+	*tmp_lba = lba;
+	cmd->cdw12 = lba_count - 1;
+	cmd->fuse = (io_flags & SPDK_NVME_IO_FLAGS_FUSE_MASK);
+	cmd->cdw12 |= (io_flags & SPDK_NVME_IO_FLAGS_CDW12_MASK);
+
+	return nvme_qpair_submit_request(qpair, req);
+}
+
+int
+spdk_nvme_ns_cmd_write_uncorrectable(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
+				     uint64_t lba, uint32_t lba_count,
+				     spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	struct nvme_request	*req;
+	struct spdk_nvme_cmd	*cmd;
+	uint64_t		*tmp_lba;
+
+	if (lba_count == 0 || lba_count > UINT16_MAX + 1) {
+		return -EINVAL;
+	}
+
+	req = nvme_allocate_request_null(qpair, cb_fn, cb_arg);
+	if (req == NULL) {
+		return -ENOMEM;
+	}
+
+	cmd = &req->cmd;
+	cmd->opc = SPDK_NVME_OPC_WRITE_UNCORRECTABLE;
+	cmd->nsid = ns->id;
+
+	tmp_lba = (uint64_t *)&cmd->cdw10;
+	*tmp_lba = lba;
+	cmd->cdw12 = lba_count - 1;
+
+	return nvme_qpair_submit_request(qpair, req);
+}
+
+int
+spdk_nvme_ns_cmd_dataset_management(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
+				    uint32_t type,
+				    const struct spdk_nvme_dsm_range *ranges, uint16_t num_ranges,
+				    spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	struct nvme_request	*req;
+	struct spdk_nvme_cmd	*cmd;
+
+	if (num_ranges == 0 || num_ranges > SPDK_NVME_DATASET_MANAGEMENT_MAX_RANGES) {
+		return -EINVAL;
+	}
+
+	if (ranges == NULL) {
+		return -EINVAL;
+	}
+
+	req = nvme_allocate_request_user_copy(qpair, (void *)ranges,
+					      num_ranges * sizeof(struct spdk_nvme_dsm_range),
+					      cb_fn, cb_arg, true);
+	if (req == NULL) {
+		return -ENOMEM;
+	}
+
+	cmd = &req->cmd;
+	cmd->opc = SPDK_NVME_OPC_DATASET_MANAGEMENT;
+	cmd->nsid = ns->id;
+
+	cmd->cdw10_bits.dsm.nr = num_ranges - 1;
+	cmd->cdw11 = type;
+
+	return nvme_qpair_submit_request(qpair, req);
+}
+
+int
+spdk_nvme_ns_cmd_flush(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
+		       spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	struct nvme_request	*req;
+	struct spdk_nvme_cmd	*cmd;
+
+	req = nvme_allocate_request_null(qpair, cb_fn, cb_arg);
+	if (req == NULL) {
+		return -ENOMEM;
+	}
+
+	cmd = &req->cmd;
+	cmd->opc = SPDK_NVME_OPC_FLUSH;
+	cmd->nsid = ns->id;
+
+	return nvme_qpair_submit_request(qpair, req);
+}
+
+int
+spdk_nvme_ns_cmd_reservation_register(struct spdk_nvme_ns *ns,
+				      struct spdk_nvme_qpair *qpair,
+				      struct spdk_nvme_reservation_register_data *payload,
+				      bool ignore_key,
+				      enum spdk_nvme_reservation_register_action action,
+				      enum spdk_nvme_reservation_register_cptpl cptpl,
+				      spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	struct nvme_request	*req;
+	struct spdk_nvme_cmd	*cmd;
+
+	req = nvme_allocate_request_user_copy(qpair,
+					      payload, sizeof(struct spdk_nvme_reservation_register_data),
+					      cb_fn, cb_arg, true);
+	if (req == NULL) {
+		return -ENOMEM;
+	}
+
+	cmd = &req->cmd;
+	cmd->opc = SPDK_NVME_OPC_RESERVATION_REGISTER;
+	cmd->nsid = ns->id;
+
+	cmd->cdw10_bits.resv_register.rrega = action;
+	cmd->cdw10_bits.resv_register.iekey = ignore_key;
+	cmd->cdw10_bits.resv_register.cptpl = cptpl;
+
+	return nvme_qpair_submit_request(qpair, req);
+}
+
+int
+spdk_nvme_ns_cmd_reservation_release(struct spdk_nvme_ns *ns,
+				     struct spdk_nvme_qpair *qpair,
+				     struct spdk_nvme_reservation_key_data *payload,
+				     bool ignore_key,
+				     enum spdk_nvme_reservation_release_action action,
+				     enum spdk_nvme_reservation_type type,
+				     spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	struct nvme_request	*req;
+	struct spdk_nvme_cmd	*cmd;
+
+	req = nvme_allocate_request_user_copy(qpair,
+					      payload, sizeof(struct spdk_nvme_reservation_key_data), cb_fn,
+					      cb_arg, true);
+	if (req == NULL) {
+		return -ENOMEM;
+	}
+
+	cmd = &req->cmd;
+	cmd->opc = SPDK_NVME_OPC_RESERVATION_RELEASE;
+	cmd->nsid = ns->id;
+
+	cmd->cdw10_bits.resv_release.rrela = action;
+	cmd->cdw10_bits.resv_release.iekey = ignore_key;
+	cmd->cdw10_bits.resv_release.rtype = type;
+
+	return nvme_qpair_submit_request(qpair, req);
+}
+
+int
+spdk_nvme_ns_cmd_reservation_acquire(struct spdk_nvme_ns *ns,
+				     struct spdk_nvme_qpair *qpair,
+				     struct spdk_nvme_reservation_acquire_data *payload,
+				     bool ignore_key,
+				     enum spdk_nvme_reservation_acquire_action action,
+				     enum spdk_nvme_reservation_type type,
+				     spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	struct nvme_request	*req;
+	struct spdk_nvme_cmd	*cmd;
+
+	req = nvme_allocate_request_user_copy(qpair,
+					      payload, sizeof(struct spdk_nvme_reservation_acquire_data),
+					      cb_fn, cb_arg, true);
+	if (req == NULL) {
+		return -ENOMEM;
+	}
+
+	cmd = &req->cmd;
+	cmd->opc = SPDK_NVME_OPC_RESERVATION_ACQUIRE;
+	cmd->nsid = ns->id;
+
+	cmd->cdw10_bits.resv_acquire.racqa = action;
+	cmd->cdw10_bits.resv_acquire.iekey = ignore_key;
+	cmd->cdw10_bits.resv_acquire.rtype = type;
+
+	return nvme_qpair_submit_request(qpair, req);
+}
+
+int
+spdk_nvme_ns_cmd_reservation_report(struct spdk_nvme_ns *ns,
+				    struct spdk_nvme_qpair *qpair,
+				    void *payload, uint32_t len,
+				    spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	uint32_t		num_dwords;
+	struct nvme_request	*req;
+	struct spdk_nvme_cmd	*cmd;
+
+	if (len % 4) {
+		return -EINVAL;
+	}
+	num_dwords = len / 4;
+
+	req = nvme_allocate_request_user_copy(qpair, payload, len, cb_fn, cb_arg, false);
+	if (req == NULL) {
+		return -ENOMEM;
+	}
+
+	cmd = &req->cmd;
+	cmd->opc = SPDK_NVME_OPC_RESERVATION_REPORT;
+	cmd->nsid = ns->id;
+
+	cmd->cdw10 = num_dwords;
+
+	return nvme_qpair_submit_request(qpair, req);
+}
diff --git a/src/spdk/lib/nvme/nvme_ns_ocssd_cmd.c b/src/spdk/lib/nvme/nvme_ns_ocssd_cmd.c
new file mode 100644
index 000000000..f60aa6789
--- /dev/null
+++ b/src/spdk/lib/nvme/nvme_ns_ocssd_cmd.c
@@ -0,0 +1,233 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/nvme_ocssd.h"
+#include "nvme_internal.h"
+
+int
+spdk_nvme_ocssd_ns_cmd_vector_reset(struct spdk_nvme_ns *ns,
+				    struct spdk_nvme_qpair *qpair,
+				    uint64_t *lba_list, uint32_t num_lbas,
+				    struct spdk_ocssd_chunk_information_entry *chunk_info,
+				    spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	struct nvme_request	*req;
+	struct spdk_nvme_cmd	*cmd;
+
+	if (!lba_list || (num_lbas == 0) ||
+	    (num_lbas > SPDK_NVME_OCSSD_MAX_LBAL_ENTRIES)) {
+		return -EINVAL;
+	}
+
+	req = nvme_allocate_request_null(qpair, cb_fn, cb_arg);
+	if (req == NULL) {
+		return -ENOMEM;
+	}
+
+	cmd = &req->cmd;
+	cmd->opc = SPDK_OCSSD_OPC_VECTOR_RESET;
+	cmd->nsid = ns->id;
+
+	if (chunk_info != NULL) {
+		cmd->mptr = spdk_vtophys(chunk_info, NULL);
+	}
+
+	/*
+	 * Dword 10 and 11 store a pointer to the list of logical block addresses.
+	 * If there is a single entry in the LBA list, the logical block
+	 * address should be stored instead.
+	 */
+	if (num_lbas == 1) {
+		*(uint64_t *)&cmd->cdw10 = *lba_list;
+	} else {
+		*(uint64_t *)&cmd->cdw10 = spdk_vtophys(lba_list, NULL);
+	}
+
+	cmd->cdw12 = num_lbas - 1;
+
+	return nvme_qpair_submit_request(qpair, req);
+}
+
+static int
+_nvme_ocssd_ns_cmd_vector_rw_with_md(struct spdk_nvme_ns *ns,
+				     struct spdk_nvme_qpair *qpair,
+				     void *buffer, void *metadata,
+				     uint64_t *lba_list, uint32_t num_lbas,
+				     spdk_nvme_cmd_cb cb_fn, void *cb_arg,
+				     enum spdk_ocssd_io_opcode opc,
+				     uint32_t io_flags)
+{
+	struct nvme_request	*req;
+	struct spdk_nvme_cmd	*cmd;
+	struct nvme_payload	payload;
+	uint32_t valid_flags = SPDK_OCSSD_IO_FLAGS_LIMITED_RETRY;
+
+	if (io_flags & ~valid_flags) {
+		return -EINVAL;
+	}
+
+	if (!buffer || !lba_list || (num_lbas == 0) ||
+	    (num_lbas > SPDK_NVME_OCSSD_MAX_LBAL_ENTRIES)) {
+		return -EINVAL;
+	}
+
+	payload = NVME_PAYLOAD_CONTIG(buffer, metadata);
+
+	req = nvme_allocate_request(qpair, &payload, num_lbas * ns->sector_size, num_lbas * ns->md_size,
+				    cb_fn, cb_arg);
+	if (req == NULL) {
+		return -ENOMEM;
+	}
+
+	cmd = &req->cmd;
+	cmd->opc = opc;
+	cmd->nsid = ns->id;
+
+	/*
+	 * Dword 10 and 11 store a pointer to the list of logical block addresses.
+	 * If there is a single entry in the LBA list, the logical block
+	 * address should be stored instead.
+	 */
+	if (num_lbas == 1) {
+		*(uint64_t *)&cmd->cdw10 = *lba_list;
+	} else {
+		*(uint64_t *)&cmd->cdw10 = spdk_vtophys(lba_list, NULL);
+	}
+
+	cmd->cdw12 = num_lbas - 1;
+	cmd->cdw12 |= io_flags;
+
+	return nvme_qpair_submit_request(qpair, req);
+}
+
+int
+spdk_nvme_ocssd_ns_cmd_vector_write_with_md(struct spdk_nvme_ns *ns,
+		struct spdk_nvme_qpair *qpair,
+		void *buffer, void *metadata,
+		uint64_t *lba_list, uint32_t num_lbas,
+		spdk_nvme_cmd_cb cb_fn, void *cb_arg,
+		uint32_t io_flags)
+{
+	return _nvme_ocssd_ns_cmd_vector_rw_with_md(ns, qpair, buffer, metadata, lba_list,
+			num_lbas, cb_fn, cb_arg, SPDK_OCSSD_OPC_VECTOR_WRITE, io_flags);
+}
+
+int
+spdk_nvme_ocssd_ns_cmd_vector_write(struct spdk_nvme_ns *ns,
+				    struct spdk_nvme_qpair *qpair,
+				    void *buffer,
+				    uint64_t *lba_list, uint32_t num_lbas,
+				    spdk_nvme_cmd_cb cb_fn, void *cb_arg,
+				    uint32_t io_flags)
+{
+	return _nvme_ocssd_ns_cmd_vector_rw_with_md(ns, qpair, buffer, NULL, lba_list,
+			num_lbas, cb_fn, cb_arg, SPDK_OCSSD_OPC_VECTOR_WRITE, io_flags);
+}
+
+int
+spdk_nvme_ocssd_ns_cmd_vector_read_with_md(struct spdk_nvme_ns *ns,
+		struct spdk_nvme_qpair *qpair,
+		void *buffer, void *metadata,
+		uint64_t *lba_list, uint32_t num_lbas,
+		spdk_nvme_cmd_cb cb_fn, void *cb_arg,
+		uint32_t io_flags)
+{
+	return _nvme_ocssd_ns_cmd_vector_rw_with_md(ns, qpair, buffer, metadata, lba_list,
+			num_lbas, cb_fn, cb_arg, SPDK_OCSSD_OPC_VECTOR_READ, io_flags);
+}
+
+int
+spdk_nvme_ocssd_ns_cmd_vector_read(struct spdk_nvme_ns *ns,
+				   struct spdk_nvme_qpair *qpair,
+				   void *buffer,
+				   uint64_t *lba_list, uint32_t num_lbas,
+				   spdk_nvme_cmd_cb cb_fn, void *cb_arg,
+				   uint32_t io_flags)
+{
+	return _nvme_ocssd_ns_cmd_vector_rw_with_md(ns, qpair, buffer, NULL, lba_list,
+			num_lbas, cb_fn, cb_arg, SPDK_OCSSD_OPC_VECTOR_READ, io_flags);
+}
+
+int
+spdk_nvme_ocssd_ns_cmd_vector_copy(struct spdk_nvme_ns *ns,
+				   struct spdk_nvme_qpair *qpair,
+				   uint64_t *dst_lba_list,
+				   uint64_t *src_lba_list,
+				   uint32_t num_lbas,
+				   spdk_nvme_cmd_cb cb_fn, void *cb_arg,
+				   uint32_t io_flags)
+{
+	struct nvme_request	*req;
+	struct spdk_nvme_cmd	*cmd;
+
+	uint32_t valid_flags = SPDK_OCSSD_IO_FLAGS_LIMITED_RETRY;
+
+	if (io_flags & ~valid_flags) {
+		return -EINVAL;
+	}
+
+	if (!dst_lba_list || !src_lba_list || (num_lbas == 0) ||
+	    (num_lbas > SPDK_NVME_OCSSD_MAX_LBAL_ENTRIES)) {
+		return -EINVAL;
+	}
+
+	req = nvme_allocate_request_null(qpair, cb_fn, cb_arg);
+	if (req == NULL) {
+		return -ENOMEM;
+	}
+
+	cmd = &req->cmd;
+	cmd->opc = SPDK_OCSSD_OPC_VECTOR_COPY;
+	cmd->nsid = ns->id;
+
+	/*
+	 * Dword 10 and 11 store a pointer to the list of source logical
+	 * block addresses.
+	 * Dword 14 and 15 store a pointer to the list of destination logical
+	 * block addresses.
+	 * If there is a single entry in the LBA list, the logical block
+	 * address should be stored instead.
+	 */
+	if (num_lbas == 1) {
+		*(uint64_t *)&cmd->cdw10 = *src_lba_list;
+		*(uint64_t *)&cmd->cdw14 = *dst_lba_list;
+	} else {
+		*(uint64_t *)&cmd->cdw10 = spdk_vtophys(src_lba_list, NULL);
+		*(uint64_t *)&cmd->cdw14 = spdk_vtophys(dst_lba_list, NULL);
+	}
+
+	cmd->cdw12 = num_lbas - 1;
+	cmd->cdw12 |= io_flags;
+
+	return nvme_qpair_submit_request(qpair, req);
+}
diff --git a/src/spdk/lib/nvme/nvme_opal.c b/src/spdk/lib/nvme/nvme_opal.c
new file mode 100644
index 000000000..e0a3aa7fa
--- /dev/null
+++ b/src/spdk/lib/nvme/nvme_opal.c
@@ -0,0 +1,2566 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#include "spdk/opal.h"
+#include "spdk_internal/log.h"
+#include "spdk/util.h"
+
+#include "nvme_opal_internal.h"
+
+static void
+opal_nvme_security_recv_done(void *arg, const struct spdk_nvme_cpl *cpl)
+{
+	struct opal_session *sess = arg;
+	struct spdk_opal_dev *dev = sess->dev;
+	void *response = sess->resp;
+	struct spdk_opal_compacket *header = response;
+	int ret;
+
+	if (spdk_nvme_cpl_is_error(cpl)) {
+		sess->sess_cb(sess, -EIO, sess->cb_arg);
+		return;
+	}
+
+	if (!header->outstanding_data && !header->min_transfer) {
+		sess->sess_cb(sess, 0, sess->cb_arg);
+		return;
+	}
+
+	memset(response, 0, IO_BUFFER_LENGTH);
+	ret = spdk_nvme_ctrlr_cmd_security_receive(dev->ctrlr, SPDK_SCSI_SECP_TCG,
+			dev->comid, 0, sess->resp, IO_BUFFER_LENGTH,
+			opal_nvme_security_recv_done, sess);
+	if (ret) {
+		sess->sess_cb(sess, ret, sess->cb_arg);
+	}
+}
+
+static void
+opal_nvme_security_send_done(void *arg, const struct spdk_nvme_cpl *cpl)
+{
+	struct opal_session *sess = arg;
+	struct spdk_opal_dev *dev = sess->dev;
+	int ret;
+
+	if (spdk_nvme_cpl_is_error(cpl)) {
+		sess->sess_cb(sess, -EIO, sess->cb_arg);
+		return;
+	}
+
+	ret = spdk_nvme_ctrlr_cmd_security_receive(dev->ctrlr, SPDK_SCSI_SECP_TCG,
+			dev->comid, 0, sess->resp, IO_BUFFER_LENGTH,
+			opal_nvme_security_recv_done, sess);
+	if (ret) {
+		sess->sess_cb(sess, ret, sess->cb_arg);
+	}
+}
+
+static int
+opal_nvme_security_send(struct spdk_opal_dev *dev, struct opal_session *sess,
+			opal_sess_cb sess_cb, void *cb_arg)
+{
+	sess->sess_cb = sess_cb;
+	sess->cb_arg = cb_arg;
+
+	return spdk_nvme_ctrlr_cmd_security_send(dev->ctrlr, SPDK_SCSI_SECP_TCG, dev->comid,
+			0, sess->cmd, IO_BUFFER_LENGTH,
+			opal_nvme_security_send_done, sess);
+}
+
+static void
+opal_send_recv_done(struct opal_session *sess, int status, void *ctx)
+{
+	sess->status = status;
+	sess->done = true;
+}
+
+static int
+opal_send_recv(struct spdk_opal_dev *dev, struct opal_session *sess)
+{
+	int ret;
+
+	sess->done = false;
+	ret = opal_nvme_security_send(dev, sess, opal_send_recv_done, NULL);
+	if (ret) {
+		return ret;
+	}
+
+	while (!sess->done) {
+		spdk_nvme_ctrlr_process_admin_completions(dev->ctrlr);
+	}
+
+	return sess->status;
+}
+
+static struct opal_session *
+opal_alloc_session(struct spdk_opal_dev *dev)
+{
+	struct opal_session *sess;
+
+	sess = calloc(1, sizeof(*sess));
+	if (!sess) {
+		return NULL;
+	}
+	sess->dev = dev;
+
+	return sess;
+}
+
+static void
+opal_add_token_u8(int *err, struct opal_session *sess, uint8_t token)
+{
+	if (*err) {
+		return;
+	}
+	if (sess->cmd_pos >= IO_BUFFER_LENGTH - 1) {
+		SPDK_ERRLOG("Error adding u8: end of buffer.\n");
+		*err = -ERANGE;
+		return;
+	}
+	sess->cmd[sess->cmd_pos++] = token;
+}
+
+static void
+opal_add_short_atom_header(struct opal_session *sess, bool bytestring,
+			   bool has_sign, size_t len)
+{
+	uint8_t atom;
+	int err = 0;
+
+	atom = SPDK_SHORT_ATOM_ID;
+	atom |= bytestring ? SPDK_SHORT_ATOM_BYTESTRING_FLAG : 0;
+	atom |= has_sign ? SPDK_SHORT_ATOM_SIGN_FLAG : 0;
+	atom |= len & SPDK_SHORT_ATOM_LEN_MASK;
+
+	opal_add_token_u8(&err, sess, atom);
+}
+
+static void
+opal_add_medium_atom_header(struct opal_session *sess, bool bytestring,
+			    bool has_sign, size_t len)
+{
+	uint8_t header;
+
+	header = SPDK_MEDIUM_ATOM_ID;
+	header |= bytestring ? SPDK_MEDIUM_ATOM_BYTESTRING_FLAG : 0;
+	header |= has_sign ? SPDK_MEDIUM_ATOM_SIGN_FLAG : 0;
+	header |= (len >> 8) & SPDK_MEDIUM_ATOM_LEN_MASK;
+	sess->cmd[sess->cmd_pos++] = header;
+	sess->cmd[sess->cmd_pos++] = len;
+}
+
+static void
+opal_add_token_bytestring(int *err, struct opal_session *sess,
+			  const uint8_t *bytestring, size_t len)
+{
+	size_t header_len = 1;
+	bool is_short_atom = true;
+
+	if (*err) {
+		return;
+	}
+
+	if (len & ~SPDK_SHORT_ATOM_LEN_MASK) {
+		header_len = 2;
+		is_short_atom = false;
+	}
+
+	if (len >= IO_BUFFER_LENGTH - sess->cmd_pos - header_len) {
+		SPDK_ERRLOG("Error adding bytestring: end of buffer.\n");
+		*err = -ERANGE;
+		return;
+	}
+
+	if (is_short_atom) {
+		opal_add_short_atom_header(sess, true, false, len);
+	} else {
+		opal_add_medium_atom_header(sess, true, false, len);
+	}
+
+	memcpy(&sess->cmd[sess->cmd_pos], bytestring, len);
+	sess->cmd_pos += len;
+}
+
+static void
+opal_add_token_u64(int *err, struct opal_session *sess, uint64_t number)
+{
+	int startat = 0;
+
+	if (*err) {
+		return;
+	}
+
+	/* add header first */
+	if (number <= SPDK_TINY_ATOM_DATA_MASK) {
+		sess->cmd[sess->cmd_pos++] = (uint8_t) number & SPDK_TINY_ATOM_DATA_MASK;
+	} else {
+		if (number < 0x100) {
+			sess->cmd[sess->cmd_pos++] = 0x81; /* short atom, 1 byte length */
+			startat = 0;
+		} else if (number < 0x10000) {
+			sess->cmd[sess->cmd_pos++] = 0x82; /* short atom, 2 byte length */
+			startat = 1;
+		} else if (number < 0x100000000) {
+			sess->cmd[sess->cmd_pos++] = 0x84; /* short atom, 4 byte length */
+			startat = 3;
+		} else {
+			sess->cmd[sess->cmd_pos++] = 0x88; /* short atom, 8 byte length */
+			startat = 7;
+		}
+
+		/* add number value */
+		for (int i = startat; i > -1; i--) {
+			sess->cmd[sess->cmd_pos++] = (uint8_t)((number >> (i * 8)) & 0xff);
+		}
+	}
+}
+
+static void
+opal_add_tokens(int *err, struct opal_session *sess, int num, ...)
+{
+	int i;
+	va_list args_ptr;
+	enum spdk_opal_token tmp;
+
+	va_start(args_ptr, num);
+
+	for (i = 0; i < num; i++) {
+		tmp = va_arg(args_ptr, enum spdk_opal_token);
+		opal_add_token_u8(err, sess, tmp);
+		if (*err != 0) { break; }
+	}
+
+	va_end(args_ptr);
+}
+
+static int
+opal_cmd_finalize(struct opal_session *sess, uint32_t hsn, uint32_t tsn, bool eod)
+{
+	struct spdk_opal_header *hdr;
+	int err = 0;
+
+	if (eod) {
+		opal_add_tokens(&err, sess, 6, SPDK_OPAL_ENDOFDATA,
+				SPDK_OPAL_STARTLIST,
+				0, 0, 0,
+				SPDK_OPAL_ENDLIST);
+	}
+
+	if (err) {
+		SPDK_ERRLOG("Error finalizing command.\n");
+		return -EFAULT;
+	}
+
+	hdr = (struct spdk_opal_header *)sess->cmd;
+
+	to_be32(&hdr->packet.session_tsn, tsn);
+	to_be32(&hdr->packet.session_hsn, hsn);
+
+	to_be32(&hdr->sub_packet.length, sess->cmd_pos - sizeof(*hdr));
+	while (sess->cmd_pos % 4) {
+		if (sess->cmd_pos >= IO_BUFFER_LENGTH) {
+			SPDK_ERRLOG("Error: Buffer overrun\n");
+			return -ERANGE;
+		}
+		sess->cmd[sess->cmd_pos++] = 0;
+	}
+	to_be32(&hdr->packet.length, sess->cmd_pos - sizeof(hdr->com_packet) -
+		sizeof(hdr->packet));
+	to_be32(&hdr->com_packet.length, sess->cmd_pos - sizeof(hdr->com_packet));
+
+	return 0;
+}
+
+static size_t
+opal_response_parse_tiny(struct spdk_opal_resp_token *token,
+			 const uint8_t *pos)
+{
+	token->pos = pos;
+	token->len = 1;
+	token->width = OPAL_WIDTH_TINY;
+
+	if (pos[0] & SPDK_TINY_ATOM_SIGN_FLAG) {
+		token->type = OPAL_DTA_TOKENID_SINT;
+	} else {
+		token->type = OPAL_DTA_TOKENID_UINT;
+		token->stored.unsigned_num = pos[0] & SPDK_TINY_ATOM_DATA_MASK;
+	}
+
+	return token->len;
+}
+
+static int
+opal_response_parse_short(struct spdk_opal_resp_token *token,
+			  const uint8_t *pos)
+{
+	token->pos = pos;
+	token->len = (pos[0] & SPDK_SHORT_ATOM_LEN_MASK) + 1; /* plus 1-byte header */
+	token->width = OPAL_WIDTH_SHORT;
+
+	if (pos[0] & SPDK_SHORT_ATOM_BYTESTRING_FLAG) {
+		token->type = OPAL_DTA_TOKENID_BYTESTRING;
+	} else if (pos[0] & SPDK_SHORT_ATOM_SIGN_FLAG) {
+		token->type = OPAL_DTA_TOKENID_SINT;
+	} else {
+		uint64_t u_integer = 0;
+		size_t i, b = 0;
+
+		token->type = OPAL_DTA_TOKENID_UINT;
+		if (token->len > 9) {
+			SPDK_ERRLOG("uint64 with more than 8 bytes\n");
+			return -EINVAL;
+		}
+		for (i = token->len - 1; i > 0; i--) {
+			u_integer |= ((uint64_t)pos[i] << (8 * b));
+			b++;
+		}
+		token->stored.unsigned_num = u_integer;
+	}
+
+	return token->len;
+}
+
+static size_t
+opal_response_parse_medium(struct spdk_opal_resp_token *token,
+			   const uint8_t *pos)
+{
+	token->pos = pos;
+	token->len = (((pos[0] & SPDK_MEDIUM_ATOM_LEN_MASK) << 8) | pos[1]) + 2; /* plus 2-byte header */
+	token->width = OPAL_WIDTH_MEDIUM;
+
+	if (pos[0] & SPDK_MEDIUM_ATOM_BYTESTRING_FLAG) {
+		token->type = OPAL_DTA_TOKENID_BYTESTRING;
+	} else if (pos[0] & SPDK_MEDIUM_ATOM_SIGN_FLAG) {
+		token->type = OPAL_DTA_TOKENID_SINT;
+	} else {
+		token->type = OPAL_DTA_TOKENID_UINT;
+	}
+
+	return token->len;
+}
+
+static size_t
+opal_response_parse_long(struct spdk_opal_resp_token *token,
+			 const uint8_t *pos)
+{
+	token->pos = pos;
+	token->len = ((pos[1] << 16) | (pos[2] << 8) | pos[3]) + 4; /* plus 4-byte header */
+	token->width = OPAL_WIDTH_LONG;
+
+	if (pos[0] & SPDK_LONG_ATOM_BYTESTRING_FLAG) {
+		token->type = OPAL_DTA_TOKENID_BYTESTRING;
+	} else if (pos[0] & SPDK_LONG_ATOM_SIGN_FLAG) {
+		token->type = OPAL_DTA_TOKENID_SINT;
+	} else {
+		token->type = OPAL_DTA_TOKENID_UINT;
+	}
+
+	return token->len;
+}
+
+static size_t
+opal_response_parse_token(struct spdk_opal_resp_token *token,
+			  const uint8_t *pos)
+{
+	token->pos = pos;
+	token->len = 1;
+	token->type = OPAL_DTA_TOKENID_TOKEN;
+	token->width = OPAL_WIDTH_TOKEN;
+
+	return token->len;
+}
+
+static int
+opal_response_parse(const uint8_t *buf, size_t length,
+		    struct spdk_opal_resp_parsed *resp)
+{
+	const struct spdk_opal_header *hdr;
+	struct spdk_opal_resp_token *token_iter;
+	int num_entries = 0;
+	int total;
+	size_t token_length;
+	const uint8_t *pos;
+	uint32_t clen, plen, slen;
+
+	if (!buf || !resp) {
+		return -EINVAL;
+	}
+
+	hdr = (struct spdk_opal_header *)buf;
+	pos = buf + sizeof(*hdr);
+
+	clen = from_be32(&hdr->com_packet.length);
+	plen = from_be32(&hdr->packet.length);
+	slen = from_be32(&hdr->sub_packet.length);
+	SPDK_DEBUGLOG(SPDK_LOG_OPAL, "Response size: cp: %u, pkt: %u, subpkt: %u\n",
+		      clen, plen, slen);
+
+	if (clen == 0 || plen == 0 || slen == 0 ||
+	    slen > IO_BUFFER_LENGTH - sizeof(*hdr)) {
+		SPDK_ERRLOG("Bad header length. cp: %u, pkt: %u, subpkt: %u\n",
+			    clen, plen, slen);
+		return -EINVAL;
+	}
+
+	if (pos > buf + length) {
+		SPDK_ERRLOG("Pointer out of range\n");
+		return -EFAULT;
+	}
+
+	token_iter = resp->resp_tokens;
+	total = slen;
+
+	while (total > 0) {
+		if (pos[0] <= SPDK_TINY_ATOM_TYPE_MAX) { /* tiny atom */
+			token_length = opal_response_parse_tiny(token_iter, pos);
+		} else if (pos[0] <= SPDK_SHORT_ATOM_TYPE_MAX) { /* short atom */
+			token_length = opal_response_parse_short(token_iter, pos);
+		} else if (pos[0] <= SPDK_MEDIUM_ATOM_TYPE_MAX) { /* medium atom */
+			token_length = opal_response_parse_medium(token_iter, pos);
+		} else if (pos[0] <= SPDK_LONG_ATOM_TYPE_MAX) { /* long atom */
+			token_length = opal_response_parse_long(token_iter, pos);
+		} else { /* TOKEN */
+			token_length = opal_response_parse_token(token_iter, pos);
+		}
+
+		if (token_length <= 0) {
+			SPDK_ERRLOG("Parse response failure.\n");
+			return -EINVAL;
+		}
+
+		pos += token_length;
+		total -= token_length;
+		token_iter++;
+		num_entries++;
+
+		if (total < 0) {
+			SPDK_ERRLOG("Length not matching.\n");
+			return -EINVAL;
+		}
+	}
+
+	if (num_entries == 0) {
+		SPDK_ERRLOG("Couldn't parse response.\n");
+		return -EINVAL;
+	}
+	resp->num = num_entries;
+
+	return 0;
+}
+
+static inline bool
+opal_response_token_matches(const struct spdk_opal_resp_token *token,
+			    uint8_t match)
+{
+	if (!token ||
+	    token->type != OPAL_DTA_TOKENID_TOKEN ||
+	    token->pos[0] != match) {
+		return false;
+	}
+	return true;
+}
+
+static const struct spdk_opal_resp_token *
+opal_response_get_token(const struct spdk_opal_resp_parsed *resp, int index)
+{
+	const struct spdk_opal_resp_token *token;
+
+	if (index >= resp->num) {
+		SPDK_ERRLOG("Token number doesn't exist: %d, resp: %d\n",
+			    index, resp->num);
+		return NULL;
+	}
+
+	token = &resp->resp_tokens[index];
+	if (token->len == 0) {
+		SPDK_ERRLOG("Token length must be non-zero\n");
+		return NULL;
+	}
+
+	return token;
+}
+
+static uint64_t
+opal_response_get_u64(const struct spdk_opal_resp_parsed *resp, int index)
+{
+	if (!resp) {
+		SPDK_ERRLOG("Response is NULL\n");
+		return 0;
+	}
+
+	if (resp->resp_tokens[index].type != OPAL_DTA_TOKENID_UINT) {
+		SPDK_ERRLOG("Token is not unsigned int: %d\n",
+			    resp->resp_tokens[index].type);
+		return 0;
+	}
+
+	if (!(resp->resp_tokens[index].width == OPAL_WIDTH_TINY ||
+	      resp->resp_tokens[index].width == OPAL_WIDTH_SHORT)) {
+		SPDK_ERRLOG("Atom is not short or tiny: %d\n",
+			    resp->resp_tokens[index].width);
+		return 0;
+	}
+
+	return resp->resp_tokens[index].stored.unsigned_num;
+}
+
+static uint16_t
+opal_response_get_u16(const struct spdk_opal_resp_parsed *resp, int index)
+{
+	uint64_t i = opal_response_get_u64(resp, index);
+	if (i > 0xffffull) {
+		SPDK_ERRLOG("parse reponse u16 failed. Overflow\n");
+		return 0;
+	}
+	return (uint16_t) i;
+}
+
+static uint8_t
+opal_response_get_u8(const struct spdk_opal_resp_parsed *resp, int index)
+{
+	uint64_t i = opal_response_get_u64(resp, index);
+	if (i > 0xffull) {
+		SPDK_ERRLOG("parse reponse u8 failed. Overflow\n");
+		return 0;
+	}
+	return (uint8_t) i;
+}
+
+static size_t
+opal_response_get_string(const struct spdk_opal_resp_parsed *resp, int n,
+			 const char **store)
+{
+	uint8_t header_len;
+	struct spdk_opal_resp_token token;
+	*store = NULL;
+	if (!resp) {
+		SPDK_ERRLOG("Response is NULL\n");
+		return 0;
+	}
+
+	if (n > resp->num) {
+		SPDK_ERRLOG("Response has %d tokens. Can't access %d\n",
+			    resp->num, n);
+		return 0;
+	}
+
+	token = resp->resp_tokens[n];
+	if (token.type != OPAL_DTA_TOKENID_BYTESTRING) {
+		SPDK_ERRLOG("Token is not a byte string!\n");
+		return 0;
+	}
+
+	switch (token.width) {
+	case OPAL_WIDTH_SHORT:
+		header_len = 1;
+		break;
+	case OPAL_WIDTH_MEDIUM:
+		header_len = 2;
+		break;
+	case OPAL_WIDTH_LONG:
+		header_len = 4;
+		break;
+	default:
+		SPDK_ERRLOG("Can't get string from this Token\n");
+		return 0;
+	}
+
+	*store = token.pos + header_len;
+	return token.len - header_len;
+}
+
+static int
+opal_response_status(const struct spdk_opal_resp_parsed *resp)
+{
+	const struct spdk_opal_resp_token *tok;
+
+	/* if we get an EOS token, just return 0 */
+	tok = opal_response_get_token(resp, 0);
+	if (opal_response_token_matches(tok, SPDK_OPAL_ENDOFSESSION)) {
+		return 0;
+	}
+
+	if (resp->num < 5) {
+		return SPDK_DTAERROR_NO_METHOD_STATUS;
+	}
+
+	tok = opal_response_get_token(resp, resp->num - 5); /* the first token should be STARTLIST */
+	if (!opal_response_token_matches(tok, SPDK_OPAL_STARTLIST)) {
+		return SPDK_DTAERROR_NO_METHOD_STATUS;
+	}
+
+	tok = opal_response_get_token(resp, resp->num - 1); /* the last token should be ENDLIST */
+	if (!opal_response_token_matches(tok, SPDK_OPAL_ENDLIST)) {
+		return SPDK_DTAERROR_NO_METHOD_STATUS;
+	}
+
+	/* The second and third values in the status list are reserved, and are
+	defined in core spec to be 0x00 and 0x00 and SHOULD be ignored by the host. */
+	return (int)opal_response_get_u64(resp,
+					  resp->num - 4); /* We only need the first value in the status list. */
+}
+
+static int
+opal_parse_and_check_status(struct opal_session *sess)
+{
+	int error;
+
+	error = opal_response_parse(sess->resp, IO_BUFFER_LENGTH, &sess->parsed_resp);
+	if (error) {
+		SPDK_ERRLOG("Couldn't parse response.\n");
+		return error;
+	}
+	return opal_response_status(&sess->parsed_resp);
+}
+
+static inline void
+opal_clear_cmd(struct opal_session *sess)
+{
+	sess->cmd_pos = sizeof(struct spdk_opal_header);
+	memset(sess->cmd, 0, IO_BUFFER_LENGTH);
+}
+
+static inline void
+opal_set_comid(struct opal_session *sess, uint16_t comid)
+{
+	struct spdk_opal_header *hdr = (struct spdk_opal_header *)sess->cmd;
+
+	hdr->com_packet.comid[0] = comid >> 8;
+	hdr->com_packet.comid[1] = comid;
+	hdr->com_packet.extended_comid[0] = 0;
+	hdr->com_packet.extended_comid[1] = 0;
+}
+
+static inline int
+opal_init_key(struct spdk_opal_key *opal_key, const char *passwd)
+{
+	int len;
+
+	if (passwd == NULL || passwd[0] == '\0') {
+		SPDK_ERRLOG("Password is empty. Create key failed\n");
+		return -EINVAL;
+	}
+
+	len = strlen(passwd);
+
+	if (len >= OPAL_KEY_MAX) {
+		SPDK_ERRLOG("Password too long. Create key failed\n");
+		return -EINVAL;
+	}
+
+	opal_key->key_len = len;
+	memcpy(opal_key->key, passwd, opal_key->key_len);
+
+	return 0;
+}
+
+static void
+opal_build_locking_range(uint8_t *buffer, uint8_t locking_range)
+{
+	memcpy(buffer, spdk_opal_uid[UID_LOCKINGRANGE_GLOBAL], OPAL_UID_LENGTH);
+
+	/* global */
+	if (locking_range == 0) {
+		return;
+	}
+
+	/* non-global */
+	buffer[5] = LOCKING_RANGE_NON_GLOBAL;
+	buffer[7] = locking_range;
+}
+
+static void
+opal_check_tper(struct spdk_opal_dev *dev, const void *data)
+{
+	const struct spdk_opal_d0_tper_feat *tper = data;
+
+	dev->feat_info.tper = *tper;
+}
+
+/*
+ * check single user mode
+ */
+static bool
+opal_check_sum(struct spdk_opal_dev *dev, const void *data)
+{
+	const struct spdk_opal_d0_single_user_mode_feat *sum = data;
+	uint32_t num_locking_objects = from_be32(&sum->num_locking_objects);
+
+	if (num_locking_objects == 0) {
+		SPDK_NOTICELOG("Need at least one locking object.\n");
+		return false;
+	}
+
+	dev->feat_info.single_user = *sum;
+
+	return true;
+}
+
+static void
+opal_check_lock(struct spdk_opal_dev *dev, const void *data)
+{
+	const struct spdk_opal_d0_locking_feat *lock = data;
+
+	dev->feat_info.locking = *lock;
+}
+
+static void
+opal_check_geometry(struct spdk_opal_dev *dev, const void *data)
+{
+	const struct spdk_opal_d0_geo_feat *geo = data;
+
+	dev->feat_info.geo = *geo;
+}
+
+static void
+opal_check_datastore(struct spdk_opal_dev *dev, const void *data)
+{
+	const struct spdk_opal_d0_datastore_feat *datastore = data;
+
+	dev->feat_info.datastore = *datastore;
+}
+
+static uint16_t
+opal_get_comid_v100(struct spdk_opal_dev *dev, const void *data)
+{
+	const struct spdk_opal_d0_v100_feat *v100 = data;
+	uint16_t base_comid = from_be16(&v100->base_comid);
+
+	dev->feat_info.v100 = *v100;
+
+	return base_comid;
+}
+
+static uint16_t
+opal_get_comid_v200(struct spdk_opal_dev *dev, const void *data)
+{
+	const struct spdk_opal_d0_v200_feat *v200 = data;
+	uint16_t base_comid = from_be16(&v200->base_comid);
+
+	dev->feat_info.v200 = *v200;
+
+	return base_comid;
+}
+
+static int
+opal_discovery0_end(struct spdk_opal_dev *dev, void *payload, uint32_t payload_size)
+{
+	bool supported = false, single_user = false;
+	const struct spdk_opal_d0_hdr *hdr = (struct spdk_opal_d0_hdr *)payload;
+	struct spdk_opal_d0_feat_hdr *feat_hdr;
+	const uint8_t *epos = payload, *cpos = payload;
+	uint16_t comid = 0;
+	uint32_t hlen = from_be32(&(hdr->length));
+
+	if (hlen > payload_size - sizeof(*hdr)) {
+		SPDK_ERRLOG("Discovery length overflows buffer (%zu+%u)/%u\n",
+			    sizeof(*hdr), hlen, payload_size);
+		return -EFAULT;
+	}
+
+	epos += hlen; /* end of buffer */
+	cpos += sizeof(*hdr); /* current position on buffer */
+
+	while (cpos < epos) {
+		feat_hdr = (struct spdk_opal_d0_feat_hdr *)cpos;
+		uint16_t feat_code = from_be16(&feat_hdr->code);
+
+		switch (feat_code) {
+		case FEATURECODE_TPER:
+			opal_check_tper(dev, cpos);
+			break;
+		case FEATURECODE_SINGLEUSER:
+			single_user = opal_check_sum(dev, cpos);
+			break;
+		case FEATURECODE_GEOMETRY:
+			opal_check_geometry(dev, cpos);
+			break;
+		case FEATURECODE_LOCKING:
+			opal_check_lock(dev, cpos);
+			break;
+		case FEATURECODE_DATASTORE:
+			opal_check_datastore(dev, cpos);
+			break;
+		case FEATURECODE_OPALV100:
+			comid = opal_get_comid_v100(dev, cpos);
+			supported = true;
+			break;
+		case FEATURECODE_OPALV200:
+			comid = opal_get_comid_v200(dev, cpos);
+			supported = true;
+			break;
+		default:
+			SPDK_INFOLOG(SPDK_LOG_OPAL, "Unknow feature code: %d\n", feat_code);
+		}
+		cpos += feat_hdr->length + sizeof(*feat_hdr);
+	}
+
+	if (supported == false) {
+		SPDK_ERRLOG("Opal Not Supported.\n");
+		return -ENOTSUP;
+	}
+
+	if (single_user == false) {
+		SPDK_INFOLOG(SPDK_LOG_OPAL, "Single User Mode Not Supported\n");
+	}
+
+	dev->comid = comid;
+	return 0;
+}
+
+static int
+opal_discovery0(struct spdk_opal_dev *dev, void *payload, uint32_t payload_size)
+{
+	int ret;
+
+	ret = spdk_nvme_ctrlr_security_receive(dev->ctrlr, SPDK_SCSI_SECP_TCG, LV0_DISCOVERY_COMID,
+					       0, payload, payload_size);
+	if (ret) {
+		return ret;
+	}
+
+	return opal_discovery0_end(dev, payload, payload_size);
+}
+
+static int
+opal_end_session(struct spdk_opal_dev *dev, struct opal_session *sess, uint16_t comid)
+{
+	int err = 0;
+	int ret;
+
+	opal_clear_cmd(sess);
+	opal_set_comid(sess, comid);
+	opal_add_token_u8(&err, sess, SPDK_OPAL_ENDOFSESSION);
+
+	if (err < 0) {
+		return err;
+	}
+
+	ret = opal_cmd_finalize(sess, sess->hsn, sess->tsn, false);
+	if (ret) {
+		return ret;
+	}
+
+	ret = opal_send_recv(dev, sess);
+	if (ret) {
+		return ret;
+	}
+
+	sess->hsn = 0;
+	sess->tsn = 0;
+
+	return opal_parse_and_check_status(sess);
+}
+
+void
+spdk_opal_dev_destruct(struct spdk_opal_dev *dev)
+{
+	free(dev);
+}
+
+static int
+opal_start_session_done(struct opal_session *sess)
+{
+	uint32_t hsn, tsn;
+	int error = 0;
+
+	error = opal_parse_and_check_status(sess);
+	if (error) {
+		return error;
+	}
+
+	hsn = opal_response_get_u64(&sess->parsed_resp, 4);
+	tsn = opal_response_get_u64(&sess->parsed_resp, 5);
+
+	if (hsn == 0 && tsn == 0) {
+		SPDK_ERRLOG("Couldn't authenticate session\n");
+		return -EPERM;
+	}
+
+	sess->hsn = hsn;
+	sess->tsn = tsn;
+
+	return 0;
+}
+
+static int
+opal_start_generic_session(struct spdk_opal_dev *dev,
+			   struct opal_session *sess,
+			   enum opal_uid_enum auth,
+			   enum opal_uid_enum sp_type,
+			   const char *key,
+			   uint8_t key_len)
+{
+	uint32_t hsn;
+	int err = 0;
+	int ret;
+
+	if (key == NULL && auth != UID_ANYBODY) {
+		return OPAL_INVAL_PARAM;
+	}
+
+	opal_clear_cmd(sess);
+
+	opal_set_comid(sess, dev->comid);
+	hsn = GENERIC_HOST_SESSION_NUM;
+
+	opal_add_token_u8(&err, sess, SPDK_OPAL_CALL);
+	opal_add_token_bytestring(&err, sess, spdk_opal_uid[UID_SMUID],
+				  OPAL_UID_LENGTH);
+	opal_add_token_bytestring(&err, sess, spdk_opal_method[STARTSESSION_METHOD],
+				  OPAL_UID_LENGTH);
+	opal_add_token_u8(&err, sess, SPDK_OPAL_STARTLIST);
+	opal_add_token_u64(&err, sess, hsn);
+	opal_add_token_bytestring(&err, sess, spdk_opal_uid[sp_type], OPAL_UID_LENGTH);
+	opal_add_token_u8(&err, sess, SPDK_OPAL_TRUE); /* Write */
+
+	switch (auth) {
+	case UID_ANYBODY:
+		opal_add_token_u8(&err, sess, SPDK_OPAL_ENDLIST);
+		break;
+	case UID_ADMIN1:
+	case UID_SID:
+		opal_add_token_u8(&err, sess, SPDK_OPAL_STARTNAME);
+		opal_add_token_u8(&err, sess, 0); /* HostChallenge */
+		opal_add_token_bytestring(&err, sess, key, key_len);
+		opal_add_tokens(&err, sess, 3,    /* number of token */
+				SPDK_OPAL_ENDNAME,
+				SPDK_OPAL_STARTNAME,
+				3);/* HostSignAuth */
+		opal_add_token_bytestring(&err, sess, spdk_opal_uid[auth],
+					  OPAL_UID_LENGTH);
+		opal_add_token_u8(&err, sess, SPDK_OPAL_ENDNAME);
+		opal_add_token_u8(&err, sess, SPDK_OPAL_ENDLIST);
+		break;
+	default:
+		SPDK_ERRLOG("Cannot start Admin SP session with auth %d\n", auth);
+		return -EINVAL;
+	}
+
+	if (err) {
+		SPDK_ERRLOG("Error building start adminsp session command.\n");
+		return err;
+	}
+
+	ret = opal_cmd_finalize(sess, sess->hsn, sess->tsn, true);
+	if (ret) {
+		return ret;
+	}
+
+	ret = opal_send_recv(dev, sess);
+	if (ret) {
+		return ret;
+	}
+
+	return opal_start_session_done(sess);
+}
+
+static int
+opal_get_msid_cpin_pin_done(struct opal_session *sess,
+			    struct spdk_opal_key *opal_key)
+{
+	const char *msid_pin;
+	size_t strlen;
+	int error = 0;
+
+	error = opal_parse_and_check_status(sess);
+	if (error) {
+		return error;
+	}
+
+	strlen = opal_response_get_string(&sess->parsed_resp, 4, &msid_pin);
+	if (!msid_pin) {
+		SPDK_ERRLOG("Couldn't extract PIN from response\n");
+		return -EINVAL;
+	}
+
+	opal_key->key_len = strlen;
+	memcpy(opal_key->key, msid_pin, opal_key->key_len);
+
+	SPDK_DEBUGLOG(SPDK_LOG_OPAL, "MSID = %p\n", opal_key->key);
+	return 0;
+}
+
+static int
+opal_get_msid_cpin_pin(struct spdk_opal_dev *dev, struct opal_session *sess,
+		       struct spdk_opal_key *opal_key)
+{
+	int err = 0;
+	int ret;
+
+	opal_clear_cmd(sess);
+	opal_set_comid(sess, dev->comid);
+
+	opal_add_token_u8(&err, sess, SPDK_OPAL_CALL);
+	opal_add_token_bytestring(&err, sess, spdk_opal_uid[UID_C_PIN_MSID],
+				  OPAL_UID_LENGTH);
+	opal_add_token_bytestring(&err, sess, spdk_opal_method[GET_METHOD], OPAL_UID_LENGTH);
+
+	opal_add_tokens(&err, sess, 12, SPDK_OPAL_STARTLIST,
+			SPDK_OPAL_STARTLIST,
+			SPDK_OPAL_STARTNAME,
+			SPDK_OPAL_STARTCOLUMN,
+			SPDK_OPAL_PIN,
+			SPDK_OPAL_ENDNAME,
+			SPDK_OPAL_STARTNAME,
+			SPDK_OPAL_ENDCOLUMN,
+			SPDK_OPAL_PIN,
+			SPDK_OPAL_ENDNAME,
+			SPDK_OPAL_ENDLIST,
+			SPDK_OPAL_ENDLIST);
+
+	if (err) {
+		SPDK_ERRLOG("Error building Get MSID CPIN PIN command.\n");
+		return err;
+	}
+
+	ret = opal_cmd_finalize(sess, sess->hsn, sess->tsn, true);
+	if (ret) {
+		return ret;
+	}
+
+	ret = opal_send_recv(dev, sess);
+	if (ret) {
+		return ret;
+	}
+
+	return opal_get_msid_cpin_pin_done(sess, opal_key);
+}
+
+static int
+opal_build_generic_pw_cmd(struct opal_session *sess, uint8_t *key, size_t key_len,
+			  uint8_t *cpin_uid, struct spdk_opal_dev *dev)
+{
+	int err = 0;
+
+	opal_clear_cmd(sess);
+	opal_set_comid(sess, dev->comid);
+
+	opal_add_token_u8(&err, sess, SPDK_OPAL_CALL);
+	opal_add_token_bytestring(&err, sess, cpin_uid, OPAL_UID_LENGTH);
+	opal_add_token_bytestring(&err, sess, spdk_opal_method[SET_METHOD],
+				  OPAL_UID_LENGTH);
+
+	opal_add_tokens(&err, sess, 6,
+			SPDK_OPAL_STARTLIST,
+			SPDK_OPAL_STARTNAME,
+			SPDK_OPAL_VALUES,
+			SPDK_OPAL_STARTLIST,
+			SPDK_OPAL_STARTNAME,
+			SPDK_OPAL_PIN);
+	opal_add_token_bytestring(&err, sess, key, key_len);
+	opal_add_tokens(&err, sess, 4,
+			SPDK_OPAL_ENDNAME,
+			SPDK_OPAL_ENDLIST,
+			SPDK_OPAL_ENDNAME,
+			SPDK_OPAL_ENDLIST);
+	if (err) {
+		return err;
+	}
+
+	return opal_cmd_finalize(sess, sess->hsn, sess->tsn, true);
+}
+
+static int
+opal_get_locking_sp_lifecycle_done(struct opal_session *sess)
+{
+	uint8_t lifecycle;
+	int error = 0;
+
+	error = opal_parse_and_check_status(sess);
+	if (error) {
+		return error;
+	}
+
+	lifecycle = opal_response_get_u64(&sess->parsed_resp, 4);
+	if (lifecycle != OPAL_MANUFACTURED_INACTIVE) { /* status before activate */
+		SPDK_ERRLOG("Couldn't determine the status of the Lifecycle state\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+opal_get_locking_sp_lifecycle(struct spdk_opal_dev *dev, struct opal_session *sess)
+{
+	int err = 0;
+	int ret;
+
+	opal_clear_cmd(sess);
+	opal_set_comid(sess, dev->comid);
+
+	opal_add_token_u8(&err, sess, SPDK_OPAL_CALL);
+	opal_add_token_bytestring(&err, sess, spdk_opal_uid[UID_LOCKINGSP],
+				  OPAL_UID_LENGTH);
+	opal_add_token_bytestring(&err, sess, spdk_opal_method[GET_METHOD], OPAL_UID_LENGTH);
+
+	opal_add_tokens(&err, sess, 12, SPDK_OPAL_STARTLIST,
+			SPDK_OPAL_STARTLIST,
+			SPDK_OPAL_STARTNAME,
+			SPDK_OPAL_STARTCOLUMN,
+			SPDK_OPAL_LIFECYCLE,
+			SPDK_OPAL_ENDNAME,
+			SPDK_OPAL_STARTNAME,
+			SPDK_OPAL_ENDCOLUMN,
+			SPDK_OPAL_LIFECYCLE,
+			SPDK_OPAL_ENDNAME,
+			SPDK_OPAL_ENDLIST,
+			SPDK_OPAL_ENDLIST);
+
+	if (err) {
+		SPDK_ERRLOG("Error Building GET Lifecycle Status command\n");
+		return err;
+	}
+
+	ret = opal_cmd_finalize(sess, sess->hsn, sess->tsn, true);
+	if (ret) {
+		return ret;
+	}
+
+	ret = opal_send_recv(dev, sess);
+	if (ret) {
+		return ret;
+	}
+
+	return opal_get_locking_sp_lifecycle_done(sess);
+}
+
+static int
+opal_activate(struct spdk_opal_dev *dev, struct opal_session *sess)
+{
+	int err = 0;
+	int ret;
+
+	opal_clear_cmd(sess);
+	opal_set_comid(sess, dev->comid);
+
+	opal_add_token_u8(&err, sess, SPDK_OPAL_CALL);
+	opal_add_token_bytestring(&err, sess, spdk_opal_uid[UID_LOCKINGSP],
+				  OPAL_UID_LENGTH);
+	opal_add_token_bytestring(&err, sess, spdk_opal_method[ACTIVATE_METHOD],
+				  OPAL_UID_LENGTH);
+
+	opal_add_tokens(&err, sess, 2, SPDK_OPAL_STARTLIST, SPDK_OPAL_ENDLIST);
+
+	if (err) {
+		SPDK_ERRLOG("Error building Activate LockingSP command.\n");
+		return err;
+	}
+
+	/* TODO: Single User Mode for activatation */
+
+	ret = opal_cmd_finalize(sess, sess->hsn, sess->tsn, true);
+	if (ret) {
+		return ret;
+	}
+
+	ret = opal_send_recv(dev, sess);
+	if (ret) {
+		return ret;
+	}
+
+	return opal_parse_and_check_status(sess);
+}
+
+static int
+opal_start_auth_session(struct spdk_opal_dev *dev,
+			struct opal_session *sess,
+			enum spdk_opal_user user,
+			struct spdk_opal_key *opal_key)
+{
+	uint8_t uid_user[OPAL_UID_LENGTH];
+	int err = 0;
+	int ret;
+	uint32_t hsn = GENERIC_HOST_SESSION_NUM;
+
+	opal_clear_cmd(sess);
+	opal_set_comid(sess, dev->comid);
+
+	if (user != OPAL_ADMIN1) {
+		memcpy(uid_user, spdk_opal_uid[UID_USER1], OPAL_UID_LENGTH);
+		uid_user[7] = user;
+	} else {
+		memcpy(uid_user, spdk_opal_uid[UID_ADMIN1], OPAL_UID_LENGTH);
+	}
+
+	opal_add_token_u8(&err, sess, SPDK_OPAL_CALL);
+	opal_add_token_bytestring(&err, sess, spdk_opal_uid[UID_SMUID],
+				  OPAL_UID_LENGTH);
+	opal_add_token_bytestring(&err, sess, spdk_opal_method[STARTSESSION_METHOD],
+				  OPAL_UID_LENGTH);
+
+	opal_add_token_u8(&err, sess, SPDK_OPAL_STARTLIST);
+	opal_add_token_u64(&err, sess, hsn);
+	opal_add_token_bytestring(&err, sess, spdk_opal_uid[UID_LOCKINGSP],
+				  OPAL_UID_LENGTH);
+	opal_add_tokens(&err, sess, 3, SPDK_OPAL_TRUE, SPDK_OPAL_STARTNAME,
+			0); /* True for a Read-Write session  */
+	opal_add_token_bytestring(&err, sess, opal_key->key, opal_key->key_len);
+	opal_add_tokens(&err, sess, 3, SPDK_OPAL_ENDNAME, SPDK_OPAL_STARTNAME, 3); /* HostSignAuth */
+	opal_add_token_bytestring(&err, sess, uid_user, OPAL_UID_LENGTH);
+	opal_add_tokens(&err, sess, 2, SPDK_OPAL_ENDNAME, SPDK_OPAL_ENDLIST);
+
+	if (err) {
+		SPDK_ERRLOG("Error building STARTSESSION command.\n");
+		return err;
+	}
+
+	ret = opal_cmd_finalize(sess, sess->hsn, sess->tsn, true);
+	if (ret) {
+		return ret;
+	}
+
+	ret = opal_send_recv(dev, sess);
+	if (ret) {
+		return ret;
+	}
+
+	return opal_start_session_done(sess);
+}
+
+static int
+opal_lock_unlock_range(struct spdk_opal_dev *dev, struct opal_session *sess,
+		       enum spdk_opal_locking_range locking_range,
+		       enum spdk_opal_lock_state l_state)
+{
+	uint8_t uid_locking_range[OPAL_UID_LENGTH];
+	uint8_t read_locked, write_locked;
+	int err = 0;
+	int ret;
+
+	opal_clear_cmd(sess);
+	opal_set_comid(sess, dev->comid);
+
+	opal_build_locking_range(uid_locking_range, locking_range);
+
+	switch (l_state) {
+	case OPAL_READONLY:
+		read_locked = 0;
+		write_locked = 1;
+		break;
+	case OPAL_READWRITE:
+		read_locked = 0;
+		write_locked = 0;
+		break;
+	case OPAL_RWLOCK:
+		read_locked = 1;
+		write_locked = 1;
+		break;
+	default:
+		SPDK_ERRLOG("Tried to set an invalid locking state.\n");
+		return -EINVAL;
+	}
+
+	opal_add_token_u8(&err, sess, SPDK_OPAL_CALL);
+	opal_add_token_bytestring(&err, sess, uid_locking_range, OPAL_UID_LENGTH);
+	opal_add_token_bytestring(&err, sess, spdk_opal_method[SET_METHOD], OPAL_UID_LENGTH);
+
+	opal_add_tokens(&err, sess, 15, SPDK_OPAL_STARTLIST,
+			SPDK_OPAL_STARTNAME,
+			SPDK_OPAL_VALUES,
+			SPDK_OPAL_STARTLIST,
+			SPDK_OPAL_STARTNAME,
+			SPDK_OPAL_READLOCKED,
+			read_locked,
+			SPDK_OPAL_ENDNAME,
+			SPDK_OPAL_STARTNAME,
+			SPDK_OPAL_WRITELOCKED,
+			write_locked,
+			SPDK_OPAL_ENDNAME,
+			SPDK_OPAL_ENDLIST,
+			SPDK_OPAL_ENDNAME,
+			SPDK_OPAL_ENDLIST);
+
+	if (err) {
+		SPDK_ERRLOG("Error building SET command.\n");
+		return err;
+	}
+	ret = opal_cmd_finalize(sess, sess->hsn, sess->tsn, true);
+	if (ret) {
+		return ret;
+	}
+
+	ret = opal_send_recv(dev, sess);
+	if (ret) {
+		return ret;
+	}
+
+	return opal_parse_and_check_status(sess);
+}
+
+static int opal_generic_locking_range_enable_disable(struct spdk_opal_dev *dev,
+		struct opal_session *sess,
+		uint8_t *uid, bool read_lock_enabled, bool write_lock_enabled)
+{
+	int err = 0;
+
+	opal_add_token_u8(&err, sess, SPDK_OPAL_CALL);
+	opal_add_token_bytestring(&err, sess, uid, OPAL_UID_LENGTH);
+	opal_add_token_bytestring(&err, sess, spdk_opal_method[SET_METHOD], OPAL_UID_LENGTH);
+
+	opal_add_tokens(&err, sess, 23, SPDK_OPAL_STARTLIST,
+			SPDK_OPAL_STARTNAME,
+			SPDK_OPAL_VALUES,
+			SPDK_OPAL_STARTLIST,
+
+			SPDK_OPAL_STARTNAME,
+			SPDK_OPAL_READLOCKENABLED,
+			read_lock_enabled,
+			SPDK_OPAL_ENDNAME,
+
+			SPDK_OPAL_STARTNAME,
+			SPDK_OPAL_WRITELOCKENABLED,
+			write_lock_enabled,
+			SPDK_OPAL_ENDNAME,
+
+			SPDK_OPAL_STARTNAME,
+			SPDK_OPAL_READLOCKED,
+			0,
+			SPDK_OPAL_ENDNAME,
+
+			SPDK_OPAL_STARTNAME,
+			SPDK_OPAL_WRITELOCKED,
+			0,
+			SPDK_OPAL_ENDNAME,
+
+			SPDK_OPAL_ENDLIST,
+			SPDK_OPAL_ENDNAME,
+			SPDK_OPAL_ENDLIST);
+	if (err) {
+		SPDK_ERRLOG("Error building locking range enable/disable command.\n");
+	}
+	return err;
+}
+
+static int
+opal_setup_locking_range(struct spdk_opal_dev *dev, struct opal_session *sess,
+			 enum spdk_opal_locking_range locking_range,
+			 uint64_t range_start, uint64_t range_length,
+			 bool read_lock_enabled, bool write_lock_enabled)
+{
+	uint8_t uid_locking_range[OPAL_UID_LENGTH];
+	int err = 0;
+	int ret;
+
+	opal_clear_cmd(sess);
+	opal_set_comid(sess, dev->comid);
+
+	opal_build_locking_range(uid_locking_range, locking_range);
+
+	if (locking_range == 0) {
+		err = opal_generic_locking_range_enable_disable(dev, sess, uid_locking_range,
+				read_lock_enabled, write_lock_enabled);
+	} else {
+		opal_add_token_u8(&err, sess, SPDK_OPAL_CALL);
+		opal_add_token_bytestring(&err, sess, uid_locking_range, OPAL_UID_LENGTH);
+		opal_add_token_bytestring(&err, sess, spdk_opal_method[SET_METHOD],
+					  OPAL_UID_LENGTH);
+
+		opal_add_tokens(&err, sess, 6,
+				SPDK_OPAL_STARTLIST,
+				SPDK_OPAL_STARTNAME,
+				SPDK_OPAL_VALUES,
+				SPDK_OPAL_STARTLIST,
+				SPDK_OPAL_STARTNAME,
+				SPDK_OPAL_RANGESTART);
+		opal_add_token_u64(&err, sess, range_start);
+		opal_add_tokens(&err, sess, 3,
+				SPDK_OPAL_ENDNAME,
+				SPDK_OPAL_STARTNAME,
+				SPDK_OPAL_RANGELENGTH);
+		opal_add_token_u64(&err, sess, range_length);
+		opal_add_tokens(&err, sess, 3,
+				SPDK_OPAL_ENDNAME,
+				SPDK_OPAL_STARTNAME,
+				SPDK_OPAL_READLOCKENABLED);
+		opal_add_token_u64(&err, sess, read_lock_enabled);
+		opal_add_tokens(&err, sess, 3,
+				SPDK_OPAL_ENDNAME,
+				SPDK_OPAL_STARTNAME,
+				SPDK_OPAL_WRITELOCKENABLED);
+		opal_add_token_u64(&err, sess, write_lock_enabled);
+		opal_add_tokens(&err, sess, 4,
+				SPDK_OPAL_ENDNAME,
+				SPDK_OPAL_ENDLIST,
+				SPDK_OPAL_ENDNAME,
+				SPDK_OPAL_ENDLIST);
+	}
+	if (err) {
+		SPDK_ERRLOG("Error building Setup Locking range command.\n");
+		return err;
+
+	}
+
+	ret = opal_cmd_finalize(sess, sess->hsn, sess->tsn, true);
+	if (ret) {
+		return ret;
+	}
+
+	ret = opal_send_recv(dev, sess);
+	if (ret) {
+		return ret;
+	}
+
+	return opal_parse_and_check_status(sess);
+}
+
+static int
+opal_get_max_ranges_done(struct opal_session *sess)
+{
+	int error = 0;
+
+	error = opal_parse_and_check_status(sess);
+	if (error) {
+		return error;
+	}
+
+	/* "MaxRanges" is token 4 of response */
+	return opal_response_get_u16(&sess->parsed_resp, 4);
+}
+
+static int
+opal_get_max_ranges(struct spdk_opal_dev *dev, struct opal_session *sess)
+{
+	int err = 0;
+	int ret;
+
+	opal_clear_cmd(sess);
+	opal_set_comid(sess, dev->comid);
+
+	opal_add_token_u8(&err, sess, SPDK_OPAL_CALL);
+	opal_add_token_bytestring(&err, sess, spdk_opal_uid[UID_LOCKING_INFO_TABLE],
+				  OPAL_UID_LENGTH);
+	opal_add_token_bytestring(&err, sess, spdk_opal_method[GET_METHOD], OPAL_UID_LENGTH);
+
+	opal_add_tokens(&err, sess, 12, SPDK_OPAL_STARTLIST,
+			SPDK_OPAL_STARTLIST,
+			SPDK_OPAL_STARTNAME,
+			SPDK_OPAL_STARTCOLUMN,
+			SPDK_OPAL_MAXRANGES,
+			SPDK_OPAL_ENDNAME,
+			SPDK_OPAL_STARTNAME,
+			SPDK_OPAL_ENDCOLUMN,
+			SPDK_OPAL_MAXRANGES,
+			SPDK_OPAL_ENDNAME,
+			SPDK_OPAL_ENDLIST,
+			SPDK_OPAL_ENDLIST);
+
+	if (err) {
+		SPDK_ERRLOG("Error Building GET Lifecycle Status command\n");
+		return err;
+	}
+
+	ret = opal_cmd_finalize(sess, sess->hsn, sess->tsn, true);
+	if (ret) {
+		return ret;
+	}
+
+	ret = opal_send_recv(dev, sess);
+	if (ret) {
+		return ret;
+	}
+
+	return opal_get_max_ranges_done(sess);
+}
+
+static int
+opal_get_locking_range_info_done(struct opal_session *sess,
+				 struct spdk_opal_locking_range_info *info)
+{
+	int error = 0;
+
+	error = opal_parse_and_check_status(sess);
+	if (error) {
+		return error;
+	}
+
+	info->range_start = opal_response_get_u64(&sess->parsed_resp, 4);
+	info->range_length = opal_response_get_u64(&sess->parsed_resp, 8);
+	info->read_lock_enabled = opal_response_get_u8(&sess->parsed_resp, 12);
+	info->write_lock_enabled = opal_response_get_u8(&sess->parsed_resp, 16);
+	info->read_locked = opal_response_get_u8(&sess->parsed_resp, 20);
+	info->write_locked = opal_response_get_u8(&sess->parsed_resp, 24);
+
+	return 0;
+}
+
+static int
+opal_get_locking_range_info(struct spdk_opal_dev *dev,
+			    struct opal_session *sess,
+			    enum spdk_opal_locking_range locking_range_id)
+{
+	int err = 0;
+	int ret;
+	uint8_t uid_locking_range[OPAL_UID_LENGTH];
+	struct spdk_opal_locking_range_info *info;
+
+	opal_build_locking_range(uid_locking_range, locking_range_id);
+
+	assert(locking_range_id < SPDK_OPAL_MAX_LOCKING_RANGE);
+	info = &dev->locking_ranges[locking_range_id];
+	memset(info, 0, sizeof(*info));
+	info->locking_range_id = locking_range_id;
+
+	opal_clear_cmd(sess);
+	opal_set_comid(sess, dev->comid);
+
+	opal_add_token_u8(&err, sess, SPDK_OPAL_CALL);
+	opal_add_token_bytestring(&err, sess, uid_locking_range, OPAL_UID_LENGTH);
+	opal_add_token_bytestring(&err, sess, spdk_opal_method[GET_METHOD], OPAL_UID_LENGTH);
+
+
+	opal_add_tokens(&err, sess, 12, SPDK_OPAL_STARTLIST,
+			SPDK_OPAL_STARTLIST,
+			SPDK_OPAL_STARTNAME,
+			SPDK_OPAL_STARTCOLUMN,
+			SPDK_OPAL_RANGESTART,
+			SPDK_OPAL_ENDNAME,
+			SPDK_OPAL_STARTNAME,
+			SPDK_OPAL_ENDCOLUMN,
+			SPDK_OPAL_WRITELOCKED,
+			SPDK_OPAL_ENDNAME,
+			SPDK_OPAL_ENDLIST,
+			SPDK_OPAL_ENDLIST);
+
+	if (err) {
+		SPDK_ERRLOG("Error Building get locking range info command\n");
+		return err;
+	}
+
+	ret = opal_cmd_finalize(sess, sess->hsn, sess->tsn, true);
+	if (ret) {
+		return ret;
+	}
+
+	ret = opal_send_recv(dev, sess);
+	if (ret) {
+		return ret;
+	}
+
+	return opal_get_locking_range_info_done(sess, info);
+}
+
+static int
+opal_enable_user(struct spdk_opal_dev *dev, struct opal_session *sess,
+		 enum spdk_opal_user user)
+{
+	int err = 0;
+	int ret;
+	uint8_t uid_user[OPAL_UID_LENGTH];
+
+	memcpy(uid_user, spdk_opal_uid[UID_USER1], OPAL_UID_LENGTH);
+	uid_user[7] = user;
+
+	opal_clear_cmd(sess);
+	opal_set_comid(sess, dev->comid);
+
+	opal_add_token_u8(&err, sess, SPDK_OPAL_CALL);
+	opal_add_token_bytestring(&err, sess, uid_user, OPAL_UID_LENGTH);
+	opal_add_token_bytestring(&err, sess, spdk_opal_method[SET_METHOD], OPAL_UID_LENGTH);
+
+	opal_add_tokens(&err, sess, 11,
+			SPDK_OPAL_STARTLIST,
+			SPDK_OPAL_STARTNAME,
+			SPDK_OPAL_VALUES,
+			SPDK_OPAL_STARTLIST,
+			SPDK_OPAL_STARTNAME,
+			SPDK_OPAL_AUTH_ENABLE,
+			SPDK_OPAL_TRUE,
+			SPDK_OPAL_ENDNAME,
+			SPDK_OPAL_ENDLIST,
+			SPDK_OPAL_ENDNAME,
+			SPDK_OPAL_ENDLIST);
+
+	if (err) {
+		SPDK_ERRLOG("Error Building enable user command\n");
+		return err;
+	}
+
+	ret = opal_cmd_finalize(sess, sess->hsn, sess->tsn, true);
+	if (ret) {
+		return ret;
+	}
+
+	ret = opal_send_recv(dev, sess);
+	if (ret) {
+		return ret;
+	}
+
+	return opal_parse_and_check_status(sess);
+}
+
+static int
+opal_add_user_to_locking_range(struct spdk_opal_dev *dev,
+			       struct opal_session *sess,
+			       enum spdk_opal_user user,
+			       enum spdk_opal_locking_range locking_range,
+			       enum spdk_opal_lock_state l_state)
+{
+	int err = 0;
+	int ret;
+	uint8_t uid_user[OPAL_UID_LENGTH];
+	uint8_t uid_locking_range[OPAL_UID_LENGTH];
+
+	memcpy(uid_user, spdk_opal_uid[UID_USER1], OPAL_UID_LENGTH);
+	uid_user[7] = user;
+
+	switch (l_state) {
+	case OPAL_READONLY:
+		memcpy(uid_locking_range, spdk_opal_uid[UID_LOCKINGRANGE_ACE_RDLOCKED], OPAL_UID_LENGTH);
+		break;
+	case OPAL_READWRITE:
+		memcpy(uid_locking_range, spdk_opal_uid[UID_LOCKINGRANGE_ACE_WRLOCKED], OPAL_UID_LENGTH);
+		break;
+	default:
+		SPDK_ERRLOG("locking state should only be OPAL_READONLY or OPAL_READWRITE\n");
+		return -EINVAL;
+	}
+
+	uid_locking_range[7] = locking_range;
+
+	opal_clear_cmd(sess);
+	opal_set_comid(sess, dev->comid);
+
+	opal_add_token_u8(&err, sess, SPDK_OPAL_CALL);
+	opal_add_token_bytestring(&err, sess, uid_locking_range, OPAL_UID_LENGTH);
+	opal_add_token_bytestring(&err, sess, spdk_opal_method[SET_METHOD], OPAL_UID_LENGTH);
+
+	opal_add_tokens(&err, sess, 8,
+			SPDK_OPAL_STARTLIST,
+			SPDK_OPAL_STARTNAME,
+			SPDK_OPAL_VALUES,
+			SPDK_OPAL_STARTLIST,
+			SPDK_OPAL_STARTNAME,
+			SPDK_OPAL_BOOLEAN_EXPR,
+			SPDK_OPAL_STARTLIST,
+			SPDK_OPAL_STARTNAME);
+	opal_add_token_bytestring(&err, sess, spdk_opal_uid[UID_HALF_AUTHORITY_OBJ_REF],
+				  OPAL_UID_LENGTH / 2);
+	opal_add_token_bytestring(&err, sess, uid_user, OPAL_UID_LENGTH);
+
+	opal_add_tokens(&err, sess, 2, SPDK_OPAL_ENDNAME, SPDK_OPAL_STARTNAME);
+	opal_add_token_bytestring(&err, sess, spdk_opal_uid[UID_HALF_AUTHORITY_OBJ_REF],
+				  OPAL_UID_LENGTH / 2);
+	opal_add_token_bytestring(&err, sess, uid_user, OPAL_UID_LENGTH);
+
+	opal_add_tokens(&err, sess, 2, SPDK_OPAL_ENDNAME, SPDK_OPAL_STARTNAME);
+	opal_add_token_bytestring(&err, sess, spdk_opal_uid[UID_HALF_BOOLEAN_ACE], OPAL_UID_LENGTH / 2);
+	opal_add_tokens(&err, sess, 7,
+			SPDK_OPAL_TRUE,
+			SPDK_OPAL_ENDNAME,
+			SPDK_OPAL_ENDLIST,
+			SPDK_OPAL_ENDNAME,
+			SPDK_OPAL_ENDLIST,
+			SPDK_OPAL_ENDNAME,
+			SPDK_OPAL_ENDLIST);
+	if (err) {
+		SPDK_ERRLOG("Error building add user to locking range command\n");
+		return err;
+	}
+
+	ret = opal_cmd_finalize(sess, sess->hsn, sess->tsn, true);
+	if (ret) {
+		return ret;
+	}
+
+	ret = opal_send_recv(dev, sess);
+	if (ret) {
+		return ret;
+	}
+
+	return opal_parse_and_check_status(sess);
+}
+
+static int
+opal_new_user_passwd(struct spdk_opal_dev *dev, struct opal_session *sess,
+		     enum spdk_opal_user user,
+		     struct spdk_opal_key *opal_key)
+{
+	uint8_t uid_cpin[OPAL_UID_LENGTH];
+	int ret;
+
+	if (user == OPAL_ADMIN1) {
+		memcpy(uid_cpin, spdk_opal_uid[UID_C_PIN_ADMIN1], OPAL_UID_LENGTH);
+	} else {
+		memcpy(uid_cpin, spdk_opal_uid[UID_C_PIN_USER1], OPAL_UID_LENGTH);
+		uid_cpin[7] = user;
+	}
+
+	ret = opal_build_generic_pw_cmd(sess, opal_key->key, opal_key->key_len, uid_cpin, dev);
+	if (ret != 0) {
+		SPDK_ERRLOG("Error building set password command\n");
+		return ret;
+	}
+
+	ret = opal_send_recv(dev, sess);
+	if (ret) {
+		return ret;
+	}
+
+	return opal_parse_and_check_status(sess);
+}
+
+static int
+opal_set_sid_cpin_pin(struct spdk_opal_dev *dev, struct opal_session *sess, char *new_passwd)
+{
+	uint8_t cpin_uid[OPAL_UID_LENGTH];
+	struct spdk_opal_key opal_key = {};
+	int ret;
+
+	ret = opal_init_key(&opal_key, new_passwd);
+	if (ret != 0) {
+		return ret;
+	}
+
+	memcpy(cpin_uid, spdk_opal_uid[UID_C_PIN_SID], OPAL_UID_LENGTH);
+
+	if (opal_build_generic_pw_cmd(sess, opal_key.key, opal_key.key_len, cpin_uid, dev)) {
+		SPDK_ERRLOG("Error building Set SID cpin\n");
+		return -ERANGE;
+	}
+
+	ret = opal_send_recv(dev, sess);
+	if (ret) {
+		return ret;
+	}
+
+	return opal_parse_and_check_status(sess);
+}
+
+int
+spdk_opal_cmd_take_ownership(struct spdk_opal_dev *dev, char *new_passwd)
+{
+	int ret;
+	struct spdk_opal_key opal_key = {};
+	struct opal_session *sess;
+
+	assert(dev != NULL);
+
+	sess = opal_alloc_session(dev);
+	if (!sess) {
+		return -ENOMEM;
+	}
+
+	ret = opal_start_generic_session(dev, sess, UID_ANYBODY, UID_ADMINSP, NULL, 0);
+	if (ret) {
+		SPDK_ERRLOG("start admin SP session error %d\n", ret);
+		goto end;
+	}
+
+	ret = opal_get_msid_cpin_pin(dev, sess, &opal_key);
+	if (ret) {
+		SPDK_ERRLOG("get msid error %d\n", ret);
+		opal_end_session(dev, sess, dev->comid);
+		goto end;
+	}
+
+	ret = opal_end_session(dev, sess, dev->comid);
+	if (ret) {
+		SPDK_ERRLOG("end session error %d\n", ret);
+		goto end;
+	}
+
+	/* reuse the session structure */
+	memset(sess, 0, sizeof(*sess));
+	sess->dev = dev;
+	ret = opal_start_generic_session(dev, sess, UID_SID, UID_ADMINSP,
+					 opal_key.key, opal_key.key_len);
+	if (ret) {
+		SPDK_ERRLOG("start admin SP session error %d\n", ret);
+		goto end;
+	}
+	memset(&opal_key, 0, sizeof(struct spdk_opal_key));
+
+	ret = opal_set_sid_cpin_pin(dev, sess, new_passwd);
+	if (ret) {
+		SPDK_ERRLOG("set cpin error %d\n", ret);
+		opal_end_session(dev, sess, dev->comid);
+		goto end;
+	}
+
+	ret = opal_end_session(dev, sess, dev->comid);
+	if (ret) {
+		SPDK_ERRLOG("end session error %d\n", ret);
+	}
+
+end:
+	free(sess);
+	return ret;
+}
+
+struct spdk_opal_dev *
+	spdk_opal_dev_construct(struct spdk_nvme_ctrlr *ctrlr)
+{
+	struct spdk_opal_dev *dev;
+	void *payload;
+
+	dev = calloc(1, sizeof(*dev));
+	if (!dev) {
+		SPDK_ERRLOG("Memory allocation failed\n");
+		return NULL;
+	}
+
+	dev->ctrlr = ctrlr;
+
+	payload = calloc(1, IO_BUFFER_LENGTH);
+	if (!payload) {
+		free(dev);
+		return NULL;
+	}
+
+	if (opal_discovery0(dev, payload, IO_BUFFER_LENGTH)) {
+		SPDK_INFOLOG(SPDK_LOG_OPAL, "Opal is not supported on this device\n");
+		free(dev);
+		free(payload);
+		return NULL;
+	}
+
+	free(payload);
+	return dev;
+}
+
+static int
+opal_build_revert_tper_cmd(struct spdk_opal_dev *dev, struct opal_session *sess)
+{
+	int err = 0;
+
+	opal_clear_cmd(sess);
+	opal_set_comid(sess, dev->comid);
+
+	opal_add_token_u8(&err, sess, SPDK_OPAL_CALL);
+	opal_add_token_bytestring(&err, sess, spdk_opal_uid[UID_ADMINSP],
+				  OPAL_UID_LENGTH);
+	opal_add_token_bytestring(&err, sess, spdk_opal_method[REVERT_METHOD],
+				  OPAL_UID_LENGTH);
+	opal_add_token_u8(&err, sess, SPDK_OPAL_STARTLIST);
+	opal_add_token_u8(&err, sess, SPDK_OPAL_ENDLIST);
+	if (err) {
+		SPDK_ERRLOG("Error building REVERT TPER command.\n");
+		return -ERANGE;
+	}
+
+	return opal_cmd_finalize(sess, sess->hsn, sess->tsn, true);
+}
+
+static int
+opal_gen_new_active_key(struct spdk_opal_dev *dev, struct opal_session *sess,
+			struct spdk_opal_key *active_key)
+{
+	uint8_t uid_data[OPAL_UID_LENGTH] = {0};
+	int err = 0;
+	int length;
+	int ret;
+
+	opal_clear_cmd(sess);
+	opal_set_comid(sess, dev->comid);
+
+	if (active_key->key_len == 0) {
+		SPDK_ERRLOG("Error finding previous data to generate new active key\n");
+		return -EINVAL;
+	}
+
+	length = spdk_min(active_key->key_len, OPAL_UID_LENGTH);
+	memcpy(uid_data, active_key->key, length);
+
+	opal_add_token_u8(&err, sess, SPDK_OPAL_CALL);
+	opal_add_token_bytestring(&err, sess, uid_data, OPAL_UID_LENGTH);
+	opal_add_token_bytestring(&err, sess, spdk_opal_method[GENKEY_METHOD],
+				  OPAL_UID_LENGTH);
+
+	opal_add_tokens(&err, sess, 2, SPDK_OPAL_STARTLIST, SPDK_OPAL_ENDLIST);
+
+	if (err) {
+		SPDK_ERRLOG("Error building new key generation command.\n");
+		return err;
+	}
+
+	ret = opal_cmd_finalize(sess, sess->hsn, sess->tsn, true);
+	if (ret) {
+		return ret;
+	}
+
+	ret = opal_send_recv(dev, sess);
+	if (ret) {
+		return ret;
+	}
+
+	return opal_parse_and_check_status(sess);
+}
+
+static int
+opal_get_active_key_done(struct opal_session *sess, struct spdk_opal_key *active_key)
+{
+	const char *key;
+	size_t str_len;
+	int error = 0;
+
+	error = opal_parse_and_check_status(sess);
+	if (error) {
+		return error;
+	}
+
+	str_len = opal_response_get_string(&sess->parsed_resp, 4, &key);
+	if (!key) {
+		SPDK_ERRLOG("Couldn't extract active key from response\n");
+		return -EINVAL;
+	}
+
+	active_key->key_len = str_len;
+	memcpy(active_key->key, key, active_key->key_len);
+
+	SPDK_DEBUGLOG(SPDK_LOG_OPAL, "active key = %p\n", active_key->key);
+	return 0;
+}
+
+static int
+opal_get_active_key(struct spdk_opal_dev *dev, struct opal_session *sess,
+		    enum spdk_opal_locking_range locking_range,
+		    struct spdk_opal_key *active_key)
+{
+	uint8_t uid_locking_range[OPAL_UID_LENGTH];
+	int err = 0;
+	int ret;
+
+	opal_clear_cmd(sess);
+	opal_set_comid(sess, dev->comid);
+
+	opal_build_locking_range(uid_locking_range, locking_range);
+
+	opal_add_token_u8(&err, sess, SPDK_OPAL_CALL);
+	opal_add_token_bytestring(&err, sess, uid_locking_range, OPAL_UID_LENGTH);
+	opal_add_token_bytestring(&err, sess, spdk_opal_method[GET_METHOD],
+				  OPAL_UID_LENGTH);
+	opal_add_tokens(&err, sess, 12,
+			SPDK_OPAL_STARTLIST,
+			SPDK_OPAL_STARTLIST,
+			SPDK_OPAL_STARTNAME,
+			SPDK_OPAL_STARTCOLUMN,
+			SPDK_OPAL_ACTIVEKEY,
+			SPDK_OPAL_ENDNAME,
+			SPDK_OPAL_STARTNAME,
+			SPDK_OPAL_ENDCOLUMN,
+			SPDK_OPAL_ACTIVEKEY,
+			SPDK_OPAL_ENDNAME,
+			SPDK_OPAL_ENDLIST,
+			SPDK_OPAL_ENDLIST);
+
+	if (err) {
+		SPDK_ERRLOG("Error building get active key command.\n");
+		return err;
+	}
+
+	ret = opal_cmd_finalize(sess, sess->hsn, sess->tsn, true);
+	if (ret) {
+		return ret;
+	}
+
+	ret = opal_send_recv(dev, sess);
+	if (ret) {
+		return ret;
+	}
+
+	return opal_get_active_key_done(sess, active_key);
+}
+
+static int
+opal_erase_locking_range(struct spdk_opal_dev *dev, struct opal_session *sess,
+			 enum spdk_opal_locking_range locking_range)
+{
+	uint8_t uid_locking_range[OPAL_UID_LENGTH];
+	int err = 0;
+	int ret;
+
+	opal_clear_cmd(sess);
+	opal_set_comid(sess, dev->comid);
+
+	opal_build_locking_range(uid_locking_range, locking_range);
+
+	opal_add_token_u8(&err, sess, SPDK_OPAL_CALL);
+	opal_add_token_bytestring(&err, sess, uid_locking_range, OPAL_UID_LENGTH);
+	opal_add_token_bytestring(&err, sess, spdk_opal_method[ERASE_METHOD],
+				  OPAL_UID_LENGTH);
+	opal_add_tokens(&err, sess, 2, SPDK_OPAL_STARTLIST, SPDK_OPAL_ENDLIST);
+
+	if (err) {
+		SPDK_ERRLOG("Error building erase locking range.\n");
+		return err;
+	}
+
+	ret = opal_cmd_finalize(sess, sess->hsn, sess->tsn, true);
+	if (ret) {
+		return ret;
+	}
+
+	ret = opal_send_recv(dev, sess);
+	if (ret) {
+		return ret;
+	}
+
+	return opal_parse_and_check_status(sess);
+}
+
+int
+spdk_opal_cmd_revert_tper(struct spdk_opal_dev *dev, const char *passwd)
+{
+	int ret;
+	struct opal_session *sess;
+	struct spdk_opal_key opal_key = {};
+
+	assert(dev != NULL);
+
+	ret = opal_init_key(&opal_key, passwd);
+	if (ret) {
+		SPDK_ERRLOG("Init key failed\n");
+		return ret;
+	}
+
+	sess = opal_alloc_session(dev);
+	if (!sess) {
+		return -ENOMEM;
+	}
+
+	ret = opal_start_generic_session(dev, sess, UID_SID, UID_ADMINSP,
+					 opal_key.key, opal_key.key_len);
+	if (ret) {
+		SPDK_ERRLOG("Error on starting admin SP session with error %d\n", ret);
+		free(sess);
+		return ret;
+	}
+
+	ret = opal_build_revert_tper_cmd(dev, sess);
+	if (ret) {
+		opal_end_session(dev, sess, dev->comid);
+		SPDK_ERRLOG("Build revert tper command with error %d\n", ret);
+		goto end;
+	}
+
+	ret = opal_send_recv(dev, sess);
+	if (ret) {
+		opal_end_session(dev, sess, dev->comid);
+		SPDK_ERRLOG("Error on reverting TPer with error %d\n", ret);
+		goto end;
+	}
+
+	ret = opal_parse_and_check_status(sess);
+	if (ret) {
+		opal_end_session(dev, sess, dev->comid);
+		SPDK_ERRLOG("Error on reverting TPer with error %d\n", ret);
+	}
+	/* No opal_end_session() required here for successful case */
+
+end:
+	free(sess);
+	return ret;
+}
+
+int
+spdk_opal_cmd_activate_locking_sp(struct spdk_opal_dev *dev, const char *passwd)
+{
+	struct opal_session *sess;
+	struct spdk_opal_key opal_key = {};
+	int ret;
+
+	ret = opal_init_key(&opal_key, passwd);
+	if (ret != 0) {
+		return ret;
+	}
+
+	sess = opal_alloc_session(dev);
+	if (!sess) {
+		return -ENOMEM;
+	}
+
+	ret = opal_start_generic_session(dev, sess, UID_SID, UID_ADMINSP,
+					 opal_key.key, opal_key.key_len);
+	if (ret) {
+		SPDK_ERRLOG("Error on starting admin SP session with error %d\n", ret);
+		free(sess);
+		return ret;
+	}
+
+	ret = opal_get_locking_sp_lifecycle(dev, sess);
+	if (ret) {
+		SPDK_ERRLOG("Error on getting SP lifecycle with error %d\n", ret);
+		goto end;
+	}
+
+	ret = opal_activate(dev, sess);
+	if (ret) {
+		SPDK_ERRLOG("Error on activation with error %d\n", ret);
+	}
+
+end:
+	ret += opal_end_session(dev, sess, dev->comid);
+	if (ret) {
+		SPDK_ERRLOG("Error on ending session with error %d\n", ret);
+	}
+
+	free(sess);
+	return ret;
+}
+
+int
+spdk_opal_cmd_lock_unlock(struct spdk_opal_dev *dev, enum spdk_opal_user user,
+			  enum spdk_opal_lock_state flag, enum spdk_opal_locking_range locking_range,
+			  const char *passwd)
+{
+	struct opal_session *sess;
+	struct spdk_opal_key opal_key = {};
+	int ret;
+
+	assert(dev != NULL);
+
+	ret = opal_init_key(&opal_key, passwd);
+	if (ret != 0) {
+		return ret;
+	}
+
+	sess = opal_alloc_session(dev);
+	if (!sess) {
+		return -ENOMEM;
+	}
+
+	ret = opal_start_auth_session(dev, sess, user, &opal_key);
+	if (ret) {
+		SPDK_ERRLOG("start authenticate session error %d\n", ret);
+		free(sess);
+		return ret;
+	}
+
+	ret = opal_lock_unlock_range(dev, sess, locking_range, flag);
+	if (ret) {
+		SPDK_ERRLOG("lock unlock range error %d\n", ret);
+	}
+
+	ret += opal_end_session(dev, sess, dev->comid);
+	if (ret) {
+		SPDK_ERRLOG("end session error %d\n", ret);
+	}
+
+	free(sess);
+	return ret;
+}
+
+int
+spdk_opal_cmd_setup_locking_range(struct spdk_opal_dev *dev, enum spdk_opal_user user,
+				  enum spdk_opal_locking_range locking_range_id, uint64_t range_start,
+				  uint64_t range_length, const char *passwd)
+{
+	struct opal_session *sess;
+	struct spdk_opal_key opal_key = {};
+	int ret;
+
+	assert(dev != NULL);
+
+	ret = opal_init_key(&opal_key, passwd);
+	if (ret != 0) {
+		return ret;
+	}
+
+	sess = opal_alloc_session(dev);
+	if (!sess) {
+		return -ENOMEM;
+	}
+
+	ret = opal_start_auth_session(dev, sess, user, &opal_key);
+	if (ret) {
+		SPDK_ERRLOG("start authenticate session error %d\n", ret);
+		free(sess);
+		return ret;
+	}
+
+	ret = opal_setup_locking_range(dev, sess, locking_range_id, range_start, range_length, true,
+				       true);
+	if (ret) {
+		SPDK_ERRLOG("setup locking range error %d\n", ret);
+	}
+
+	ret += opal_end_session(dev, sess, dev->comid);
+	if (ret) {
+		SPDK_ERRLOG("end session error %d\n", ret);
+	}
+
+	free(sess);
+	return ret;
+}
+
+int
+spdk_opal_cmd_get_max_ranges(struct spdk_opal_dev *dev, const char *passwd)
+{
+	struct opal_session *sess;
+	struct spdk_opal_key opal_key = {};
+	int ret;
+
+	assert(dev != NULL);
+
+	if (dev->max_ranges) {
+		return dev->max_ranges;
+	}
+
+	ret = opal_init_key(&opal_key, passwd);
+	if (ret != 0) {
+		return ret;
+	}
+
+	sess = opal_alloc_session(dev);
+	if (!sess) {
+		return -ENOMEM;
+	}
+
+	ret = opal_start_auth_session(dev, sess, OPAL_ADMIN1, &opal_key);
+	if (ret) {
+		SPDK_ERRLOG("start authenticate session error %d\n", ret);
+		free(sess);
+		return ret;
+	}
+
+	ret = opal_get_max_ranges(dev, sess);
+	if (ret > 0) {
+		dev->max_ranges = ret;
+	}
+
+	ret = opal_end_session(dev, sess, dev->comid);
+	if (ret) {
+		SPDK_ERRLOG("end session error %d\n", ret);
+	}
+
+	free(sess);
+
+	return (ret == 0 ? dev->max_ranges : ret);
+}
+
+int
+spdk_opal_cmd_get_locking_range_info(struct spdk_opal_dev *dev, const char *passwd,
+				     enum spdk_opal_user user_id,
+				     enum spdk_opal_locking_range locking_range_id)
+{
+	struct opal_session *sess;
+	struct spdk_opal_key opal_key = {};
+	int ret;
+
+	assert(dev != NULL);
+
+	ret = opal_init_key(&opal_key, passwd);
+	if (ret != 0) {
+		return ret;
+	}
+
+	sess = opal_alloc_session(dev);
+	if (!sess) {
+		return -ENOMEM;
+	}
+
+	ret = opal_start_auth_session(dev, sess, user_id, &opal_key);
+	if (ret) {
+		SPDK_ERRLOG("start authenticate session error %d\n", ret);
+		free(sess);
+		return ret;
+	}
+
+	ret = opal_get_locking_range_info(dev, sess, locking_range_id);
+	if (ret) {
+		SPDK_ERRLOG("get locking range info error %d\n", ret);
+	}
+
+	ret += opal_end_session(dev, sess, dev->comid);
+	if (ret) {
+		SPDK_ERRLOG("end session error %d\n", ret);
+	}
+
+	free(sess);
+	return ret;
+}
+
+int
+spdk_opal_cmd_enable_user(struct spdk_opal_dev *dev, enum spdk_opal_user user_id,
+			  const char *passwd)
+{
+	struct opal_session *sess;
+	struct spdk_opal_key opal_key = {};
+	int ret;
+
+	assert(dev != NULL);
+
+	ret = opal_init_key(&opal_key, passwd);
+	if (ret != 0) {
+		return ret;
+	}
+
+	sess = opal_alloc_session(dev);
+	if (!sess) {
+		return -ENOMEM;
+	}
+
+	ret =  opal_start_generic_session(dev, sess, UID_ADMIN1, UID_LOCKINGSP,
+					  opal_key.key, opal_key.key_len);
+	if (ret) {
+		SPDK_ERRLOG("start locking SP session error %d\n", ret);
+		free(sess);
+		return ret;
+	}
+
+	ret = opal_enable_user(dev, sess, user_id);
+	if (ret) {
+		SPDK_ERRLOG("enable user error %d\n", ret);
+	}
+
+	ret += opal_end_session(dev, sess, dev->comid);
+	if (ret) {
+		SPDK_ERRLOG("end session error %d\n", ret);
+	}
+
+	free(sess);
+	return ret;
+}
+
+int
+spdk_opal_cmd_add_user_to_locking_range(struct spdk_opal_dev *dev, enum spdk_opal_user user_id,
+					enum spdk_opal_locking_range locking_range_id,
+					enum spdk_opal_lock_state lock_flag, const char *passwd)
+{
+	struct opal_session *sess;
+	struct spdk_opal_key opal_key = {};
+	int ret;
+
+	assert(dev != NULL);
+
+	ret = opal_init_key(&opal_key, passwd);
+	if (ret != 0) {
+		return ret;
+	}
+
+	sess = opal_alloc_session(dev);
+	if (!sess) {
+		return -ENOMEM;
+	}
+
+	ret =  opal_start_generic_session(dev, sess, UID_ADMIN1, UID_LOCKINGSP,
+					  opal_key.key, opal_key.key_len);
+	if (ret) {
+		SPDK_ERRLOG("start locking SP session error %d\n", ret);
+		free(sess);
+		return ret;
+	}
+
+	ret = opal_add_user_to_locking_range(dev, sess, user_id, locking_range_id, lock_flag);
+	if (ret) {
+		SPDK_ERRLOG("add user to locking range error %d\n", ret);
+	}
+
+	ret += opal_end_session(dev, sess, dev->comid);
+	if (ret) {
+		SPDK_ERRLOG("end session error %d\n", ret);
+	}
+
+	free(sess);
+	return ret;
+}
+
+int
+spdk_opal_cmd_set_new_passwd(struct spdk_opal_dev *dev, enum spdk_opal_user user_id,
+			     const char *new_passwd, const char *old_passwd, bool new_user)
+{
+	struct opal_session *sess;
+	struct spdk_opal_key old_key = {};
+	struct spdk_opal_key new_key = {};
+	int ret;
+
+	assert(dev != NULL);
+
+	ret = opal_init_key(&old_key, old_passwd);
+	if (ret != 0) {
+		return ret;
+	}
+
+	ret = opal_init_key(&new_key, new_passwd);
+	if (ret != 0) {
+		return ret;
+	}
+
+	sess = opal_alloc_session(dev);
+	if (!sess) {
+		return -ENOMEM;
+	}
+
+	ret = opal_start_auth_session(dev, sess, new_user ? OPAL_ADMIN1 : user_id,
+				      &old_key);
+	if (ret) {
+		SPDK_ERRLOG("start authenticate session error %d\n", ret);
+		free(sess);
+		return ret;
+	}
+
+	ret = opal_new_user_passwd(dev, sess, user_id, &new_key);
+	if (ret) {
+		SPDK_ERRLOG("set new passwd error %d\n", ret);
+	}
+
+	ret += opal_end_session(dev, sess, dev->comid);
+	if (ret) {
+		SPDK_ERRLOG("end session error %d\n", ret);
+	}
+
+	free(sess);
+	return ret;
+}
+
+int
+spdk_opal_cmd_erase_locking_range(struct spdk_opal_dev *dev, enum spdk_opal_user user_id,
+				  enum spdk_opal_locking_range locking_range_id, const char *password)
+{
+	struct opal_session *sess;
+	struct spdk_opal_key opal_key = {};
+	int ret;
+
+	assert(dev != NULL);
+
+	ret = opal_init_key(&opal_key, password);
+	if (ret != 0) {
+		return ret;
+	}
+
+	sess = opal_alloc_session(dev);
+	if (!sess) {
+		return -ENOMEM;
+	}
+
+	ret = opal_start_auth_session(dev, sess, user_id, &opal_key);
+	if (ret) {
+		SPDK_ERRLOG("start authenticate session error %d\n", ret);
+		free(sess);
+		return ret;
+	}
+
+	ret = opal_erase_locking_range(dev, sess, locking_range_id);
+	if (ret) {
+		SPDK_ERRLOG("get active key error %d\n", ret);
+	}
+
+	ret += opal_end_session(dev, sess, dev->comid);
+	if (ret) {
+		SPDK_ERRLOG("end session error %d\n", ret);
+	}
+
+	free(sess);
+	return ret;
+}
+
+int
+spdk_opal_cmd_secure_erase_locking_range(struct spdk_opal_dev *dev, enum spdk_opal_user user_id,
+		enum spdk_opal_locking_range locking_range_id, const char *password)
+{
+	struct opal_session *sess;
+	struct spdk_opal_key opal_key = {};
+	struct spdk_opal_key *active_key;
+	int ret;
+
+	assert(dev != NULL);
+
+	ret = opal_init_key(&opal_key, password);
+	if (ret != 0) {
+		return ret;
+	}
+
+	active_key = calloc(1, sizeof(*active_key));
+	if (!active_key) {
+		return -ENOMEM;
+	}
+
+	sess = opal_alloc_session(dev);
+	if (!sess) {
+		free(active_key);
+		return -ENOMEM;
+	}
+
+	ret = opal_start_auth_session(dev, sess, user_id, &opal_key);
+	if (ret) {
+		SPDK_ERRLOG("start authenticate session error %d\n", ret);
+		free(active_key);
+		free(sess);
+		return ret;
+	}
+
+	ret = opal_get_active_key(dev, sess, locking_range_id, active_key);
+	if (ret) {
+		SPDK_ERRLOG("get active key error %d\n", ret);
+		goto end;
+	}
+
+	ret = opal_gen_new_active_key(dev, sess, active_key);
+	if (ret) {
+		SPDK_ERRLOG("generate new active key error %d\n", ret);
+		goto end;
+	}
+	memset(active_key, 0, sizeof(struct spdk_opal_key));
+
+end:
+	ret += opal_end_session(dev, sess, dev->comid);
+	if (ret) {
+		SPDK_ERRLOG("end session error %d\n", ret);
+	}
+	free(active_key);
+	free(sess);
+	return ret;
+}
+
+struct spdk_opal_d0_features_info *
+spdk_opal_get_d0_features_info(struct spdk_opal_dev *dev)
+{
+	return &dev->feat_info;
+}
+
+bool
+spdk_opal_supported(struct spdk_opal_dev *dev)
+{
+	return false;
+}
+
+struct spdk_opal_locking_range_info *
+spdk_opal_get_locking_range_info(struct spdk_opal_dev *dev, enum spdk_opal_locking_range id)
+{
+	assert(id < SPDK_OPAL_MAX_LOCKING_RANGE);
+	return &dev->locking_ranges[id];
+}
+
+void
+spdk_opal_free_locking_range_info(struct spdk_opal_dev *dev, enum spdk_opal_locking_range id)
+{
+	struct spdk_opal_locking_range_info *info;
+
+	assert(id < SPDK_OPAL_MAX_LOCKING_RANGE);
+	info = &dev->locking_ranges[id];
+	memset(info, 0, sizeof(*info));
+}
+
+/* Log component for opal submodule */
+SPDK_LOG_REGISTER_COMPONENT("opal", SPDK_LOG_OPAL)
diff --git a/src/spdk/lib/nvme/nvme_opal_internal.h b/src/spdk/lib/nvme/nvme_opal_internal.h
new file mode 100644
index 000000000..11815d435
--- /dev/null
+++ b/src/spdk/lib/nvme/nvme_opal_internal.h
@@ -0,0 +1,272 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef SPDK_OPAL_INTERNAL_H
+#define SPDK_OPAL_INTERNAL_H
+
+#include "spdk/opal_spec.h"
+#include "spdk/opal.h"
+#include "spdk/scsi_spec.h"
+
+#define IO_BUFFER_LENGTH		2048
+#define MAX_TOKS			64
+#define OPAL_KEY_MAX			256
+#define OPAL_UID_LENGTH			8
+
+#define GENERIC_HOST_SESSION_NUM	0x69
+
+#define OPAL_INVAL_PARAM		12
+
+#define SPDK_DTAERROR_NO_METHOD_STATUS	0x89
+
+enum opal_token_type {
+	OPAL_DTA_TOKENID_BYTESTRING	= 0xE0,
+	OPAL_DTA_TOKENID_SINT		= 0xE1,
+	OPAL_DTA_TOKENID_UINT		= 0xE2,
+	OPAL_DTA_TOKENID_TOKEN		= 0xE3, /* actual token is returned */
+	OPAL_DTA_TOKENID_INVALID	= 0X0,
+};
+
+enum opal_atom_width {
+	OPAL_WIDTH_TINY,    /* 1 byte in length */
+	OPAL_WIDTH_SHORT,   /* a 1-byte header and contain up to 15 bytes of data */
+	OPAL_WIDTH_MEDIUM,  /* a 2-byte header and contain up to 2047 bytes of data */
+	OPAL_WIDTH_LONG,    /* a 4-byte header and which contain up to 16,777,215 bytes of data */
+	OPAL_WIDTH_TOKEN
+};
+
+enum opal_uid_enum {
+	/* users */
+	UID_SMUID,
+	UID_THISSP,
+	UID_ADMINSP,
+	UID_LOCKINGSP,
+	UID_ANYBODY,
+	UID_SID,
+	UID_ADMIN1,
+	UID_USER1,
+	UID_USER2,
+
+	/* tables */
+	UID_LOCKINGRANGE_GLOBAL,
+	UID_LOCKINGRANGE_ACE_RDLOCKED,
+	UID_LOCKINGRANGE_ACE_WRLOCKED,
+	UID_MBRCONTROL,
+	UID_MBR,
+	UID_AUTHORITY_TABLE,
+	UID_C_PIN_TABLE,
+	UID_LOCKING_INFO_TABLE,
+	UID_PSID,
+
+	/* C_PIN_TABLE object ID's */
+	UID_C_PIN_MSID,
+	UID_C_PIN_SID,
+	UID_C_PIN_ADMIN1,
+	UID_C_PIN_USER1,
+
+	/* half UID's (only first 4 bytes used) */
+	UID_HALF_AUTHORITY_OBJ_REF,
+	UID_HALF_BOOLEAN_ACE,
+};
+
+/* enum for indexing the spdk_opal_method array */
+enum opal_method_enum {
+	PROPERTIES_METHOD,
+	STARTSESSION_METHOD,
+	REVERT_METHOD,
+	ACTIVATE_METHOD,
+	NEXT_METHOD,
+	GETACL_METHOD,
+	GENKEY_METHOD,
+	REVERTSP_METHOD,
+	GET_METHOD,
+	SET_METHOD,
+	AUTHENTICATE_METHOD,
+	RANDOM_METHOD,
+	ERASE_METHOD,
+};
+
+struct spdk_opal_key {
+	uint8_t key_len;
+	uint8_t key[OPAL_KEY_MAX];
+};
+
+const uint8_t spdk_opal_uid[][OPAL_UID_LENGTH] = {
+	/* users */
+	[UID_SMUID] = /* Session Manager UID */
+	{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff },
+	[UID_THISSP] =
+	{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01 },
+	[UID_ADMINSP] =
+	{ 0x00, 0x00, 0x02, 0x05, 0x00, 0x00, 0x00, 0x01 },
+	[UID_LOCKINGSP] =
+	{ 0x00, 0x00, 0x02, 0x05, 0x00, 0x00, 0x00, 0x02 },
+	[UID_ANYBODY] =
+	{ 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x01 },
+	[UID_SID] = /* Security Identifier UID */
+	{ 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x06 },
+	[UID_ADMIN1] =
+	{ 0x00, 0x00, 0x00, 0x09, 0x00, 0x01, 0x00, 0x01 },
+	[UID_USER1] =
+	{ 0x00, 0x00, 0x00, 0x09, 0x00, 0x03, 0x00, 0x01 },
+	[UID_USER2] =
+	{ 0x00, 0x00, 0x00, 0x09, 0x00, 0x03, 0x00, 0x02 },
+
+	/* tables */
+	[UID_LOCKINGRANGE_GLOBAL] =
+	{ 0x00, 0x00, 0x08, 0x02, 0x00, 0x00, 0x00, 0x01 },
+	[UID_LOCKINGRANGE_ACE_RDLOCKED] =
+	{ 0x00, 0x00, 0x00, 0x08, 0x00, 0x03, 0xE0, 0x01 },
+	[UID_LOCKINGRANGE_ACE_WRLOCKED] =
+	{ 0x00, 0x00, 0x00, 0x08, 0x00, 0x03, 0xE8, 0x01 },
+	[UID_MBRCONTROL] =
+	{ 0x00, 0x00, 0x08, 0x03, 0x00, 0x00, 0x00, 0x01 },
+	[UID_MBR] =
+	{ 0x00, 0x00, 0x08, 0x04, 0x00, 0x00, 0x00, 0x00 },
+	[UID_AUTHORITY_TABLE] =
+	{ 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x00},
+	[UID_C_PIN_TABLE] =
+	{ 0x00, 0x00, 0x00, 0x0B, 0x00, 0x00, 0x00, 0x00},
+	[UID_LOCKING_INFO_TABLE] =
+	{ 0x00, 0x00, 0x08, 0x01, 0x00, 0x00, 0x00, 0x01 },
+	[UID_PSID] =
+	{ 0x00, 0x00, 0x00, 0x09, 0x00, 0x01, 0xff, 0x01 },
+
+	/* C_PIN_TABLE object ID's */
+	[UID_C_PIN_MSID] =
+	{ 0x00, 0x00, 0x00, 0x0B, 0x00, 0x00, 0x84, 0x02},
+	[UID_C_PIN_SID] =
+	{ 0x00, 0x00, 0x00, 0x0B, 0x00, 0x00, 0x00, 0x01},
+	[UID_C_PIN_ADMIN1] =
+	{ 0x00, 0x00, 0x00, 0x0B, 0x00, 0x01, 0x00, 0x01},
+	[UID_C_PIN_USER1] =
+	{ 0x00, 0x00, 0x00, 0x0B, 0x00, 0x03, 0x00, 0x01},
+
+	/* half UID's (only first 4 bytes used) */
+	[UID_HALF_AUTHORITY_OBJ_REF] =
+	{ 0x00, 0x00, 0x0C, 0x05, 0xff, 0xff, 0xff, 0xff },
+	[UID_HALF_BOOLEAN_ACE] =
+	{ 0x00, 0x00, 0x04, 0x0E, 0xff, 0xff, 0xff, 0xff },
+};
+
+/*
+ * TCG Storage SSC Methods.
+ */
+const uint8_t spdk_opal_method[][OPAL_UID_LENGTH] = {
+	[PROPERTIES_METHOD] =
+	{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0x01 },
+	[STARTSESSION_METHOD] =
+	{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0x02 },
+	[REVERT_METHOD] =
+	{ 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x02, 0x02 },
+	[ACTIVATE_METHOD] =
+	{ 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x02, 0x03 },
+	[NEXT_METHOD] =
+	{ 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x08 },
+	[GETACL_METHOD] =
+	{ 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x0d },
+	[GENKEY_METHOD] =
+	{ 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x10 },
+	[REVERTSP_METHOD] =
+	{ 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x11 },
+	[GET_METHOD] =
+	{ 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x16 },
+	[SET_METHOD] =
+	{ 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x17 },
+	[AUTHENTICATE_METHOD] =
+	{ 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x1c },
+	[RANDOM_METHOD] =
+	{ 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x06, 0x01 },
+	[ERASE_METHOD] =
+	{ 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x08, 0x03 },
+};
+
+/*
+ * Response token
+ */
+struct spdk_opal_resp_token {
+	const uint8_t *pos;
+	uint8_t _padding[7];
+	union {
+		uint64_t unsigned_num;
+		int64_t signed_num;
+	} stored;
+	size_t len; /* header + data */
+	enum opal_token_type type;
+	enum opal_atom_width width;
+};
+
+struct spdk_opal_resp_parsed {
+	int num;
+	struct spdk_opal_resp_token resp_tokens[MAX_TOKS];
+};
+
+/* header of a response */
+struct spdk_opal_header {
+	struct spdk_opal_compacket com_packet;
+	struct spdk_opal_packet packet;
+	struct spdk_opal_data_subpacket sub_packet;
+};
+
+struct opal_session;
+struct spdk_opal_dev;
+
+typedef void (*opal_sess_cb)(struct opal_session *sess, int status, void *ctx);
+
+struct opal_session {
+	uint32_t hsn;
+	uint32_t tsn;
+	size_t cmd_pos;
+	uint8_t cmd[IO_BUFFER_LENGTH];
+	uint8_t resp[IO_BUFFER_LENGTH];
+	struct spdk_opal_resp_parsed parsed_resp;
+
+	opal_sess_cb sess_cb;
+	void *cb_arg;
+	bool done;
+	int status;
+	struct spdk_opal_dev *dev;
+};
+
+struct spdk_opal_dev {
+	struct spdk_nvme_ctrlr *ctrlr;
+
+	uint16_t comid;
+
+	struct spdk_opal_d0_features_info feat_info;
+
+	uint8_t max_ranges; /* max locking range number */
+	struct spdk_opal_locking_range_info locking_ranges[SPDK_OPAL_MAX_LOCKING_RANGE];
+};
+
+#endif
diff --git a/src/spdk/lib/nvme/nvme_pcie.c b/src/spdk/lib/nvme/nvme_pcie.c
new file mode 100644
index 000000000..132e34cdc
--- /dev/null
+++ b/src/spdk/lib/nvme/nvme_pcie.c
@@ -0,0 +1,2604 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation. All rights reserved.
+ *   Copyright (c) 2017, IBM Corporation. All rights reserved.
+ *   Copyright (c) 2019, 2020 Mellanox Technologies LTD. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * NVMe over PCIe transport
+ */
+
+#include "spdk/stdinc.h"
+#include "spdk/env.h"
+#include "spdk/likely.h"
+#include "spdk/string.h"
+#include "nvme_internal.h"
+#include "nvme_uevent.h"
+
+/*
+ * Number of completion queue entries to process before ringing the
+ *  completion queue doorbell.
+ */
+#define NVME_MIN_COMPLETIONS	(1)
+#define NVME_MAX_COMPLETIONS	(128)
+
+/*
+ * NVME_MAX_SGL_DESCRIPTORS defines the maximum number of descriptors in one SGL
+ *  segment.
+ */
+#define NVME_MAX_SGL_DESCRIPTORS	(250)
+
+#define NVME_MAX_PRP_LIST_ENTRIES	(503)
+
+struct nvme_pcie_enum_ctx {
+	struct spdk_nvme_probe_ctx *probe_ctx;
+	struct spdk_pci_addr pci_addr;
+	bool has_pci_addr;
+};
+
+/* PCIe transport extensions for spdk_nvme_ctrlr */
+struct nvme_pcie_ctrlr {
+	struct spdk_nvme_ctrlr ctrlr;
+
+	/** NVMe MMIO register space */
+	volatile struct spdk_nvme_registers *regs;
+
+	/** NVMe MMIO register size */
+	uint64_t regs_size;
+
+	struct {
+		/* BAR mapping address which contains controller memory buffer */
+		void *bar_va;
+
+		/* BAR physical address which contains controller memory buffer */
+		uint64_t bar_pa;
+
+		/* Controller memory buffer size in Bytes */
+		uint64_t size;
+
+		/* Current offset of controller memory buffer, relative to start of BAR virt addr */
+		uint64_t current_offset;
+
+		void *mem_register_addr;
+		size_t mem_register_size;
+	} cmb;
+
+	/** stride in uint32_t units between doorbell registers (1 = 4 bytes, 2 = 8 bytes, ...) */
+	uint32_t doorbell_stride_u32;
+
+	/* Opaque handle to associated PCI device. */
+	struct spdk_pci_device *devhandle;
+
+	/* Flag to indicate the MMIO register has been remapped */
+	bool is_remapped;
+};
+
+struct nvme_tracker {
+	TAILQ_ENTRY(nvme_tracker)       tq_list;
+
+	struct nvme_request		*req;
+	uint16_t			cid;
+
+	uint16_t			rsvd0;
+	uint32_t			rsvd1;
+
+	spdk_nvme_cmd_cb		cb_fn;
+	void				*cb_arg;
+
+	uint64_t			prp_sgl_bus_addr;
+
+	/* Don't move, metadata SGL is always contiguous with Data Block SGL */
+	struct spdk_nvme_sgl_descriptor		meta_sgl;
+	union {
+		uint64_t			prp[NVME_MAX_PRP_LIST_ENTRIES];
+		struct spdk_nvme_sgl_descriptor	sgl[NVME_MAX_SGL_DESCRIPTORS];
+	} u;
+};
+/*
+ * struct nvme_tracker must be exactly 4K so that the prp[] array does not cross a page boundary
+ * and so that there is no padding required to meet alignment requirements.
+ */
+SPDK_STATIC_ASSERT(sizeof(struct nvme_tracker) == 4096, "nvme_tracker is not 4K");
+SPDK_STATIC_ASSERT((offsetof(struct nvme_tracker, u.sgl) & 7) == 0, "SGL must be Qword aligned");
+SPDK_STATIC_ASSERT((offsetof(struct nvme_tracker, meta_sgl) & 7) == 0, "SGL must be Qword aligned");
+
+struct nvme_pcie_poll_group {
+	struct spdk_nvme_transport_poll_group group;
+};
+
+/* PCIe transport extensions for spdk_nvme_qpair */
+struct nvme_pcie_qpair {
+	/* Submission queue tail doorbell */
+	volatile uint32_t *sq_tdbl;
+
+	/* Completion queue head doorbell */
+	volatile uint32_t *cq_hdbl;
+
+	/* Submission queue */
+	struct spdk_nvme_cmd *cmd;
+
+	/* Completion queue */
+	struct spdk_nvme_cpl *cpl;
+
+	TAILQ_HEAD(, nvme_tracker) free_tr;
+	TAILQ_HEAD(nvme_outstanding_tr_head, nvme_tracker) outstanding_tr;
+
+	/* Array of trackers indexed by command ID. */
+	struct nvme_tracker *tr;
+
+	uint16_t num_entries;
+
+	uint8_t retry_count;
+
+	uint16_t max_completions_cap;
+
+	uint16_t last_sq_tail;
+	uint16_t sq_tail;
+	uint16_t cq_head;
+	uint16_t sq_head;
+
+	struct {
+		uint8_t phase			: 1;
+		uint8_t delay_cmd_submit	: 1;
+		uint8_t has_shadow_doorbell	: 1;
+	} flags;
+
+	/*
+	 * Base qpair structure.
+	 * This is located after the hot data in this structure so that the important parts of
+	 * nvme_pcie_qpair are in the same cache line.
+	 */
+	struct spdk_nvme_qpair qpair;
+
+	struct {
+		/* Submission queue shadow tail doorbell */
+		volatile uint32_t *sq_tdbl;
+
+		/* Completion queue shadow head doorbell */
+		volatile uint32_t *cq_hdbl;
+
+		/* Submission queue event index */
+		volatile uint32_t *sq_eventidx;
+
+		/* Completion queue event index */
+		volatile uint32_t *cq_eventidx;
+	} shadow_doorbell;
+
+	/*
+	 * Fields below this point should not be touched on the normal I/O path.
+	 */
+
+	bool sq_in_cmb;
+
+	uint64_t cmd_bus_addr;
+	uint64_t cpl_bus_addr;
+
+	struct spdk_nvme_cmd *sq_vaddr;
+	struct spdk_nvme_cpl *cq_vaddr;
+};
+
+static int nvme_pcie_ctrlr_attach(struct spdk_nvme_probe_ctx *probe_ctx,
+				  struct spdk_pci_addr *pci_addr);
+static int nvme_pcie_qpair_construct(struct spdk_nvme_qpair *qpair,
+				     const struct spdk_nvme_io_qpair_opts *opts);
+static int nvme_pcie_qpair_destroy(struct spdk_nvme_qpair *qpair);
+
+__thread struct nvme_pcie_ctrlr *g_thread_mmio_ctrlr = NULL;
+static uint16_t g_signal_lock;
+static bool g_sigset = false;
+
+static void
+nvme_sigbus_fault_sighandler(int signum, siginfo_t *info, void *ctx)
+{
+	void *map_address;
+	uint16_t flag = 0;
+
+	if (!__atomic_compare_exchange_n(&g_signal_lock, &flag, 1, false, __ATOMIC_ACQUIRE,
+					 __ATOMIC_RELAXED)) {
+		SPDK_DEBUGLOG(SPDK_LOG_NVME, "request g_signal_lock failed\n");
+		return;
+	}
+
+	assert(g_thread_mmio_ctrlr != NULL);
+
+	if (!g_thread_mmio_ctrlr->is_remapped) {
+		map_address = mmap((void *)g_thread_mmio_ctrlr->regs, g_thread_mmio_ctrlr->regs_size,
+				   PROT_READ | PROT_WRITE,
+				   MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0);
+		if (map_address == MAP_FAILED) {
+			SPDK_ERRLOG("mmap failed\n");
+			__atomic_store_n(&g_signal_lock, 0, __ATOMIC_RELEASE);
+			return;
+		}
+		memset(map_address, 0xFF, sizeof(struct spdk_nvme_registers));
+		g_thread_mmio_ctrlr->regs = (volatile struct spdk_nvme_registers *)map_address;
+		g_thread_mmio_ctrlr->is_remapped = true;
+	}
+	__atomic_store_n(&g_signal_lock, 0, __ATOMIC_RELEASE);
+}
+
+static void
+nvme_pcie_ctrlr_setup_signal(void)
+{
+	struct sigaction sa;
+
+	sa.sa_sigaction = nvme_sigbus_fault_sighandler;
+	sigemptyset(&sa.sa_mask);
+	sa.sa_flags = SA_SIGINFO;
+	sigaction(SIGBUS, &sa, NULL);
+}
+
+static inline struct nvme_pcie_ctrlr *
+nvme_pcie_ctrlr(struct spdk_nvme_ctrlr *ctrlr)
+{
+	assert(ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_PCIE);
+	return SPDK_CONTAINEROF(ctrlr, struct nvme_pcie_ctrlr, ctrlr);
+}
+
+static int
+_nvme_pcie_hotplug_monitor(struct spdk_nvme_probe_ctx *probe_ctx)
+{
+	struct spdk_nvme_ctrlr *ctrlr, *tmp;
+	struct spdk_uevent event;
+	struct spdk_pci_addr pci_addr;
+
+	if (g_spdk_nvme_driver->hotplug_fd < 0) {
+		return 0;
+	}
+
+	while (nvme_get_uevent(g_spdk_nvme_driver->hotplug_fd, &event) > 0) {
+		if (event.subsystem == SPDK_NVME_UEVENT_SUBSYSTEM_UIO ||
+		    event.subsystem == SPDK_NVME_UEVENT_SUBSYSTEM_VFIO) {
+			if (event.action == SPDK_NVME_UEVENT_ADD) {
+				SPDK_DEBUGLOG(SPDK_LOG_NVME, "add nvme address: %s\n",
+					      event.traddr);
+				if (spdk_process_is_primary()) {
+					if (!spdk_pci_addr_parse(&pci_addr, event.traddr)) {
+						nvme_pcie_ctrlr_attach(probe_ctx, &pci_addr);
+					}
+				}
+			} else if (event.action == SPDK_NVME_UEVENT_REMOVE) {
+				struct spdk_nvme_transport_id trid;
+
+				memset(&trid, 0, sizeof(trid));
+				spdk_nvme_trid_populate_transport(&trid, SPDK_NVME_TRANSPORT_PCIE);
+				snprintf(trid.traddr, sizeof(trid.traddr), "%s", event.traddr);
+
+				ctrlr = nvme_get_ctrlr_by_trid_unsafe(&trid);
+				if (ctrlr == NULL) {
+					return 0;
+				}
+				SPDK_DEBUGLOG(SPDK_LOG_NVME, "remove nvme address: %s\n",
+					      event.traddr);
+
+				nvme_ctrlr_fail(ctrlr, true);
+
+				/* get the user app to clean up and stop I/O */
+				if (ctrlr->remove_cb) {
+					nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock);
+					ctrlr->remove_cb(probe_ctx->cb_ctx, ctrlr);
+					nvme_robust_mutex_lock(&g_spdk_nvme_driver->lock);
+				}
+			}
+		}
+	}
+
+	/* Initiate removal of physically hotremoved PCI controllers. Even after
+	 * they're hotremoved from the system, SPDK might still report them via RPC.
+	 */
+	TAILQ_FOREACH_SAFE(ctrlr, &g_spdk_nvme_driver->shared_attached_ctrlrs, tailq, tmp) {
+		bool do_remove = false;
+		struct nvme_pcie_ctrlr *pctrlr;
+
+		if (ctrlr->trid.trtype != SPDK_NVME_TRANSPORT_PCIE) {
+			continue;
+		}
+
+		pctrlr = nvme_pcie_ctrlr(ctrlr);
+		if (spdk_pci_device_is_removed(pctrlr->devhandle)) {
+			do_remove = true;
+		}
+
+		if (do_remove) {
+			nvme_ctrlr_fail(ctrlr, true);
+			if (ctrlr->remove_cb) {
+				nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock);
+				ctrlr->remove_cb(probe_ctx->cb_ctx, ctrlr);
+				nvme_robust_mutex_lock(&g_spdk_nvme_driver->lock);
+			}
+		}
+	}
+	return 0;
+}
+
+static inline struct nvme_pcie_qpair *
+nvme_pcie_qpair(struct spdk_nvme_qpair *qpair)
+{
+	assert(qpair->trtype == SPDK_NVME_TRANSPORT_PCIE);
+	return SPDK_CONTAINEROF(qpair, struct nvme_pcie_qpair, qpair);
+}
+
+static volatile void *
+nvme_pcie_reg_addr(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset)
+{
+	struct nvme_pcie_ctrlr *pctrlr = nvme_pcie_ctrlr(ctrlr);
+
+	return (volatile void *)((uintptr_t)pctrlr->regs + offset);
+}
+
+static int
+nvme_pcie_ctrlr_set_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t value)
+{
+	struct nvme_pcie_ctrlr *pctrlr = nvme_pcie_ctrlr(ctrlr);
+
+	assert(offset <= sizeof(struct spdk_nvme_registers) - 4);
+	g_thread_mmio_ctrlr = pctrlr;
+	spdk_mmio_write_4(nvme_pcie_reg_addr(ctrlr, offset), value);
+	g_thread_mmio_ctrlr = NULL;
+	return 0;
+}
+
+static int
+nvme_pcie_ctrlr_set_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t value)
+{
+	struct nvme_pcie_ctrlr *pctrlr = nvme_pcie_ctrlr(ctrlr);
+
+	assert(offset <= sizeof(struct spdk_nvme_registers) - 8);
+	g_thread_mmio_ctrlr = pctrlr;
+	spdk_mmio_write_8(nvme_pcie_reg_addr(ctrlr, offset), value);
+	g_thread_mmio_ctrlr = NULL;
+	return 0;
+}
+
+static int
+nvme_pcie_ctrlr_get_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t *value)
+{
+	struct nvme_pcie_ctrlr *pctrlr = nvme_pcie_ctrlr(ctrlr);
+
+	assert(offset <= sizeof(struct spdk_nvme_registers) - 4);
+	assert(value != NULL);
+	g_thread_mmio_ctrlr = pctrlr;
+	*value = spdk_mmio_read_4(nvme_pcie_reg_addr(ctrlr, offset));
+	g_thread_mmio_ctrlr = NULL;
+	if (~(*value) == 0) {
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+nvme_pcie_ctrlr_get_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t *value)
+{
+	struct nvme_pcie_ctrlr *pctrlr = nvme_pcie_ctrlr(ctrlr);
+
+	assert(offset <= sizeof(struct spdk_nvme_registers) - 8);
+	assert(value != NULL);
+	g_thread_mmio_ctrlr = pctrlr;
+	*value = spdk_mmio_read_8(nvme_pcie_reg_addr(ctrlr, offset));
+	g_thread_mmio_ctrlr = NULL;
+	if (~(*value) == 0) {
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+nvme_pcie_ctrlr_set_asq(struct nvme_pcie_ctrlr *pctrlr, uint64_t value)
+{
+	return nvme_pcie_ctrlr_set_reg_8(&pctrlr->ctrlr, offsetof(struct spdk_nvme_registers, asq),
+					 value);
+}
+
+static int
+nvme_pcie_ctrlr_set_acq(struct nvme_pcie_ctrlr *pctrlr, uint64_t value)
+{
+	return nvme_pcie_ctrlr_set_reg_8(&pctrlr->ctrlr, offsetof(struct spdk_nvme_registers, acq),
+					 value);
+}
+
+static int
+nvme_pcie_ctrlr_set_aqa(struct nvme_pcie_ctrlr *pctrlr, const union spdk_nvme_aqa_register *aqa)
+{
+	return nvme_pcie_ctrlr_set_reg_4(&pctrlr->ctrlr, offsetof(struct spdk_nvme_registers, aqa.raw),
+					 aqa->raw);
+}
+
+static int
+nvme_pcie_ctrlr_get_cmbloc(struct nvme_pcie_ctrlr *pctrlr, union spdk_nvme_cmbloc_register *cmbloc)
+{
+	return nvme_pcie_ctrlr_get_reg_4(&pctrlr->ctrlr, offsetof(struct spdk_nvme_registers, cmbloc.raw),
+					 &cmbloc->raw);
+}
+
+static int
+nvme_pcie_ctrlr_get_cmbsz(struct nvme_pcie_ctrlr *pctrlr, union spdk_nvme_cmbsz_register *cmbsz)
+{
+	return nvme_pcie_ctrlr_get_reg_4(&pctrlr->ctrlr, offsetof(struct spdk_nvme_registers, cmbsz.raw),
+					 &cmbsz->raw);
+}
+
+static  uint32_t
+nvme_pcie_ctrlr_get_max_xfer_size(struct spdk_nvme_ctrlr *ctrlr)
+{
+	/*
+	 * For commands requiring more than 2 PRP entries, one PRP will be
+	 *  embedded in the command (prp1), and the rest of the PRP entries
+	 *  will be in a list pointed to by the command (prp2).  This means
+	 *  that real max number of PRP entries we support is 506+1, which
+	 *  results in a max xfer size of 506*ctrlr->page_size.
+	 */
+	return NVME_MAX_PRP_LIST_ENTRIES * ctrlr->page_size;
+}
+
+static uint16_t
+nvme_pcie_ctrlr_get_max_sges(struct spdk_nvme_ctrlr *ctrlr)
+{
+	return NVME_MAX_SGL_DESCRIPTORS;
+}
+
+static void
+nvme_pcie_ctrlr_map_cmb(struct nvme_pcie_ctrlr *pctrlr)
+{
+	int rc;
+	void *addr = NULL;
+	uint32_t bir;
+	union spdk_nvme_cmbsz_register cmbsz;
+	union spdk_nvme_cmbloc_register cmbloc;
+	uint64_t size, unit_size, offset, bar_size = 0, bar_phys_addr = 0;
+
+	if (nvme_pcie_ctrlr_get_cmbsz(pctrlr, &cmbsz) ||
+	    nvme_pcie_ctrlr_get_cmbloc(pctrlr, &cmbloc)) {
+		SPDK_ERRLOG("get registers failed\n");
+		goto exit;
+	}
+
+	if (!cmbsz.bits.sz) {
+		goto exit;
+	}
+
+	bir = cmbloc.bits.bir;
+	/* Values 0 2 3 4 5 are valid for BAR */
+	if (bir > 5 || bir == 1) {
+		goto exit;
+	}
+
+	/* unit size for 4KB/64KB/1MB/16MB/256MB/4GB/64GB */
+	unit_size = (uint64_t)1 << (12 + 4 * cmbsz.bits.szu);
+	/* controller memory buffer size in Bytes */
+	size = unit_size * cmbsz.bits.sz;
+	/* controller memory buffer offset from BAR in Bytes */
+	offset = unit_size * cmbloc.bits.ofst;
+
+	rc = spdk_pci_device_map_bar(pctrlr->devhandle, bir, &addr,
+				     &bar_phys_addr, &bar_size);
+	if ((rc != 0) || addr == NULL) {
+		goto exit;
+	}
+
+	if (offset > bar_size) {
+		goto exit;
+	}
+
+	if (size > bar_size - offset) {
+		goto exit;
+	}
+
+	pctrlr->cmb.bar_va = addr;
+	pctrlr->cmb.bar_pa = bar_phys_addr;
+	pctrlr->cmb.size = size;
+	pctrlr->cmb.current_offset = offset;
+
+	if (!cmbsz.bits.sqs) {
+		pctrlr->ctrlr.opts.use_cmb_sqs = false;
+	}
+
+	return;
+exit:
+	pctrlr->ctrlr.opts.use_cmb_sqs = false;
+	return;
+}
+
+static int
+nvme_pcie_ctrlr_unmap_cmb(struct nvme_pcie_ctrlr *pctrlr)
+{
+	int rc = 0;
+	union spdk_nvme_cmbloc_register cmbloc;
+	void *addr = pctrlr->cmb.bar_va;
+
+	if (addr) {
+		if (pctrlr->cmb.mem_register_addr) {
+			spdk_mem_unregister(pctrlr->cmb.mem_register_addr, pctrlr->cmb.mem_register_size);
+		}
+
+		if (nvme_pcie_ctrlr_get_cmbloc(pctrlr, &cmbloc)) {
+			SPDK_ERRLOG("get_cmbloc() failed\n");
+			return -EIO;
+		}
+		rc = spdk_pci_device_unmap_bar(pctrlr->devhandle, cmbloc.bits.bir, addr);
+	}
+	return rc;
+}
+
+static int
+nvme_pcie_ctrlr_reserve_cmb(struct spdk_nvme_ctrlr *ctrlr)
+{
+	struct nvme_pcie_ctrlr *pctrlr = nvme_pcie_ctrlr(ctrlr);
+
+	if (pctrlr->cmb.bar_va == NULL) {
+		SPDK_DEBUGLOG(SPDK_LOG_NVME, "CMB not available\n");
+		return -ENOTSUP;
+	}
+
+	if (ctrlr->opts.use_cmb_sqs) {
+		SPDK_ERRLOG("CMB is already in use for submission queues.\n");
+		return -ENOTSUP;
+	}
+
+	return 0;
+}
+
+static void *
+nvme_pcie_ctrlr_map_io_cmb(struct spdk_nvme_ctrlr *ctrlr, size_t *size)
+{
+	struct nvme_pcie_ctrlr *pctrlr = nvme_pcie_ctrlr(ctrlr);
+	union spdk_nvme_cmbsz_register cmbsz;
+	union spdk_nvme_cmbloc_register cmbloc;
+	uint64_t mem_register_start, mem_register_end;
+	int rc;
+
+	if (pctrlr->cmb.mem_register_addr != NULL) {
+		*size = pctrlr->cmb.mem_register_size;
+		return pctrlr->cmb.mem_register_addr;
+	}
+
+	*size = 0;
+
+	if (pctrlr->cmb.bar_va == NULL) {
+		SPDK_DEBUGLOG(SPDK_LOG_NVME, "CMB not available\n");
+		return NULL;
+	}
+
+	if (ctrlr->opts.use_cmb_sqs) {
+		SPDK_ERRLOG("CMB is already in use for submission queues.\n");
+		return NULL;
+	}
+
+	if (nvme_pcie_ctrlr_get_cmbsz(pctrlr, &cmbsz) ||
+	    nvme_pcie_ctrlr_get_cmbloc(pctrlr, &cmbloc)) {
+		SPDK_ERRLOG("get registers failed\n");
+		return NULL;
+	}
+
+	/* If only SQS is supported */
+	if (!(cmbsz.bits.wds || cmbsz.bits.rds)) {
+		return NULL;
+	}
+
+	/* If CMB is less than 4MiB in size then abort CMB mapping */
+	if (pctrlr->cmb.size < (1ULL << 22)) {
+		return NULL;
+	}
+
+	mem_register_start = _2MB_PAGE((uintptr_t)pctrlr->cmb.bar_va + pctrlr->cmb.current_offset +
+				       VALUE_2MB - 1);
+	mem_register_end = _2MB_PAGE((uintptr_t)pctrlr->cmb.bar_va + pctrlr->cmb.current_offset +
+				     pctrlr->cmb.size);
+	pctrlr->cmb.mem_register_addr = (void *)mem_register_start;
+	pctrlr->cmb.mem_register_size = mem_register_end - mem_register_start;
+
+	rc = spdk_mem_register((void *)mem_register_start, mem_register_end - mem_register_start);
+	if (rc) {
+		SPDK_ERRLOG("spdk_mem_register() failed\n");
+		return NULL;
+	}
+
+	pctrlr->cmb.mem_register_addr = (void *)mem_register_start;
+	pctrlr->cmb.mem_register_size = mem_register_end - mem_register_start;
+
+	*size = pctrlr->cmb.mem_register_size;
+	return pctrlr->cmb.mem_register_addr;
+}
+
+static int
+nvme_pcie_ctrlr_unmap_io_cmb(struct spdk_nvme_ctrlr *ctrlr)
+{
+	struct nvme_pcie_ctrlr *pctrlr = nvme_pcie_ctrlr(ctrlr);
+	int rc;
+
+	if (pctrlr->cmb.mem_register_addr == NULL) {
+		return 0;
+	}
+
+	rc = spdk_mem_unregister(pctrlr->cmb.mem_register_addr, pctrlr->cmb.mem_register_size);
+
+	if (rc == 0) {
+		pctrlr->cmb.mem_register_addr = NULL;
+		pctrlr->cmb.mem_register_size = 0;
+	}
+
+	return rc;
+}
+
+static int
+nvme_pcie_ctrlr_allocate_bars(struct nvme_pcie_ctrlr *pctrlr)
+{
+	int rc;
+	void *addr = NULL;
+	uint64_t phys_addr = 0, size = 0;
+
+	rc = spdk_pci_device_map_bar(pctrlr->devhandle, 0, &addr,
+				     &phys_addr, &size);
+
+	if ((addr == NULL) || (rc != 0)) {
+		SPDK_ERRLOG("nvme_pcicfg_map_bar failed with rc %d or bar %p\n",
+			    rc, addr);
+		return -1;
+	}
+
+	pctrlr->regs = (volatile struct spdk_nvme_registers *)addr;
+	pctrlr->regs_size = size;
+	nvme_pcie_ctrlr_map_cmb(pctrlr);
+
+	return 0;
+}
+
+static int
+nvme_pcie_ctrlr_free_bars(struct nvme_pcie_ctrlr *pctrlr)
+{
+	int rc = 0;
+	void *addr = (void *)pctrlr->regs;
+
+	if (pctrlr->ctrlr.is_removed) {
+		return rc;
+	}
+
+	rc = nvme_pcie_ctrlr_unmap_cmb(pctrlr);
+	if (rc != 0) {
+		SPDK_ERRLOG("nvme_ctrlr_unmap_cmb failed with error code %d\n", rc);
+		return -1;
+	}
+
+	if (addr) {
+		/* NOTE: addr may have been remapped here. We're relying on DPDK to call
+		 * munmap internally.
+		 */
+		rc = spdk_pci_device_unmap_bar(pctrlr->devhandle, 0, addr);
+	}
+	return rc;
+}
+
+static int
+nvme_pcie_ctrlr_construct_admin_qpair(struct spdk_nvme_ctrlr *ctrlr, uint16_t num_entries)
+{
+	struct nvme_pcie_qpair *pqpair;
+	int rc;
+
+	pqpair = spdk_zmalloc(sizeof(*pqpair), 64, NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_SHARE);
+	if (pqpair == NULL) {
+		return -ENOMEM;
+	}
+
+	pqpair->num_entries = num_entries;
+	pqpair->flags.delay_cmd_submit = 0;
+
+	ctrlr->adminq = &pqpair->qpair;
+
+	rc = nvme_qpair_init(ctrlr->adminq,
+			     0, /* qpair ID */
+			     ctrlr,
+			     SPDK_NVME_QPRIO_URGENT,
+			     num_entries);
+	if (rc != 0) {
+		return rc;
+	}
+
+	return nvme_pcie_qpair_construct(ctrlr->adminq, NULL);
+}
+
+/* This function must only be called while holding g_spdk_nvme_driver->lock */
+static int
+pcie_nvme_enum_cb(void *ctx, struct spdk_pci_device *pci_dev)
+{
+	struct spdk_nvme_transport_id trid = {};
+	struct nvme_pcie_enum_ctx *enum_ctx = ctx;
+	struct spdk_nvme_ctrlr *ctrlr;
+	struct spdk_pci_addr pci_addr;
+
+	pci_addr = spdk_pci_device_get_addr(pci_dev);
+
+	spdk_nvme_trid_populate_transport(&trid, SPDK_NVME_TRANSPORT_PCIE);
+	spdk_pci_addr_fmt(trid.traddr, sizeof(trid.traddr), &pci_addr);
+
+	ctrlr = nvme_get_ctrlr_by_trid_unsafe(&trid);
+	if (!spdk_process_is_primary()) {
+		if (!ctrlr) {
+			SPDK_ERRLOG("Controller must be constructed in the primary process first.\n");
+			return -1;
+		}
+
+		return nvme_ctrlr_add_process(ctrlr, pci_dev);
+	}
+
+	/* check whether user passes the pci_addr */
+	if (enum_ctx->has_pci_addr &&
+	    (spdk_pci_addr_compare(&pci_addr, &enum_ctx->pci_addr) != 0)) {
+		return 1;
+	}
+
+	return nvme_ctrlr_probe(&trid, enum_ctx->probe_ctx, pci_dev);
+}
+
+static int
+nvme_pcie_ctrlr_scan(struct spdk_nvme_probe_ctx *probe_ctx,
+		     bool direct_connect)
+{
+	struct nvme_pcie_enum_ctx enum_ctx = {};
+
+	enum_ctx.probe_ctx = probe_ctx;
+
+	if (strlen(probe_ctx->trid.traddr) != 0) {
+		if (spdk_pci_addr_parse(&enum_ctx.pci_addr, probe_ctx->trid.traddr)) {
+			return -1;
+		}
+		enum_ctx.has_pci_addr = true;
+	}
+
+	/* Only the primary process can monitor hotplug. */
+	if (spdk_process_is_primary()) {
+		_nvme_pcie_hotplug_monitor(probe_ctx);
+	}
+
+	if (enum_ctx.has_pci_addr == false) {
+		return spdk_pci_enumerate(spdk_pci_nvme_get_driver(),
+					  pcie_nvme_enum_cb, &enum_ctx);
+	} else {
+		return spdk_pci_device_attach(spdk_pci_nvme_get_driver(),
+					      pcie_nvme_enum_cb, &enum_ctx, &enum_ctx.pci_addr);
+	}
+}
+
+static int
+nvme_pcie_ctrlr_attach(struct spdk_nvme_probe_ctx *probe_ctx, struct spdk_pci_addr *pci_addr)
+{
+	struct nvme_pcie_enum_ctx enum_ctx;
+
+	enum_ctx.probe_ctx = probe_ctx;
+	enum_ctx.has_pci_addr = true;
+	enum_ctx.pci_addr = *pci_addr;
+
+	return spdk_pci_enumerate(spdk_pci_nvme_get_driver(), pcie_nvme_enum_cb, &enum_ctx);
+}
+
+static struct spdk_nvme_ctrlr *nvme_pcie_ctrlr_construct(const struct spdk_nvme_transport_id *trid,
+		const struct spdk_nvme_ctrlr_opts *opts,
+		void *devhandle)
+{
+	struct spdk_pci_device *pci_dev = devhandle;
+	struct nvme_pcie_ctrlr *pctrlr;
+	union spdk_nvme_cap_register cap;
+	union spdk_nvme_vs_register vs;
+	uint16_t cmd_reg;
+	int rc;
+	struct spdk_pci_id pci_id;
+
+	rc = spdk_pci_device_claim(pci_dev);
+	if (rc < 0) {
+		SPDK_ERRLOG("could not claim device %s (%s)\n",
+			    trid->traddr, spdk_strerror(-rc));
+		return NULL;
+	}
+
+	pctrlr = spdk_zmalloc(sizeof(struct nvme_pcie_ctrlr), 64, NULL,
+			      SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_SHARE);
+	if (pctrlr == NULL) {
+		spdk_pci_device_unclaim(pci_dev);
+		SPDK_ERRLOG("could not allocate ctrlr\n");
+		return NULL;
+	}
+
+	pctrlr->is_remapped = false;
+	pctrlr->ctrlr.is_removed = false;
+	pctrlr->devhandle = devhandle;
+	pctrlr->ctrlr.opts = *opts;
+	pctrlr->ctrlr.trid = *trid;
+
+	rc = nvme_ctrlr_construct(&pctrlr->ctrlr);
+	if (rc != 0) {
+		spdk_pci_device_unclaim(pci_dev);
+		spdk_free(pctrlr);
+		return NULL;
+	}
+
+	rc = nvme_pcie_ctrlr_allocate_bars(pctrlr);
+	if (rc != 0) {
+		spdk_pci_device_unclaim(pci_dev);
+		spdk_free(pctrlr);
+		return NULL;
+	}
+
+	/* Enable PCI busmaster and disable INTx */
+	spdk_pci_device_cfg_read16(pci_dev, &cmd_reg, 4);
+	cmd_reg |= 0x404;
+	spdk_pci_device_cfg_write16(pci_dev, cmd_reg, 4);
+
+	if (nvme_ctrlr_get_cap(&pctrlr->ctrlr, &cap)) {
+		SPDK_ERRLOG("get_cap() failed\n");
+		spdk_pci_device_unclaim(pci_dev);
+		spdk_free(pctrlr);
+		return NULL;
+	}
+
+	if (nvme_ctrlr_get_vs(&pctrlr->ctrlr, &vs)) {
+		SPDK_ERRLOG("get_vs() failed\n");
+		spdk_pci_device_unclaim(pci_dev);
+		spdk_free(pctrlr);
+		return NULL;
+	}
+
+	nvme_ctrlr_init_cap(&pctrlr->ctrlr, &cap, &vs);
+
+	/* Doorbell stride is 2 ^ (dstrd + 2),
+	 * but we want multiples of 4, so drop the + 2 */
+	pctrlr->doorbell_stride_u32 = 1 << cap.bits.dstrd;
+
+	pci_id = spdk_pci_device_get_id(pci_dev);
+	pctrlr->ctrlr.quirks = nvme_get_quirks(&pci_id);
+
+	rc = nvme_pcie_ctrlr_construct_admin_qpair(&pctrlr->ctrlr, pctrlr->ctrlr.opts.admin_queue_size);
+	if (rc != 0) {
+		nvme_ctrlr_destruct(&pctrlr->ctrlr);
+		return NULL;
+	}
+
+	/* Construct the primary process properties */
+	rc = nvme_ctrlr_add_process(&pctrlr->ctrlr, pci_dev);
+	if (rc != 0) {
+		nvme_ctrlr_destruct(&pctrlr->ctrlr);
+		return NULL;
+	}
+
+	if (g_sigset != true) {
+		nvme_pcie_ctrlr_setup_signal();
+		g_sigset = true;
+	}
+
+	return &pctrlr->ctrlr;
+}
+
+static int
+nvme_pcie_ctrlr_enable(struct spdk_nvme_ctrlr *ctrlr)
+{
+	struct nvme_pcie_ctrlr *pctrlr = nvme_pcie_ctrlr(ctrlr);
+	struct nvme_pcie_qpair *padminq = nvme_pcie_qpair(ctrlr->adminq);
+	union spdk_nvme_aqa_register aqa;
+
+	if (nvme_pcie_ctrlr_set_asq(pctrlr, padminq->cmd_bus_addr)) {
+		SPDK_ERRLOG("set_asq() failed\n");
+		return -EIO;
+	}
+
+	if (nvme_pcie_ctrlr_set_acq(pctrlr, padminq->cpl_bus_addr)) {
+		SPDK_ERRLOG("set_acq() failed\n");
+		return -EIO;
+	}
+
+	aqa.raw = 0;
+	/* acqs and asqs are 0-based. */
+	aqa.bits.acqs = nvme_pcie_qpair(ctrlr->adminq)->num_entries - 1;
+	aqa.bits.asqs = nvme_pcie_qpair(ctrlr->adminq)->num_entries - 1;
+
+	if (nvme_pcie_ctrlr_set_aqa(pctrlr, &aqa)) {
+		SPDK_ERRLOG("set_aqa() failed\n");
+		return -EIO;
+	}
+
+	return 0;
+}
+
+static int
+nvme_pcie_ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr)
+{
+	struct nvme_pcie_ctrlr *pctrlr = nvme_pcie_ctrlr(ctrlr);
+	struct spdk_pci_device *devhandle = nvme_ctrlr_proc_get_devhandle(ctrlr);
+
+	if (ctrlr->adminq) {
+		nvme_pcie_qpair_destroy(ctrlr->adminq);
+	}
+
+	nvme_ctrlr_destruct_finish(ctrlr);
+
+	nvme_ctrlr_free_processes(ctrlr);
+
+	nvme_pcie_ctrlr_free_bars(pctrlr);
+
+	if (devhandle) {
+		spdk_pci_device_unclaim(devhandle);
+		spdk_pci_device_detach(devhandle);
+	}
+
+	spdk_free(pctrlr);
+
+	return 0;
+}
+
+static void
+nvme_qpair_construct_tracker(struct nvme_tracker *tr, uint16_t cid, uint64_t phys_addr)
+{
+	tr->prp_sgl_bus_addr = phys_addr + offsetof(struct nvme_tracker, u.prp);
+	tr->cid = cid;
+	tr->req = NULL;
+}
+
+static int
+nvme_pcie_qpair_reset(struct spdk_nvme_qpair *qpair)
+{
+	struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
+	uint32_t i;
+
+	/* all head/tail vals are set to 0 */
+	pqpair->last_sq_tail = pqpair->sq_tail = pqpair->sq_head = pqpair->cq_head = 0;
+
+	/*
+	 * First time through the completion queue, HW will set phase
+	 *  bit on completions to 1.  So set this to 1 here, indicating
+	 *  we're looking for a 1 to know which entries have completed.
+	 *  we'll toggle the bit each time when the completion queue
+	 *  rolls over.
+	 */
+	pqpair->flags.phase = 1;
+	for (i = 0; i < pqpair->num_entries; i++) {
+		pqpair->cpl[i].status.p = 0;
+	}
+
+	return 0;
+}
+
+static void *
+nvme_pcie_ctrlr_alloc_cmb(struct spdk_nvme_ctrlr *ctrlr, uint64_t size, uint64_t alignment,
+			  uint64_t *phys_addr)
+{
+	struct nvme_pcie_ctrlr *pctrlr = nvme_pcie_ctrlr(ctrlr);
+	uintptr_t addr;
+
+	if (pctrlr->cmb.mem_register_addr != NULL) {
+		/* BAR is mapped for data */
+		return NULL;
+	}
+
+	addr = (uintptr_t)pctrlr->cmb.bar_va + pctrlr->cmb.current_offset;
+	addr = (addr + (alignment - 1)) & ~(alignment - 1);
+
+	/* CMB may only consume part of the BAR, calculate accordingly */
+	if (addr + size > ((uintptr_t)pctrlr->cmb.bar_va + pctrlr->cmb.size)) {
+		SPDK_ERRLOG("Tried to allocate past valid CMB range!\n");
+		return NULL;
+	}
+	*phys_addr = pctrlr->cmb.bar_pa + addr - (uintptr_t)pctrlr->cmb.bar_va;
+
+	pctrlr->cmb.current_offset = (addr + size) - (uintptr_t)pctrlr->cmb.bar_va;
+
+	return (void *)addr;
+}
+
+static int
+nvme_pcie_qpair_construct(struct spdk_nvme_qpair *qpair,
+			  const struct spdk_nvme_io_qpair_opts *opts)
+{
+	struct spdk_nvme_ctrlr	*ctrlr = qpair->ctrlr;
+	struct nvme_pcie_ctrlr	*pctrlr = nvme_pcie_ctrlr(ctrlr);
+	struct nvme_pcie_qpair	*pqpair = nvme_pcie_qpair(qpair);
+	struct nvme_tracker	*tr;
+	uint16_t		i;
+	volatile uint32_t	*doorbell_base;
+	uint16_t		num_trackers;
+	size_t			page_align = sysconf(_SC_PAGESIZE);
+	size_t			queue_align, queue_len;
+	uint32_t                flags = SPDK_MALLOC_DMA;
+	uint64_t		sq_paddr = 0;
+	uint64_t		cq_paddr = 0;
+
+	if (opts) {
+		pqpair->sq_vaddr = opts->sq.vaddr;
+		pqpair->cq_vaddr = opts->cq.vaddr;
+		sq_paddr = opts->sq.paddr;
+		cq_paddr = opts->cq.paddr;
+	}
+
+	pqpair->retry_count = ctrlr->opts.transport_retry_count;
+
+	/*
+	 * Limit the maximum number of completions to return per call to prevent wraparound,
+	 * and calculate how many trackers can be submitted at once without overflowing the
+	 * completion queue.
+	 */
+	pqpair->max_completions_cap = pqpair->num_entries / 4;
+	pqpair->max_completions_cap = spdk_max(pqpair->max_completions_cap, NVME_MIN_COMPLETIONS);
+	pqpair->max_completions_cap = spdk_min(pqpair->max_completions_cap, NVME_MAX_COMPLETIONS);
+	num_trackers = pqpair->num_entries - pqpair->max_completions_cap;
+
+	SPDK_INFOLOG(SPDK_LOG_NVME, "max_completions_cap = %" PRIu16 " num_trackers = %" PRIu16 "\n",
+		     pqpair->max_completions_cap, num_trackers);
+
+	assert(num_trackers != 0);
+
+	pqpair->sq_in_cmb = false;
+
+	if (nvme_qpair_is_admin_queue(&pqpair->qpair)) {
+		flags |= SPDK_MALLOC_SHARE;
+	}
+
+	/* cmd and cpl rings must be aligned on page size boundaries. */
+	if (ctrlr->opts.use_cmb_sqs) {
+		pqpair->cmd = nvme_pcie_ctrlr_alloc_cmb(ctrlr, pqpair->num_entries * sizeof(struct spdk_nvme_cmd),
+							page_align, &pqpair->cmd_bus_addr);
+		if (pqpair->cmd != NULL) {
+			pqpair->sq_in_cmb = true;
+		}
+	}
+
+	if (pqpair->sq_in_cmb == false) {
+		if (pqpair->sq_vaddr) {
+			pqpair->cmd = pqpair->sq_vaddr;
+		} else {
+			/* To ensure physical address contiguity we make each ring occupy
+			 * a single hugepage only. See MAX_IO_QUEUE_ENTRIES.
+			 */
+			queue_len = pqpair->num_entries * sizeof(struct spdk_nvme_cmd);
+			queue_align = spdk_max(spdk_align32pow2(queue_len), page_align);
+			pqpair->cmd = spdk_zmalloc(queue_len, queue_align, NULL, SPDK_ENV_SOCKET_ID_ANY, flags);
+			if (pqpair->cmd == NULL) {
+				SPDK_ERRLOG("alloc qpair_cmd failed\n");
+				return -ENOMEM;
+			}
+		}
+		if (sq_paddr) {
+			assert(pqpair->sq_vaddr != NULL);
+			pqpair->cmd_bus_addr = sq_paddr;
+		} else {
+			pqpair->cmd_bus_addr = spdk_vtophys(pqpair->cmd, NULL);
+			if (pqpair->cmd_bus_addr == SPDK_VTOPHYS_ERROR) {
+				SPDK_ERRLOG("spdk_vtophys(pqpair->cmd) failed\n");
+				return -EFAULT;
+			}
+		}
+	}
+
+	if (pqpair->cq_vaddr) {
+		pqpair->cpl = pqpair->cq_vaddr;
+	} else {
+		queue_len = pqpair->num_entries * sizeof(struct spdk_nvme_cpl);
+		queue_align = spdk_max(spdk_align32pow2(queue_len), page_align);
+		pqpair->cpl = spdk_zmalloc(queue_len, queue_align, NULL, SPDK_ENV_SOCKET_ID_ANY, flags);
+		if (pqpair->cpl == NULL) {
+			SPDK_ERRLOG("alloc qpair_cpl failed\n");
+			return -ENOMEM;
+		}
+	}
+	if (cq_paddr) {
+		assert(pqpair->cq_vaddr != NULL);
+		pqpair->cpl_bus_addr = cq_paddr;
+	} else {
+		pqpair->cpl_bus_addr = spdk_vtophys(pqpair->cpl, NULL);
+		if (pqpair->cpl_bus_addr == SPDK_VTOPHYS_ERROR) {
+			SPDK_ERRLOG("spdk_vtophys(pqpair->cpl) failed\n");
+			return -EFAULT;
+		}
+	}
+
+	doorbell_base = &pctrlr->regs->doorbell[0].sq_tdbl;
+	pqpair->sq_tdbl = doorbell_base + (2 * qpair->id + 0) * pctrlr->doorbell_stride_u32;
+	pqpair->cq_hdbl = doorbell_base + (2 * qpair->id + 1) * pctrlr->doorbell_stride_u32;
+
+	/*
+	 * Reserve space for all of the trackers in a single allocation.
+	 *   struct nvme_tracker must be padded so that its size is already a power of 2.
+	 *   This ensures the PRP list embedded in the nvme_tracker object will not span a
+	 *   4KB boundary, while allowing access to trackers in tr[] via normal array indexing.
+	 */
+	pqpair->tr = spdk_zmalloc(num_trackers * sizeof(*tr), sizeof(*tr), NULL,
+				  SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_SHARE);
+	if (pqpair->tr == NULL) {
+		SPDK_ERRLOG("nvme_tr failed\n");
+		return -ENOMEM;
+	}
+
+	TAILQ_INIT(&pqpair->free_tr);
+	TAILQ_INIT(&pqpair->outstanding_tr);
+
+	for (i = 0; i < num_trackers; i++) {
+		tr = &pqpair->tr[i];
+		nvme_qpair_construct_tracker(tr, i, spdk_vtophys(tr, NULL));
+		TAILQ_INSERT_HEAD(&pqpair->free_tr, tr, tq_list);
+	}
+
+	nvme_pcie_qpair_reset(qpair);
+
+	return 0;
+}
+
+/* Used when dst points to MMIO (i.e. CMB) in a virtual machine - in these cases we must
+ * not use wide instructions because QEMU will not emulate such instructions to MMIO space.
+ * So this function ensures we only copy 8 bytes at a time.
+ */
+static inline void
+nvme_pcie_copy_command_mmio(struct spdk_nvme_cmd *dst, const struct spdk_nvme_cmd *src)
+{
+	uint64_t *dst64 = (uint64_t *)dst;
+	const uint64_t *src64 = (const uint64_t *)src;
+	uint32_t i;
+
+	for (i = 0; i < sizeof(*dst) / 8; i++) {
+		dst64[i] = src64[i];
+	}
+}
+
+static inline void
+nvme_pcie_copy_command(struct spdk_nvme_cmd *dst, const struct spdk_nvme_cmd *src)
+{
+	/* dst and src are known to be non-overlapping and 64-byte aligned. */
+#if defined(__SSE2__)
+	__m128i *d128 = (__m128i *)dst;
+	const __m128i *s128 = (const __m128i *)src;
+
+	_mm_stream_si128(&d128[0], _mm_load_si128(&s128[0]));
+	_mm_stream_si128(&d128[1], _mm_load_si128(&s128[1]));
+	_mm_stream_si128(&d128[2], _mm_load_si128(&s128[2]));
+	_mm_stream_si128(&d128[3], _mm_load_si128(&s128[3]));
+#else
+	*dst = *src;
+#endif
+}
+
+/**
+ * Note: the ctrlr_lock must be held when calling this function.
+ */
+static void
+nvme_pcie_qpair_insert_pending_admin_request(struct spdk_nvme_qpair *qpair,
+		struct nvme_request *req, struct spdk_nvme_cpl *cpl)
+{
+	struct spdk_nvme_ctrlr		*ctrlr = qpair->ctrlr;
+	struct nvme_request		*active_req = req;
+	struct spdk_nvme_ctrlr_process	*active_proc;
+
+	/*
+	 * The admin request is from another process. Move to the per
+	 *  process list for that process to handle it later.
+	 */
+	assert(nvme_qpair_is_admin_queue(qpair));
+	assert(active_req->pid != getpid());
+
+	active_proc = nvme_ctrlr_get_process(ctrlr, active_req->pid);
+	if (active_proc) {
+		/* Save the original completion information */
+		memcpy(&active_req->cpl, cpl, sizeof(*cpl));
+		STAILQ_INSERT_TAIL(&active_proc->active_reqs, active_req, stailq);
+	} else {
+		SPDK_ERRLOG("The owning process (pid %d) is not found. Dropping the request.\n",
+			    active_req->pid);
+
+		nvme_free_request(active_req);
+	}
+}
+
+/**
+ * Note: the ctrlr_lock must be held when calling this function.
+ */
+static void
+nvme_pcie_qpair_complete_pending_admin_request(struct spdk_nvme_qpair *qpair)
+{
+	struct spdk_nvme_ctrlr		*ctrlr = qpair->ctrlr;
+	struct nvme_request		*req, *tmp_req;
+	pid_t				pid = getpid();
+	struct spdk_nvme_ctrlr_process	*proc;
+
+	/*
+	 * Check whether there is any pending admin request from
+	 * other active processes.
+	 */
+	assert(nvme_qpair_is_admin_queue(qpair));
+
+	proc = nvme_ctrlr_get_current_process(ctrlr);
+	if (!proc) {
+		SPDK_ERRLOG("the active process (pid %d) is not found for this controller.\n", pid);
+		assert(proc);
+		return;
+	}
+
+	STAILQ_FOREACH_SAFE(req, &proc->active_reqs, stailq, tmp_req) {
+		STAILQ_REMOVE(&proc->active_reqs, req, nvme_request, stailq);
+
+		assert(req->pid == pid);
+
+		nvme_complete_request(req->cb_fn, req->cb_arg, qpair, req, &req->cpl);
+		nvme_free_request(req);
+	}
+}
+
+static inline int
+nvme_pcie_qpair_need_event(uint16_t event_idx, uint16_t new_idx, uint16_t old)
+{
+	return (uint16_t)(new_idx - event_idx) <= (uint16_t)(new_idx - old);
+}
+
+static bool
+nvme_pcie_qpair_update_mmio_required(struct spdk_nvme_qpair *qpair, uint16_t value,
+				     volatile uint32_t *shadow_db,
+				     volatile uint32_t *eventidx)
+{
+	uint16_t old;
+
+	if (!shadow_db) {
+		return true;
+	}
+
+	old = *shadow_db;
+	*shadow_db = value;
+
+	/*
+	 * Ensure that the doorbell is updated before reading the EventIdx from
+	 * memory
+	 */
+	spdk_mb();
+
+	if (!nvme_pcie_qpair_need_event(*eventidx, value, old)) {
+		return false;
+	}
+
+	return true;
+}
+
+static inline void
+nvme_pcie_qpair_ring_sq_doorbell(struct spdk_nvme_qpair *qpair)
+{
+	struct nvme_pcie_qpair	*pqpair = nvme_pcie_qpair(qpair);
+	struct nvme_pcie_ctrlr	*pctrlr = nvme_pcie_ctrlr(qpair->ctrlr);
+	bool need_mmio = true;
+
+	if (qpair->first_fused_submitted) {
+		/* This is first cmd of two fused commands - don't ring doorbell */
+		qpair->first_fused_submitted = 0;
+		return;
+	}
+
+	if (spdk_unlikely(pqpair->flags.has_shadow_doorbell)) {
+		need_mmio = nvme_pcie_qpair_update_mmio_required(qpair,
+				pqpair->sq_tail,
+				pqpair->shadow_doorbell.sq_tdbl,
+				pqpair->shadow_doorbell.sq_eventidx);
+	}
+
+	if (spdk_likely(need_mmio)) {
+		spdk_wmb();
+		g_thread_mmio_ctrlr = pctrlr;
+		spdk_mmio_write_4(pqpair->sq_tdbl, pqpair->sq_tail);
+		g_thread_mmio_ctrlr = NULL;
+	}
+}
+
+static inline void
+nvme_pcie_qpair_ring_cq_doorbell(struct spdk_nvme_qpair *qpair)
+{
+	struct nvme_pcie_qpair	*pqpair = nvme_pcie_qpair(qpair);
+	struct nvme_pcie_ctrlr	*pctrlr = nvme_pcie_ctrlr(qpair->ctrlr);
+	bool need_mmio = true;
+
+	if (spdk_unlikely(pqpair->flags.has_shadow_doorbell)) {
+		need_mmio = nvme_pcie_qpair_update_mmio_required(qpair,
+				pqpair->cq_head,
+				pqpair->shadow_doorbell.cq_hdbl,
+				pqpair->shadow_doorbell.cq_eventidx);
+	}
+
+	if (spdk_likely(need_mmio)) {
+		g_thread_mmio_ctrlr = pctrlr;
+		spdk_mmio_write_4(pqpair->cq_hdbl, pqpair->cq_head);
+		g_thread_mmio_ctrlr = NULL;
+	}
+}
+
+static void
+nvme_pcie_qpair_submit_tracker(struct spdk_nvme_qpair *qpair, struct nvme_tracker *tr)
+{
+	struct nvme_request	*req;
+	struct nvme_pcie_qpair	*pqpair = nvme_pcie_qpair(qpair);
+	struct spdk_nvme_ctrlr	*ctrlr = qpair->ctrlr;
+
+	req = tr->req;
+	assert(req != NULL);
+
+	if (req->cmd.fuse == SPDK_NVME_IO_FLAGS_FUSE_FIRST) {
+		/* This is first cmd of two fused commands - don't ring doorbell */
+		qpair->first_fused_submitted = 1;
+	}
+
+	/* Don't use wide instructions to copy NVMe command, this is limited by QEMU
+	 * virtual NVMe controller, the maximum access width is 8 Bytes for one time.
+	 */
+	if (spdk_unlikely((ctrlr->quirks & NVME_QUIRK_MAXIMUM_PCI_ACCESS_WIDTH) && pqpair->sq_in_cmb)) {
+		nvme_pcie_copy_command_mmio(&pqpair->cmd[pqpair->sq_tail], &req->cmd);
+	} else {
+		/* Copy the command from the tracker to the submission queue. */
+		nvme_pcie_copy_command(&pqpair->cmd[pqpair->sq_tail], &req->cmd);
+	}
+
+	if (spdk_unlikely(++pqpair->sq_tail == pqpair->num_entries)) {
+		pqpair->sq_tail = 0;
+	}
+
+	if (spdk_unlikely(pqpair->sq_tail == pqpair->sq_head)) {
+		SPDK_ERRLOG("sq_tail is passing sq_head!\n");
+	}
+
+	if (!pqpair->flags.delay_cmd_submit) {
+		nvme_pcie_qpair_ring_sq_doorbell(qpair);
+	}
+}
+
+static void
+nvme_pcie_qpair_complete_tracker(struct spdk_nvme_qpair *qpair, struct nvme_tracker *tr,
+				 struct spdk_nvme_cpl *cpl, bool print_on_error)
+{
+	struct nvme_pcie_qpair		*pqpair = nvme_pcie_qpair(qpair);
+	struct nvme_request		*req;
+	bool				retry, error;
+	bool				req_from_current_proc = true;
+
+	req = tr->req;
+
+	assert(req != NULL);
+
+	error = spdk_nvme_cpl_is_error(cpl);
+	retry = error && nvme_completion_is_retry(cpl) &&
+		req->retries < pqpair->retry_count;
+
+	if (error && print_on_error && !qpair->ctrlr->opts.disable_error_logging) {
+		spdk_nvme_qpair_print_command(qpair, &req->cmd);
+		spdk_nvme_qpair_print_completion(qpair, cpl);
+	}
+
+	assert(cpl->cid == req->cmd.cid);
+
+	if (retry) {
+		req->retries++;
+		nvme_pcie_qpair_submit_tracker(qpair, tr);
+	} else {
+		TAILQ_REMOVE(&pqpair->outstanding_tr, tr, tq_list);
+
+		/* Only check admin requests from different processes. */
+		if (nvme_qpair_is_admin_queue(qpair) && req->pid != getpid()) {
+			req_from_current_proc = false;
+			nvme_pcie_qpair_insert_pending_admin_request(qpair, req, cpl);
+		} else {
+			nvme_complete_request(tr->cb_fn, tr->cb_arg, qpair, req, cpl);
+		}
+
+		if (req_from_current_proc == true) {
+			nvme_qpair_free_request(qpair, req);
+		}
+
+		tr->req = NULL;
+
+		TAILQ_INSERT_HEAD(&pqpair->free_tr, tr, tq_list);
+	}
+}
+
+static void
+nvme_pcie_qpair_manual_complete_tracker(struct spdk_nvme_qpair *qpair,
+					struct nvme_tracker *tr, uint32_t sct, uint32_t sc, uint32_t dnr,
+					bool print_on_error)
+{
+	struct spdk_nvme_cpl	cpl;
+
+	memset(&cpl, 0, sizeof(cpl));
+	cpl.sqid = qpair->id;
+	cpl.cid = tr->cid;
+	cpl.status.sct = sct;
+	cpl.status.sc = sc;
+	cpl.status.dnr = dnr;
+	nvme_pcie_qpair_complete_tracker(qpair, tr, &cpl, print_on_error);
+}
+
+static void
+nvme_pcie_qpair_abort_trackers(struct spdk_nvme_qpair *qpair, uint32_t dnr)
+{
+	struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
+	struct nvme_tracker *tr, *temp, *last;
+
+	last = TAILQ_LAST(&pqpair->outstanding_tr, nvme_outstanding_tr_head);
+
+	/* Abort previously submitted (outstanding) trs */
+	TAILQ_FOREACH_SAFE(tr, &pqpair->outstanding_tr, tq_list, temp) {
+		if (!qpair->ctrlr->opts.disable_error_logging) {
+			SPDK_ERRLOG("aborting outstanding command\n");
+		}
+		nvme_pcie_qpair_manual_complete_tracker(qpair, tr, SPDK_NVME_SCT_GENERIC,
+							SPDK_NVME_SC_ABORTED_BY_REQUEST, dnr, true);
+
+		if (tr == last) {
+			break;
+		}
+	}
+}
+
+static int
+nvme_pcie_qpair_iterate_requests(struct spdk_nvme_qpair *qpair,
+				 int (*iter_fn)(struct nvme_request *req, void *arg),
+				 void *arg)
+{
+	struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
+	struct nvme_tracker *tr, *tmp;
+	int rc;
+
+	assert(iter_fn != NULL);
+
+	TAILQ_FOREACH_SAFE(tr, &pqpair->outstanding_tr, tq_list, tmp) {
+		assert(tr->req != NULL);
+
+		rc = iter_fn(tr->req, arg);
+		if (rc != 0) {
+			return rc;
+		}
+	}
+
+	return 0;
+}
+
+static void
+nvme_pcie_admin_qpair_abort_aers(struct spdk_nvme_qpair *qpair)
+{
+	struct nvme_pcie_qpair	*pqpair = nvme_pcie_qpair(qpair);
+	struct nvme_tracker	*tr;
+
+	tr = TAILQ_FIRST(&pqpair->outstanding_tr);
+	while (tr != NULL) {
+		assert(tr->req != NULL);
+		if (tr->req->cmd.opc == SPDK_NVME_OPC_ASYNC_EVENT_REQUEST) {
+			nvme_pcie_qpair_manual_complete_tracker(qpair, tr,
+								SPDK_NVME_SCT_GENERIC, SPDK_NVME_SC_ABORTED_SQ_DELETION, 0,
+								false);
+			tr = TAILQ_FIRST(&pqpair->outstanding_tr);
+		} else {
+			tr = TAILQ_NEXT(tr, tq_list);
+		}
+	}
+}
+
+static void
+nvme_pcie_admin_qpair_destroy(struct spdk_nvme_qpair *qpair)
+{
+	nvme_pcie_admin_qpair_abort_aers(qpair);
+}
+
+static int
+nvme_pcie_qpair_destroy(struct spdk_nvme_qpair *qpair)
+{
+	struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
+
+	if (nvme_qpair_is_admin_queue(qpair)) {
+		nvme_pcie_admin_qpair_destroy(qpair);
+	}
+	/*
+	 * We check sq_vaddr and cq_vaddr to see if the user specified the memory
+	 * buffers when creating the I/O queue.
+	 * If the user specified them, we cannot free that memory.
+	 * Nor do we free it if it's in the CMB.
+	 */
+	if (!pqpair->sq_vaddr && pqpair->cmd && !pqpair->sq_in_cmb) {
+		spdk_free(pqpair->cmd);
+	}
+	if (!pqpair->cq_vaddr && pqpair->cpl) {
+		spdk_free(pqpair->cpl);
+	}
+	if (pqpair->tr) {
+		spdk_free(pqpair->tr);
+	}
+
+	nvme_qpair_deinit(qpair);
+
+	spdk_free(pqpair);
+
+	return 0;
+}
+
+static void
+nvme_pcie_qpair_abort_reqs(struct spdk_nvme_qpair *qpair, uint32_t dnr)
+{
+	nvme_pcie_qpair_abort_trackers(qpair, dnr);
+}
+
+static int
+nvme_pcie_ctrlr_cmd_create_io_cq(struct spdk_nvme_ctrlr *ctrlr,
+				 struct spdk_nvme_qpair *io_que, spdk_nvme_cmd_cb cb_fn,
+				 void *cb_arg)
+{
+	struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(io_que);
+	struct nvme_request *req;
+	struct spdk_nvme_cmd *cmd;
+
+	req = nvme_allocate_request_null(ctrlr->adminq, cb_fn, cb_arg);
+	if (req == NULL) {
+		return -ENOMEM;
+	}
+
+	cmd = &req->cmd;
+	cmd->opc = SPDK_NVME_OPC_CREATE_IO_CQ;
+
+	cmd->cdw10_bits.create_io_q.qid = io_que->id;
+	cmd->cdw10_bits.create_io_q.qsize = pqpair->num_entries - 1;
+
+	cmd->cdw11_bits.create_io_cq.pc = 1;
+	cmd->dptr.prp.prp1 = pqpair->cpl_bus_addr;
+
+	return nvme_ctrlr_submit_admin_request(ctrlr, req);
+}
+
+static int
+nvme_pcie_ctrlr_cmd_create_io_sq(struct spdk_nvme_ctrlr *ctrlr,
+				 struct spdk_nvme_qpair *io_que, spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(io_que);
+	struct nvme_request *req;
+	struct spdk_nvme_cmd *cmd;
+
+	req = nvme_allocate_request_null(ctrlr->adminq, cb_fn, cb_arg);
+	if (req == NULL) {
+		return -ENOMEM;
+	}
+
+	cmd = &req->cmd;
+	cmd->opc = SPDK_NVME_OPC_CREATE_IO_SQ;
+
+	cmd->cdw10_bits.create_io_q.qid = io_que->id;
+	cmd->cdw10_bits.create_io_q.qsize = pqpair->num_entries - 1;
+	cmd->cdw11_bits.create_io_sq.pc = 1;
+	cmd->cdw11_bits.create_io_sq.qprio = io_que->qprio;
+	cmd->cdw11_bits.create_io_sq.cqid = io_que->id;
+	cmd->dptr.prp.prp1 = pqpair->cmd_bus_addr;
+
+	return nvme_ctrlr_submit_admin_request(ctrlr, req);
+}
+
+static int
+nvme_pcie_ctrlr_cmd_delete_io_cq(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair,
+				 spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	struct nvme_request *req;
+	struct spdk_nvme_cmd *cmd;
+
+	req = nvme_allocate_request_null(ctrlr->adminq, cb_fn, cb_arg);
+	if (req == NULL) {
+		return -ENOMEM;
+	}
+
+	cmd = &req->cmd;
+	cmd->opc = SPDK_NVME_OPC_DELETE_IO_CQ;
+	cmd->cdw10_bits.delete_io_q.qid = qpair->id;
+
+	return nvme_ctrlr_submit_admin_request(ctrlr, req);
+}
+
+static int
+nvme_pcie_ctrlr_cmd_delete_io_sq(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair,
+				 spdk_nvme_cmd_cb cb_fn, void *cb_arg)
+{
+	struct nvme_request *req;
+	struct spdk_nvme_cmd *cmd;
+
+	req = nvme_allocate_request_null(ctrlr->adminq, cb_fn, cb_arg);
+	if (req == NULL) {
+		return -ENOMEM;
+	}
+
+	cmd = &req->cmd;
+	cmd->opc = SPDK_NVME_OPC_DELETE_IO_SQ;
+	cmd->cdw10_bits.delete_io_q.qid = qpair->id;
+
+	return nvme_ctrlr_submit_admin_request(ctrlr, req);
+}
+
+static int
+_nvme_pcie_ctrlr_create_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair,
+				 uint16_t qid)
+{
+	struct nvme_pcie_ctrlr	*pctrlr = nvme_pcie_ctrlr(ctrlr);
+	struct nvme_pcie_qpair	*pqpair = nvme_pcie_qpair(qpair);
+	struct nvme_completion_poll_status	*status;
+	int					rc;
+
+	status = calloc(1, sizeof(*status));
+	if (!status) {
+		SPDK_ERRLOG("Failed to allocate status tracker\n");
+		return -ENOMEM;
+	}
+
+	rc = nvme_pcie_ctrlr_cmd_create_io_cq(ctrlr, qpair, nvme_completion_poll_cb, status);
+	if (rc != 0) {
+		free(status);
+		return rc;
+	}
+
+	if (nvme_wait_for_completion(ctrlr->adminq, status)) {
+		SPDK_ERRLOG("nvme_create_io_cq failed!\n");
+		if (!status->timed_out) {
+			free(status);
+		}
+		return -1;
+	}
+
+	memset(status, 0, sizeof(*status));
+	rc = nvme_pcie_ctrlr_cmd_create_io_sq(qpair->ctrlr, qpair, nvme_completion_poll_cb, status);
+	if (rc != 0) {
+		free(status);
+		return rc;
+	}
+
+	if (nvme_wait_for_completion(ctrlr->adminq, status)) {
+		SPDK_ERRLOG("nvme_create_io_sq failed!\n");
+		if (status->timed_out) {
+			/* Request is still queued, the memory will be freed in a completion callback.
+			   allocate a new request */
+			status = calloc(1, sizeof(*status));
+			if (!status) {
+				SPDK_ERRLOG("Failed to allocate status tracker\n");
+				return -ENOMEM;
+			}
+		}
+
+		memset(status, 0, sizeof(*status));
+		/* Attempt to delete the completion queue */
+		rc = nvme_pcie_ctrlr_cmd_delete_io_cq(qpair->ctrlr, qpair, nvme_completion_poll_cb, status);
+		if (rc != 0) {
+			/* The originall or newly allocated status structure can be freed since
+			 * the corresponding request has been completed of failed to submit */
+			free(status);
+			return -1;
+		}
+		nvme_wait_for_completion(ctrlr->adminq, status);
+		if (!status->timed_out) {
+			/* status can be freed regardless of nvme_wait_for_completion return value */
+			free(status);
+		}
+		return -1;
+	}
+
+	if (ctrlr->shadow_doorbell) {
+		pqpair->shadow_doorbell.sq_tdbl = ctrlr->shadow_doorbell + (2 * qpair->id + 0) *
+						  pctrlr->doorbell_stride_u32;
+		pqpair->shadow_doorbell.cq_hdbl = ctrlr->shadow_doorbell + (2 * qpair->id + 1) *
+						  pctrlr->doorbell_stride_u32;
+		pqpair->shadow_doorbell.sq_eventidx = ctrlr->eventidx + (2 * qpair->id + 0) *
+						      pctrlr->doorbell_stride_u32;
+		pqpair->shadow_doorbell.cq_eventidx = ctrlr->eventidx + (2 * qpair->id + 1) *
+						      pctrlr->doorbell_stride_u32;
+		pqpair->flags.has_shadow_doorbell = 1;
+	} else {
+		pqpair->flags.has_shadow_doorbell = 0;
+	}
+	nvme_pcie_qpair_reset(qpair);
+	free(status);
+
+	return 0;
+}
+
+static struct spdk_nvme_qpair *
+nvme_pcie_ctrlr_create_io_qpair(struct spdk_nvme_ctrlr *ctrlr, uint16_t qid,
+				const struct spdk_nvme_io_qpair_opts *opts)
+{
+	struct nvme_pcie_qpair *pqpair;
+	struct spdk_nvme_qpair *qpair;
+	int rc;
+
+	assert(ctrlr != NULL);
+
+	pqpair = spdk_zmalloc(sizeof(*pqpair), 64, NULL,
+			      SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_SHARE);
+	if (pqpair == NULL) {
+		return NULL;
+	}
+
+	pqpair->num_entries = opts->io_queue_size;
+	pqpair->flags.delay_cmd_submit = opts->delay_cmd_submit;
+
+	qpair = &pqpair->qpair;
+
+	rc = nvme_qpair_init(qpair, qid, ctrlr, opts->qprio, opts->io_queue_requests);
+	if (rc != 0) {
+		nvme_pcie_qpair_destroy(qpair);
+		return NULL;
+	}
+
+	rc = nvme_pcie_qpair_construct(qpair, opts);
+
+	if (rc != 0) {
+		nvme_pcie_qpair_destroy(qpair);
+		return NULL;
+	}
+
+	return qpair;
+}
+
+static int
+nvme_pcie_ctrlr_connect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)
+{
+	if (nvme_qpair_is_admin_queue(qpair)) {
+		return 0;
+	} else {
+		return _nvme_pcie_ctrlr_create_io_qpair(ctrlr, qpair, qpair->id);
+	}
+}
+
+static void
+nvme_pcie_ctrlr_disconnect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)
+{
+}
+
+static int
+nvme_pcie_ctrlr_delete_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)
+{
+	struct nvme_completion_poll_status *status;
+	int rc;
+
+	assert(ctrlr != NULL);
+
+	if (ctrlr->is_removed) {
+		goto free;
+	}
+
+	status = calloc(1, sizeof(*status));
+	if (!status) {
+		SPDK_ERRLOG("Failed to allocate status tracker\n");
+		return -ENOMEM;
+	}
+
+	/* Delete the I/O submission queue */
+	rc = nvme_pcie_ctrlr_cmd_delete_io_sq(ctrlr, qpair, nvme_completion_poll_cb, status);
+	if (rc != 0) {
+		SPDK_ERRLOG("Failed to send request to delete_io_sq with rc=%d\n", rc);
+		free(status);
+		return rc;
+	}
+	if (nvme_wait_for_completion(ctrlr->adminq, status)) {
+		if (!status->timed_out) {
+			free(status);
+		}
+		return -1;
+	}
+
+	memset(status, 0, sizeof(*status));
+	/* Delete the completion queue */
+	rc = nvme_pcie_ctrlr_cmd_delete_io_cq(ctrlr, qpair, nvme_completion_poll_cb, status);
+	if (rc != 0) {
+		SPDK_ERRLOG("Failed to send request to delete_io_cq with rc=%d\n", rc);
+		free(status);
+		return rc;
+	}
+	if (nvme_wait_for_completion(ctrlr->adminq, status)) {
+		if (!status->timed_out) {
+			free(status);
+		}
+		return -1;
+	}
+	free(status);
+
+free:
+	if (qpair->no_deletion_notification_needed == 0) {
+		/* Abort the rest of the I/O */
+		nvme_pcie_qpair_abort_trackers(qpair, 1);
+	}
+
+	nvme_pcie_qpair_destroy(qpair);
+	return 0;
+}
+
+static void
+nvme_pcie_fail_request_bad_vtophys(struct spdk_nvme_qpair *qpair, struct nvme_tracker *tr)
+{
+	/*
+	 * Bad vtophys translation, so abort this request and return
+	 *  immediately.
+	 */
+	nvme_pcie_qpair_manual_complete_tracker(qpair, tr, SPDK_NVME_SCT_GENERIC,
+						SPDK_NVME_SC_INVALID_FIELD,
+						1 /* do not retry */, true);
+}
+
+/*
+ * Append PRP list entries to describe a virtually contiguous buffer starting at virt_addr of len bytes.
+ *
+ * *prp_index will be updated to account for the number of PRP entries used.
+ */
+static inline int
+nvme_pcie_prp_list_append(struct nvme_tracker *tr, uint32_t *prp_index, void *virt_addr, size_t len,
+			  uint32_t page_size)
+{
+	struct spdk_nvme_cmd *cmd = &tr->req->cmd;
+	uintptr_t page_mask = page_size - 1;
+	uint64_t phys_addr;
+	uint32_t i;
+
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "prp_index:%u virt_addr:%p len:%u\n",
+		      *prp_index, virt_addr, (uint32_t)len);
+
+	if (spdk_unlikely(((uintptr_t)virt_addr & 3) != 0)) {
+		SPDK_ERRLOG("virt_addr %p not dword aligned\n", virt_addr);
+		return -EFAULT;
+	}
+
+	i = *prp_index;
+	while (len) {
+		uint32_t seg_len;
+
+		/*
+		 * prp_index 0 is stored in prp1, and the rest are stored in the prp[] array,
+		 * so prp_index == count is valid.
+		 */
+		if (spdk_unlikely(i > SPDK_COUNTOF(tr->u.prp))) {
+			SPDK_ERRLOG("out of PRP entries\n");
+			return -EFAULT;
+		}
+
+		phys_addr = spdk_vtophys(virt_addr, NULL);
+		if (spdk_unlikely(phys_addr == SPDK_VTOPHYS_ERROR)) {
+			SPDK_ERRLOG("vtophys(%p) failed\n", virt_addr);
+			return -EFAULT;
+		}
+
+		if (i == 0) {
+			SPDK_DEBUGLOG(SPDK_LOG_NVME, "prp1 = %p\n", (void *)phys_addr);
+			cmd->dptr.prp.prp1 = phys_addr;
+			seg_len = page_size - ((uintptr_t)virt_addr & page_mask);
+		} else {
+			if ((phys_addr & page_mask) != 0) {
+				SPDK_ERRLOG("PRP %u not page aligned (%p)\n", i, virt_addr);
+				return -EFAULT;
+			}
+
+			SPDK_DEBUGLOG(SPDK_LOG_NVME, "prp[%u] = %p\n", i - 1, (void *)phys_addr);
+			tr->u.prp[i - 1] = phys_addr;
+			seg_len = page_size;
+		}
+
+		seg_len = spdk_min(seg_len, len);
+		virt_addr += seg_len;
+		len -= seg_len;
+		i++;
+	}
+
+	cmd->psdt = SPDK_NVME_PSDT_PRP;
+	if (i <= 1) {
+		cmd->dptr.prp.prp2 = 0;
+	} else if (i == 2) {
+		cmd->dptr.prp.prp2 = tr->u.prp[0];
+		SPDK_DEBUGLOG(SPDK_LOG_NVME, "prp2 = %p\n", (void *)cmd->dptr.prp.prp2);
+	} else {
+		cmd->dptr.prp.prp2 = tr->prp_sgl_bus_addr;
+		SPDK_DEBUGLOG(SPDK_LOG_NVME, "prp2 = %p (PRP list)\n", (void *)cmd->dptr.prp.prp2);
+	}
+
+	*prp_index = i;
+	return 0;
+}
+
+static int
+nvme_pcie_qpair_build_request_invalid(struct spdk_nvme_qpair *qpair,
+				      struct nvme_request *req, struct nvme_tracker *tr, bool dword_aligned)
+{
+	assert(0);
+	nvme_pcie_fail_request_bad_vtophys(qpair, tr);
+	return -EINVAL;
+}
+
+/**
+ * Build PRP list describing physically contiguous payload buffer.
+ */
+static int
+nvme_pcie_qpair_build_contig_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req,
+				     struct nvme_tracker *tr, bool dword_aligned)
+{
+	uint32_t prp_index = 0;
+	int rc;
+
+	rc = nvme_pcie_prp_list_append(tr, &prp_index, req->payload.contig_or_cb_arg + req->payload_offset,
+				       req->payload_size, qpair->ctrlr->page_size);
+	if (rc) {
+		nvme_pcie_fail_request_bad_vtophys(qpair, tr);
+	}
+
+	return rc;
+}
+
+/**
+ * Build an SGL describing a physically contiguous payload buffer.
+ *
+ * This is more efficient than using PRP because large buffers can be
+ * described this way.
+ */
+static int
+nvme_pcie_qpair_build_contig_hw_sgl_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req,
+		struct nvme_tracker *tr, bool dword_aligned)
+{
+	void *virt_addr;
+	uint64_t phys_addr, mapping_length;
+	uint32_t length;
+	struct spdk_nvme_sgl_descriptor *sgl;
+	uint32_t nseg = 0;
+
+	assert(req->payload_size != 0);
+	assert(nvme_payload_type(&req->payload) == NVME_PAYLOAD_TYPE_CONTIG);
+
+	sgl = tr->u.sgl;
+	req->cmd.psdt = SPDK_NVME_PSDT_SGL_MPTR_CONTIG;
+	req->cmd.dptr.sgl1.unkeyed.subtype = 0;
+
+	length = req->payload_size;
+	virt_addr = req->payload.contig_or_cb_arg + req->payload_offset;
+	mapping_length = length;
+
+	while (length > 0) {
+		if (nseg >= NVME_MAX_SGL_DESCRIPTORS) {
+			nvme_pcie_fail_request_bad_vtophys(qpair, tr);
+			return -EFAULT;
+		}
+
+		if (dword_aligned && ((uintptr_t)virt_addr & 3)) {
+			SPDK_ERRLOG("virt_addr %p not dword aligned\n", virt_addr);
+			nvme_pcie_fail_request_bad_vtophys(qpair, tr);
+			return -EFAULT;
+		}
+
+		phys_addr = spdk_vtophys(virt_addr, &mapping_length);
+		if (phys_addr == SPDK_VTOPHYS_ERROR) {
+			nvme_pcie_fail_request_bad_vtophys(qpair, tr);
+			return -EFAULT;
+		}
+
+		mapping_length = spdk_min(length, mapping_length);
+
+		length -= mapping_length;
+		virt_addr += mapping_length;
+
+		sgl->unkeyed.type = SPDK_NVME_SGL_TYPE_DATA_BLOCK;
+		sgl->unkeyed.length = mapping_length;
+		sgl->address = phys_addr;
+		sgl->unkeyed.subtype = 0;
+
+		sgl++;
+		nseg++;
+	}
+
+	if (nseg == 1) {
+		/*
+		 * The whole transfer can be described by a single SGL descriptor.
+		 *  Use the special case described by the spec where SGL1's type is Data Block.
+		 *  This means the SGL in the tracker is not used at all, so copy the first (and only)
+		 *  SGL element into SGL1.
+		 */
+		req->cmd.dptr.sgl1.unkeyed.type = SPDK_NVME_SGL_TYPE_DATA_BLOCK;
+		req->cmd.dptr.sgl1.address = tr->u.sgl[0].address;
+		req->cmd.dptr.sgl1.unkeyed.length = tr->u.sgl[0].unkeyed.length;
+	} else {
+		/* SPDK NVMe driver supports only 1 SGL segment for now, it is enough because
+		 *  NVME_MAX_SGL_DESCRIPTORS * 16 is less than one page.
+		 */
+		req->cmd.dptr.sgl1.unkeyed.type = SPDK_NVME_SGL_TYPE_LAST_SEGMENT;
+		req->cmd.dptr.sgl1.address = tr->prp_sgl_bus_addr;
+		req->cmd.dptr.sgl1.unkeyed.length = nseg * sizeof(struct spdk_nvme_sgl_descriptor);
+	}
+
+	return 0;
+}
+
+/**
+ * Build SGL list describing scattered payload buffer.
+ */
+static int
+nvme_pcie_qpair_build_hw_sgl_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req,
+				     struct nvme_tracker *tr, bool dword_aligned)
+{
+	int rc;
+	void *virt_addr;
+	uint64_t phys_addr;
+	uint32_t remaining_transfer_len, remaining_user_sge_len, length;
+	struct spdk_nvme_sgl_descriptor *sgl;
+	uint32_t nseg = 0;
+
+	/*
+	 * Build scattered payloads.
+	 */
+	assert(req->payload_size != 0);
+	assert(nvme_payload_type(&req->payload) == NVME_PAYLOAD_TYPE_SGL);
+	assert(req->payload.reset_sgl_fn != NULL);
+	assert(req->payload.next_sge_fn != NULL);
+	req->payload.reset_sgl_fn(req->payload.contig_or_cb_arg, req->payload_offset);
+
+	sgl = tr->u.sgl;
+	req->cmd.psdt = SPDK_NVME_PSDT_SGL_MPTR_CONTIG;
+	req->cmd.dptr.sgl1.unkeyed.subtype = 0;
+
+	remaining_transfer_len = req->payload_size;
+
+	while (remaining_transfer_len > 0) {
+		rc = req->payload.next_sge_fn(req->payload.contig_or_cb_arg,
+					      &virt_addr, &remaining_user_sge_len);
+		if (rc) {
+			nvme_pcie_fail_request_bad_vtophys(qpair, tr);
+			return -EFAULT;
+		}
+
+		/* Bit Bucket SGL descriptor */
+		if ((uint64_t)virt_addr == UINT64_MAX) {
+			/* TODO: enable WRITE and COMPARE when necessary */
+			if (req->cmd.opc != SPDK_NVME_OPC_READ) {
+				SPDK_ERRLOG("Only READ command can be supported\n");
+				goto exit;
+			}
+			if (nseg >= NVME_MAX_SGL_DESCRIPTORS) {
+				SPDK_ERRLOG("Too many SGL entries\n");
+				goto exit;
+			}
+
+			sgl->unkeyed.type = SPDK_NVME_SGL_TYPE_BIT_BUCKET;
+			/* If the SGL describes a destination data buffer, the length of data
+			 * buffer shall be discarded by controller, and the length is included
+			 * in Number of Logical Blocks (NLB) parameter. Otherwise, the length
+			 * is not included in the NLB parameter.
+			 */
+			remaining_user_sge_len = spdk_min(remaining_user_sge_len, remaining_transfer_len);
+			remaining_transfer_len -= remaining_user_sge_len;
+
+			sgl->unkeyed.length = remaining_user_sge_len;
+			sgl->address = 0;
+			sgl->unkeyed.subtype = 0;
+
+			sgl++;
+			nseg++;
+
+			continue;
+		}
+
+		remaining_user_sge_len = spdk_min(remaining_user_sge_len, remaining_transfer_len);
+		remaining_transfer_len -= remaining_user_sge_len;
+		while (remaining_user_sge_len > 0) {
+			if (nseg >= NVME_MAX_SGL_DESCRIPTORS) {
+				SPDK_ERRLOG("Too many SGL entries\n");
+				goto exit;
+			}
+
+			if (dword_aligned && ((uintptr_t)virt_addr & 3)) {
+				SPDK_ERRLOG("virt_addr %p not dword aligned\n", virt_addr);
+				goto exit;
+			}
+
+			phys_addr = spdk_vtophys(virt_addr, NULL);
+			if (phys_addr == SPDK_VTOPHYS_ERROR) {
+				goto exit;
+			}
+
+			length = spdk_min(remaining_user_sge_len, VALUE_2MB - _2MB_OFFSET(virt_addr));
+			remaining_user_sge_len -= length;
+			virt_addr += length;
+
+			if (nseg > 0 && phys_addr ==
+			    (*(sgl - 1)).address + (*(sgl - 1)).unkeyed.length) {
+				/* extend previous entry */
+				(*(sgl - 1)).unkeyed.length += length;
+				continue;
+			}
+
+			sgl->unkeyed.type = SPDK_NVME_SGL_TYPE_DATA_BLOCK;
+			sgl->unkeyed.length = length;
+			sgl->address = phys_addr;
+			sgl->unkeyed.subtype = 0;
+
+			sgl++;
+			nseg++;
+		}
+	}
+
+	if (nseg == 1) {
+		/*
+		 * The whole transfer can be described by a single SGL descriptor.
+		 *  Use the special case described by the spec where SGL1's type is Data Block.
+		 *  This means the SGL in the tracker is not used at all, so copy the first (and only)
+		 *  SGL element into SGL1.
+		 */
+		req->cmd.dptr.sgl1.unkeyed.type = SPDK_NVME_SGL_TYPE_DATA_BLOCK;
+		req->cmd.dptr.sgl1.address = tr->u.sgl[0].address;
+		req->cmd.dptr.sgl1.unkeyed.length = tr->u.sgl[0].unkeyed.length;
+	} else {
+		/* SPDK NVMe driver supports only 1 SGL segment for now, it is enough because
+		 *  NVME_MAX_SGL_DESCRIPTORS * 16 is less than one page.
+		 */
+		req->cmd.dptr.sgl1.unkeyed.type = SPDK_NVME_SGL_TYPE_LAST_SEGMENT;
+		req->cmd.dptr.sgl1.address = tr->prp_sgl_bus_addr;
+		req->cmd.dptr.sgl1.unkeyed.length = nseg * sizeof(struct spdk_nvme_sgl_descriptor);
+	}
+
+	return 0;
+
+exit:
+	nvme_pcie_fail_request_bad_vtophys(qpair, tr);
+	return -EFAULT;
+}
+
+/**
+ * Build PRP list describing scattered payload buffer.
+ */
+static int
+nvme_pcie_qpair_build_prps_sgl_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req,
+				       struct nvme_tracker *tr, bool dword_aligned)
+{
+	int rc;
+	void *virt_addr;
+	uint32_t remaining_transfer_len, length;
+	uint32_t prp_index = 0;
+	uint32_t page_size = qpair->ctrlr->page_size;
+
+	/*
+	 * Build scattered payloads.
+	 */
+	assert(nvme_payload_type(&req->payload) == NVME_PAYLOAD_TYPE_SGL);
+	assert(req->payload.reset_sgl_fn != NULL);
+	req->payload.reset_sgl_fn(req->payload.contig_or_cb_arg, req->payload_offset);
+
+	remaining_transfer_len = req->payload_size;
+	while (remaining_transfer_len > 0) {
+		assert(req->payload.next_sge_fn != NULL);
+		rc = req->payload.next_sge_fn(req->payload.contig_or_cb_arg, &virt_addr, &length);
+		if (rc) {
+			nvme_pcie_fail_request_bad_vtophys(qpair, tr);
+			return -EFAULT;
+		}
+
+		length = spdk_min(remaining_transfer_len, length);
+
+		/*
+		 * Any incompatible sges should have been handled up in the splitting routine,
+		 *  but assert here as an additional check.
+		 *
+		 * All SGEs except last must end on a page boundary.
+		 */
+		assert((length == remaining_transfer_len) ||
+		       _is_page_aligned((uintptr_t)virt_addr + length, page_size));
+
+		rc = nvme_pcie_prp_list_append(tr, &prp_index, virt_addr, length, page_size);
+		if (rc) {
+			nvme_pcie_fail_request_bad_vtophys(qpair, tr);
+			return rc;
+		}
+
+		remaining_transfer_len -= length;
+	}
+
+	return 0;
+}
+
+typedef int(*build_req_fn)(struct spdk_nvme_qpair *, struct nvme_request *, struct nvme_tracker *,
+			   bool);
+
+static build_req_fn const g_nvme_pcie_build_req_table[][2] = {
+	[NVME_PAYLOAD_TYPE_INVALID] = {
+		nvme_pcie_qpair_build_request_invalid,			/* PRP */
+		nvme_pcie_qpair_build_request_invalid			/* SGL */
+	},
+	[NVME_PAYLOAD_TYPE_CONTIG] = {
+		nvme_pcie_qpair_build_contig_request,			/* PRP */
+		nvme_pcie_qpair_build_contig_hw_sgl_request		/* SGL */
+	},
+	[NVME_PAYLOAD_TYPE_SGL] = {
+		nvme_pcie_qpair_build_prps_sgl_request,			/* PRP */
+		nvme_pcie_qpair_build_hw_sgl_request			/* SGL */
+	}
+};
+
+static int
+nvme_pcie_qpair_build_metadata(struct spdk_nvme_qpair *qpair, struct nvme_tracker *tr,
+			       bool sgl_supported, bool dword_aligned)
+{
+	void *md_payload;
+	struct nvme_request *req = tr->req;
+
+	if (req->payload.md) {
+		md_payload = req->payload.md + req->md_offset;
+		if (dword_aligned && ((uintptr_t)md_payload & 3)) {
+			SPDK_ERRLOG("virt_addr %p not dword aligned\n", md_payload);
+			goto exit;
+		}
+
+		if (sgl_supported && dword_aligned) {
+			assert(req->cmd.psdt == SPDK_NVME_PSDT_SGL_MPTR_CONTIG);
+			req->cmd.psdt = SPDK_NVME_PSDT_SGL_MPTR_SGL;
+			tr->meta_sgl.address = spdk_vtophys(md_payload, NULL);
+			if (tr->meta_sgl.address == SPDK_VTOPHYS_ERROR) {
+				goto exit;
+			}
+			tr->meta_sgl.unkeyed.type = SPDK_NVME_SGL_TYPE_DATA_BLOCK;
+			tr->meta_sgl.unkeyed.length = req->md_size;
+			tr->meta_sgl.unkeyed.subtype = 0;
+			req->cmd.mptr = tr->prp_sgl_bus_addr - sizeof(struct spdk_nvme_sgl_descriptor);
+		} else {
+			req->cmd.mptr = spdk_vtophys(md_payload, NULL);
+			if (req->cmd.mptr == SPDK_VTOPHYS_ERROR) {
+				goto exit;
+			}
+		}
+	}
+
+	return 0;
+
+exit:
+	nvme_pcie_fail_request_bad_vtophys(qpair, tr);
+	return -EINVAL;
+}
+
+static int
+nvme_pcie_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req)
+{
+	struct nvme_tracker	*tr;
+	int			rc = 0;
+	struct spdk_nvme_ctrlr	*ctrlr = qpair->ctrlr;
+	struct nvme_pcie_qpair	*pqpair = nvme_pcie_qpair(qpair);
+	enum nvme_payload_type	payload_type;
+	bool			sgl_supported;
+	bool			dword_aligned = true;
+
+	if (spdk_unlikely(nvme_qpair_is_admin_queue(qpair))) {
+		nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+	}
+
+	tr = TAILQ_FIRST(&pqpair->free_tr);
+
+	if (tr == NULL) {
+		/* Inform the upper layer to try again later. */
+		rc = -EAGAIN;
+		goto exit;
+	}
+
+	TAILQ_REMOVE(&pqpair->free_tr, tr, tq_list); /* remove tr from free_tr */
+	TAILQ_INSERT_TAIL(&pqpair->outstanding_tr, tr, tq_list);
+	tr->req = req;
+	tr->cb_fn = req->cb_fn;
+	tr->cb_arg = req->cb_arg;
+	req->cmd.cid = tr->cid;
+
+	if (req->payload_size != 0) {
+		payload_type = nvme_payload_type(&req->payload);
+		/* According to the specification, PRPs shall be used for all
+		 *  Admin commands for NVMe over PCIe implementations.
+		 */
+		sgl_supported = (ctrlr->flags & SPDK_NVME_CTRLR_SGL_SUPPORTED) != 0 &&
+				!nvme_qpair_is_admin_queue(qpair);
+
+		if (sgl_supported && !(ctrlr->flags & SPDK_NVME_CTRLR_SGL_REQUIRES_DWORD_ALIGNMENT)) {
+			dword_aligned = false;
+		}
+		rc = g_nvme_pcie_build_req_table[payload_type][sgl_supported](qpair, req, tr, dword_aligned);
+		if (rc < 0) {
+			goto exit;
+		}
+
+		rc = nvme_pcie_qpair_build_metadata(qpair, tr, sgl_supported, dword_aligned);
+		if (rc < 0) {
+			goto exit;
+		}
+	}
+
+	nvme_pcie_qpair_submit_tracker(qpair, tr);
+
+exit:
+	if (spdk_unlikely(nvme_qpair_is_admin_queue(qpair))) {
+		nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+	}
+
+	return rc;
+}
+
+static void
+nvme_pcie_qpair_check_timeout(struct spdk_nvme_qpair *qpair)
+{
+	uint64_t t02;
+	struct nvme_tracker *tr, *tmp;
+	struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
+	struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
+	struct spdk_nvme_ctrlr_process *active_proc;
+
+	/* Don't check timeouts during controller initialization. */
+	if (ctrlr->state != NVME_CTRLR_STATE_READY) {
+		return;
+	}
+
+	if (nvme_qpair_is_admin_queue(qpair)) {
+		active_proc = nvme_ctrlr_get_current_process(ctrlr);
+	} else {
+		active_proc = qpair->active_proc;
+	}
+
+	/* Only check timeouts if the current process has a timeout callback. */
+	if (active_proc == NULL || active_proc->timeout_cb_fn == NULL) {
+		return;
+	}
+
+	t02 = spdk_get_ticks();
+	TAILQ_FOREACH_SAFE(tr, &pqpair->outstanding_tr, tq_list, tmp) {
+		assert(tr->req != NULL);
+
+		if (nvme_request_check_timeout(tr->req, tr->cid, active_proc, t02)) {
+			/*
+			 * The requests are in order, so as soon as one has not timed out,
+			 * stop iterating.
+			 */
+			break;
+		}
+	}
+}
+
+static int32_t
+nvme_pcie_qpair_process_completions(struct spdk_nvme_qpair *qpair, uint32_t max_completions)
+{
+	struct nvme_pcie_qpair	*pqpair = nvme_pcie_qpair(qpair);
+	struct nvme_tracker	*tr;
+	struct spdk_nvme_cpl	*cpl, *next_cpl;
+	uint32_t		 num_completions = 0;
+	struct spdk_nvme_ctrlr	*ctrlr = qpair->ctrlr;
+	uint16_t		 next_cq_head;
+	uint8_t			 next_phase;
+	bool			 next_is_valid = false;
+
+	if (spdk_unlikely(nvme_qpair_is_admin_queue(qpair))) {
+		nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+	}
+
+	if (max_completions == 0 || max_completions > pqpair->max_completions_cap) {
+		/*
+		 * max_completions == 0 means unlimited, but complete at most
+		 * max_completions_cap batch of I/O at a time so that the completion
+		 * queue doorbells don't wrap around.
+		 */
+		max_completions = pqpair->max_completions_cap;
+	}
+
+	while (1) {
+		cpl = &pqpair->cpl[pqpair->cq_head];
+
+		if (!next_is_valid && cpl->status.p != pqpair->flags.phase) {
+			break;
+		}
+
+		if (spdk_likely(pqpair->cq_head + 1 != pqpair->num_entries)) {
+			next_cq_head = pqpair->cq_head + 1;
+			next_phase = pqpair->flags.phase;
+		} else {
+			next_cq_head = 0;
+			next_phase = !pqpair->flags.phase;
+		}
+		next_cpl = &pqpair->cpl[next_cq_head];
+		next_is_valid = (next_cpl->status.p == next_phase);
+		if (next_is_valid) {
+			__builtin_prefetch(&pqpair->tr[next_cpl->cid]);
+		}
+
+#ifdef __PPC64__
+		/*
+		 * This memory barrier prevents reordering of:
+		 * - load after store from/to tr
+		 * - load after load cpl phase and cpl cid
+		 */
+		spdk_mb();
+#elif defined(__aarch64__)
+		__asm volatile("dmb oshld" ::: "memory");
+#endif
+
+		if (spdk_unlikely(++pqpair->cq_head == pqpair->num_entries)) {
+			pqpair->cq_head = 0;
+			pqpair->flags.phase = !pqpair->flags.phase;
+		}
+
+		tr = &pqpair->tr[cpl->cid];
+		/* Prefetch the req's STAILQ_ENTRY since we'll need to access it
+		 * as part of putting the req back on the qpair's free list.
+		 */
+		__builtin_prefetch(&tr->req->stailq);
+		pqpair->sq_head = cpl->sqhd;
+
+		if (tr->req) {
+			nvme_pcie_qpair_complete_tracker(qpair, tr, cpl, true);
+		} else {
+			SPDK_ERRLOG("cpl does not map to outstanding cmd\n");
+			spdk_nvme_qpair_print_completion(qpair, cpl);
+			assert(0);
+		}
+
+		if (++num_completions == max_completions) {
+			break;
+		}
+	}
+
+	if (num_completions > 0) {
+		nvme_pcie_qpair_ring_cq_doorbell(qpair);
+	}
+
+	if (pqpair->flags.delay_cmd_submit) {
+		if (pqpair->last_sq_tail != pqpair->sq_tail) {
+			nvme_pcie_qpair_ring_sq_doorbell(qpair);
+			pqpair->last_sq_tail = pqpair->sq_tail;
+		}
+	}
+
+	if (spdk_unlikely(ctrlr->timeout_enabled)) {
+		/*
+		 * User registered for timeout callback
+		 */
+		nvme_pcie_qpair_check_timeout(qpair);
+	}
+
+	/* Before returning, complete any pending admin request. */
+	if (spdk_unlikely(nvme_qpair_is_admin_queue(qpair))) {
+		nvme_pcie_qpair_complete_pending_admin_request(qpair);
+
+		nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+	}
+
+	return num_completions;
+}
+
+static struct spdk_nvme_transport_poll_group *
+nvme_pcie_poll_group_create(void)
+{
+	struct nvme_pcie_poll_group *group = calloc(1, sizeof(*group));
+
+	if (group == NULL) {
+		SPDK_ERRLOG("Unable to allocate poll group.\n");
+		return NULL;
+	}
+
+	return &group->group;
+}
+
+static int
+nvme_pcie_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair)
+{
+	return 0;
+}
+
+static int
+nvme_pcie_poll_group_disconnect_qpair(struct spdk_nvme_qpair *qpair)
+{
+	return 0;
+}
+
+static int
+nvme_pcie_poll_group_add(struct spdk_nvme_transport_poll_group *tgroup,
+			 struct spdk_nvme_qpair *qpair)
+{
+	return 0;
+}
+
+static int
+nvme_pcie_poll_group_remove(struct spdk_nvme_transport_poll_group *tgroup,
+			    struct spdk_nvme_qpair *qpair)
+{
+	return 0;
+}
+
+static int64_t
+nvme_pcie_poll_group_process_completions(struct spdk_nvme_transport_poll_group *tgroup,
+		uint32_t completions_per_qpair, spdk_nvme_disconnected_qpair_cb disconnected_qpair_cb)
+{
+	struct spdk_nvme_qpair *qpair, *tmp_qpair;
+	int32_t local_completions = 0;
+	int64_t total_completions = 0;
+
+	STAILQ_FOREACH_SAFE(qpair, &tgroup->disconnected_qpairs, poll_group_stailq, tmp_qpair) {
+		disconnected_qpair_cb(qpair, tgroup->group->ctx);
+	}
+
+	STAILQ_FOREACH_SAFE(qpair, &tgroup->connected_qpairs, poll_group_stailq, tmp_qpair) {
+		local_completions = spdk_nvme_qpair_process_completions(qpair, completions_per_qpair);
+		if (local_completions < 0) {
+			disconnected_qpair_cb(qpair, tgroup->group->ctx);
+			local_completions = 0;
+		}
+		total_completions += local_completions;
+	}
+
+	return total_completions;
+}
+
+static int
+nvme_pcie_poll_group_destroy(struct spdk_nvme_transport_poll_group *tgroup)
+{
+	if (!STAILQ_EMPTY(&tgroup->connected_qpairs) || !STAILQ_EMPTY(&tgroup->disconnected_qpairs)) {
+		return -EBUSY;
+	}
+
+	free(tgroup);
+
+	return 0;
+}
+
+static struct spdk_pci_id nvme_pci_driver_id[] = {
+	{
+		.class_id = SPDK_PCI_CLASS_NVME,
+		.vendor_id = SPDK_PCI_ANY_ID,
+		.device_id = SPDK_PCI_ANY_ID,
+		.subvendor_id = SPDK_PCI_ANY_ID,
+		.subdevice_id = SPDK_PCI_ANY_ID,
+	},
+	{ .vendor_id = 0, /* sentinel */ },
+};
+
+SPDK_PCI_DRIVER_REGISTER("nvme", nvme_pci_driver_id,
+			 SPDK_PCI_DRIVER_NEED_MAPPING | SPDK_PCI_DRIVER_WC_ACTIVATE);
+
+const struct spdk_nvme_transport_ops pcie_ops = {
+	.name = "PCIE",
+	.type = SPDK_NVME_TRANSPORT_PCIE,
+	.ctrlr_construct = nvme_pcie_ctrlr_construct,
+	.ctrlr_scan = nvme_pcie_ctrlr_scan,
+	.ctrlr_destruct = nvme_pcie_ctrlr_destruct,
+	.ctrlr_enable = nvme_pcie_ctrlr_enable,
+
+	.ctrlr_set_reg_4 = nvme_pcie_ctrlr_set_reg_4,
+	.ctrlr_set_reg_8 = nvme_pcie_ctrlr_set_reg_8,
+	.ctrlr_get_reg_4 = nvme_pcie_ctrlr_get_reg_4,
+	.ctrlr_get_reg_8 = nvme_pcie_ctrlr_get_reg_8,
+
+	.ctrlr_get_max_xfer_size = nvme_pcie_ctrlr_get_max_xfer_size,
+	.ctrlr_get_max_sges = nvme_pcie_ctrlr_get_max_sges,
+
+	.ctrlr_reserve_cmb = nvme_pcie_ctrlr_reserve_cmb,
+	.ctrlr_map_cmb = nvme_pcie_ctrlr_map_io_cmb,
+	.ctrlr_unmap_cmb = nvme_pcie_ctrlr_unmap_io_cmb,
+
+	.ctrlr_create_io_qpair = nvme_pcie_ctrlr_create_io_qpair,
+	.ctrlr_delete_io_qpair = nvme_pcie_ctrlr_delete_io_qpair,
+	.ctrlr_connect_qpair = nvme_pcie_ctrlr_connect_qpair,
+	.ctrlr_disconnect_qpair = nvme_pcie_ctrlr_disconnect_qpair,
+
+	.qpair_abort_reqs = nvme_pcie_qpair_abort_reqs,
+	.qpair_reset = nvme_pcie_qpair_reset,
+	.qpair_submit_request = nvme_pcie_qpair_submit_request,
+	.qpair_process_completions = nvme_pcie_qpair_process_completions,
+	.qpair_iterate_requests = nvme_pcie_qpair_iterate_requests,
+	.admin_qpair_abort_aers = nvme_pcie_admin_qpair_abort_aers,
+
+	.poll_group_create = nvme_pcie_poll_group_create,
+	.poll_group_connect_qpair = nvme_pcie_poll_group_connect_qpair,
+	.poll_group_disconnect_qpair = nvme_pcie_poll_group_disconnect_qpair,
+	.poll_group_add = nvme_pcie_poll_group_add,
+	.poll_group_remove = nvme_pcie_poll_group_remove,
+	.poll_group_process_completions = nvme_pcie_poll_group_process_completions,
+	.poll_group_destroy = nvme_pcie_poll_group_destroy,
+};
+
+SPDK_NVME_TRANSPORT_REGISTER(pcie, &pcie_ops);
diff --git a/src/spdk/lib/nvme/nvme_poll_group.c b/src/spdk/lib/nvme/nvme_poll_group.c
new file mode 100644
index 000000000..291f55e63
--- /dev/null
+++ b/src/spdk/lib/nvme/nvme_poll_group.c
@@ -0,0 +1,164 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#include "nvme_internal.h"
+
+struct spdk_nvme_poll_group *
+spdk_nvme_poll_group_create(void *ctx)
+{
+	struct spdk_nvme_poll_group *group;
+
+	group = calloc(1, sizeof(*group));
+	if (group == NULL) {
+		return NULL;
+	}
+
+	group->ctx = ctx;
+	STAILQ_INIT(&group->tgroups);
+
+	return group;
+}
+
+int
+spdk_nvme_poll_group_add(struct spdk_nvme_poll_group *group, struct spdk_nvme_qpair *qpair)
+{
+	struct spdk_nvme_transport_poll_group *tgroup;
+	const struct spdk_nvme_transport *transport;
+
+	if (nvme_qpair_get_state(qpair) != NVME_QPAIR_DISCONNECTED) {
+		return -EINVAL;
+	}
+
+	STAILQ_FOREACH(tgroup, &group->tgroups, link) {
+		if (tgroup->transport == qpair->transport) {
+			break;
+		}
+	}
+
+	/* See if a new transport has been added (dlopen style) and we need to update the poll group */
+	if (!tgroup) {
+		transport = nvme_get_first_transport();
+		while (transport != NULL) {
+			if (transport == qpair->transport) {
+				tgroup = nvme_transport_poll_group_create(transport);
+				if (tgroup == NULL) {
+					return -ENOMEM;
+				}
+				tgroup->group = group;
+				STAILQ_INSERT_TAIL(&group->tgroups, tgroup, link);
+				break;
+			}
+			transport = nvme_get_next_transport(transport);
+		}
+	}
+
+	return tgroup ? nvme_transport_poll_group_add(tgroup, qpair) : -ENODEV;
+}
+
+int
+spdk_nvme_poll_group_remove(struct spdk_nvme_poll_group *group, struct spdk_nvme_qpair *qpair)
+{
+	struct spdk_nvme_transport_poll_group *tgroup;
+
+	STAILQ_FOREACH(tgroup, &group->tgroups, link) {
+		if (tgroup->transport == qpair->transport) {
+			return nvme_transport_poll_group_remove(tgroup, qpair);
+		}
+	}
+
+	return -ENODEV;
+}
+
+int
+nvme_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair)
+{
+	return nvme_transport_poll_group_connect_qpair(qpair);
+}
+
+int
+nvme_poll_group_disconnect_qpair(struct spdk_nvme_qpair *qpair)
+{
+	return nvme_transport_poll_group_disconnect_qpair(qpair);
+}
+
+int64_t
+spdk_nvme_poll_group_process_completions(struct spdk_nvme_poll_group *group,
+		uint32_t completions_per_qpair, spdk_nvme_disconnected_qpair_cb disconnected_qpair_cb)
+{
+	struct spdk_nvme_transport_poll_group *tgroup;
+	int64_t local_completions = 0, error_reason = 0, num_completions = 0;
+
+	if (disconnected_qpair_cb == NULL) {
+		return -EINVAL;
+	}
+
+	STAILQ_FOREACH(tgroup, &group->tgroups, link) {
+		local_completions = nvme_transport_poll_group_process_completions(tgroup, completions_per_qpair,
+				    disconnected_qpair_cb);
+		if (local_completions < 0 && error_reason == 0) {
+			error_reason = local_completions;
+		} else {
+			num_completions += local_completions;
+			/* Just to be safe */
+			assert(num_completions >= 0);
+		}
+	}
+
+	return error_reason ? error_reason : num_completions;
+}
+
+void *
+spdk_nvme_poll_group_get_ctx(struct spdk_nvme_poll_group *group)
+{
+	return group->ctx;
+}
+
+int
+spdk_nvme_poll_group_destroy(struct spdk_nvme_poll_group *group)
+{
+	struct spdk_nvme_transport_poll_group *tgroup, *tmp_tgroup;
+
+	STAILQ_FOREACH_SAFE(tgroup, &group->tgroups, link, tmp_tgroup) {
+		STAILQ_REMOVE(&group->tgroups, tgroup, spdk_nvme_transport_poll_group, link);
+		if (nvme_transport_poll_group_destroy(tgroup) != 0) {
+			STAILQ_INSERT_TAIL(&group->tgroups, tgroup, link);
+			return -EBUSY;
+		}
+
+	}
+
+	free(group);
+
+	return 0;
+}
diff --git a/src/spdk/lib/nvme/nvme_qpair.c b/src/spdk/lib/nvme/nvme_qpair.c
new file mode 100644
index 000000000..a3fdc2169
--- /dev/null
+++ b/src/spdk/lib/nvme/nvme_qpair.c
@@ -0,0 +1,1064 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "nvme_internal.h"
+#include "spdk/nvme_ocssd.h"
+
+#define NVME_CMD_DPTR_STR_SIZE 256
+
+static int nvme_qpair_resubmit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req);
+
+struct nvme_string {
+	uint16_t	value;
+	const char	*str;
+};
+
+static const struct nvme_string admin_opcode[] = {
+	{ SPDK_NVME_OPC_DELETE_IO_SQ, "DELETE IO SQ" },
+	{ SPDK_NVME_OPC_CREATE_IO_SQ, "CREATE IO SQ" },
+	{ SPDK_NVME_OPC_GET_LOG_PAGE, "GET LOG PAGE" },
+	{ SPDK_NVME_OPC_DELETE_IO_CQ, "DELETE IO CQ" },
+	{ SPDK_NVME_OPC_CREATE_IO_CQ, "CREATE IO CQ" },
+	{ SPDK_NVME_OPC_IDENTIFY, "IDENTIFY" },
+	{ SPDK_NVME_OPC_ABORT, "ABORT" },
+	{ SPDK_NVME_OPC_SET_FEATURES, "SET FEATURES" },
+	{ SPDK_NVME_OPC_GET_FEATURES, "GET FEATURES" },
+	{ SPDK_NVME_OPC_ASYNC_EVENT_REQUEST, "ASYNC EVENT REQUEST" },
+	{ SPDK_NVME_OPC_NS_MANAGEMENT, "NAMESPACE MANAGEMENT" },
+	{ SPDK_NVME_OPC_FIRMWARE_COMMIT, "FIRMWARE COMMIT" },
+	{ SPDK_NVME_OPC_FIRMWARE_IMAGE_DOWNLOAD, "FIRMWARE IMAGE DOWNLOAD" },
+	{ SPDK_NVME_OPC_DEVICE_SELF_TEST, "DEVICE SELF-TEST" },
+	{ SPDK_NVME_OPC_NS_ATTACHMENT, "NAMESPACE ATTACHMENT" },
+	{ SPDK_NVME_OPC_KEEP_ALIVE, "KEEP ALIVE" },
+	{ SPDK_NVME_OPC_DIRECTIVE_SEND, "DIRECTIVE SEND" },
+	{ SPDK_NVME_OPC_DIRECTIVE_RECEIVE, "DIRECTIVE RECEIVE" },
+	{ SPDK_NVME_OPC_VIRTUALIZATION_MANAGEMENT, "VIRTUALIZATION MANAGEMENT" },
+	{ SPDK_NVME_OPC_NVME_MI_SEND, "NVME-MI SEND" },
+	{ SPDK_NVME_OPC_NVME_MI_RECEIVE, "NVME-MI RECEIVE" },
+	{ SPDK_NVME_OPC_DOORBELL_BUFFER_CONFIG, "DOORBELL BUFFER CONFIG" },
+	{ SPDK_NVME_OPC_FABRIC, "FABRIC" },
+	{ SPDK_NVME_OPC_FORMAT_NVM, "FORMAT NVM" },
+	{ SPDK_NVME_OPC_SECURITY_SEND, "SECURITY SEND" },
+	{ SPDK_NVME_OPC_SECURITY_RECEIVE, "SECURITY RECEIVE" },
+	{ SPDK_NVME_OPC_SANITIZE, "SANITIZE" },
+	{ SPDK_NVME_OPC_GET_LBA_STATUS, "GET LBA STATUS" },
+	{ SPDK_OCSSD_OPC_GEOMETRY, "OCSSD / GEOMETRY" },
+	{ 0xFFFF, "ADMIN COMMAND" }
+};
+
+static const struct nvme_string fabric_opcode[] = {
+	{ SPDK_NVMF_FABRIC_COMMAND_PROPERTY_SET, "PROPERTY SET" },
+	{ SPDK_NVMF_FABRIC_COMMAND_CONNECT, "CONNECT" },
+	{ SPDK_NVMF_FABRIC_COMMAND_PROPERTY_GET, "PROPERTY GET" },
+	{ SPDK_NVMF_FABRIC_COMMAND_AUTHENTICATION_SEND, "AUTHENTICATION SEND" },
+	{ SPDK_NVMF_FABRIC_COMMAND_AUTHENTICATION_RECV, "AUTHENTICATION RECV" },
+	{ 0xFFFF, "RESERVED / VENDOR SPECIFIC" }
+};
+
+static const struct nvme_string feat_opcode[] = {
+	{ SPDK_NVME_FEAT_ARBITRATION, "ARBITRATION" },
+	{ SPDK_NVME_FEAT_POWER_MANAGEMENT, "POWER MANAGEMENT" },
+	{ SPDK_NVME_FEAT_LBA_RANGE_TYPE, "LBA RANGE TYPE" },
+	{ SPDK_NVME_FEAT_TEMPERATURE_THRESHOLD, "TEMPERATURE THRESHOLD" },
+	{ SPDK_NVME_FEAT_ERROR_RECOVERY, "ERROR_RECOVERY" },
+	{ SPDK_NVME_FEAT_VOLATILE_WRITE_CACHE, "VOLATILE WRITE CACHE" },
+	{ SPDK_NVME_FEAT_NUMBER_OF_QUEUES, "NUMBER OF QUEUES" },
+	{ SPDK_NVME_FEAT_INTERRUPT_COALESCING, "INTERRUPT COALESCING" },
+	{ SPDK_NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION, "INTERRUPT VECTOR CONFIGURATION" },
+	{ SPDK_NVME_FEAT_WRITE_ATOMICITY, "WRITE ATOMICITY" },
+	{ SPDK_NVME_FEAT_ASYNC_EVENT_CONFIGURATION, "ASYNC EVENT CONFIGURATION" },
+	{ SPDK_NVME_FEAT_AUTONOMOUS_POWER_STATE_TRANSITION, "AUTONOMOUS POWER STATE TRANSITION" },
+	{ SPDK_NVME_FEAT_HOST_MEM_BUFFER, "HOST MEM BUFFER" },
+	{ SPDK_NVME_FEAT_TIMESTAMP, "TIMESTAMP" },
+	{ SPDK_NVME_FEAT_KEEP_ALIVE_TIMER, "KEEP ALIVE TIMER" },
+	{ SPDK_NVME_FEAT_HOST_CONTROLLED_THERMAL_MANAGEMENT, "HOST CONTROLLED THERMAL MANAGEMENT" },
+	{ SPDK_NVME_FEAT_NON_OPERATIONAL_POWER_STATE_CONFIG, "NON OPERATIONAL POWER STATE CONFIG" },
+	{ SPDK_NVME_FEAT_SOFTWARE_PROGRESS_MARKER, "SOFTWARE PROGRESS MARKER" },
+	{ SPDK_NVME_FEAT_HOST_IDENTIFIER, "HOST IDENTIFIER" },
+	{ SPDK_NVME_FEAT_HOST_RESERVE_MASK, "HOST RESERVE MASK" },
+	{ SPDK_NVME_FEAT_HOST_RESERVE_PERSIST, "HOST RESERVE PERSIST" },
+	{ 0xFFFF, "RESERVED" }
+};
+
+static const struct nvme_string io_opcode[] = {
+	{ SPDK_NVME_OPC_FLUSH, "FLUSH" },
+	{ SPDK_NVME_OPC_WRITE, "WRITE" },
+	{ SPDK_NVME_OPC_READ, "READ" },
+	{ SPDK_NVME_OPC_WRITE_UNCORRECTABLE, "WRITE UNCORRECTABLE" },
+	{ SPDK_NVME_OPC_COMPARE, "COMPARE" },
+	{ SPDK_NVME_OPC_WRITE_ZEROES, "WRITE ZEROES" },
+	{ SPDK_NVME_OPC_DATASET_MANAGEMENT, "DATASET MANAGEMENT" },
+	{ SPDK_NVME_OPC_RESERVATION_REGISTER, "RESERVATION REGISTER" },
+	{ SPDK_NVME_OPC_RESERVATION_REPORT, "RESERVATION REPORT" },
+	{ SPDK_NVME_OPC_RESERVATION_ACQUIRE, "RESERVATION ACQUIRE" },
+	{ SPDK_NVME_OPC_RESERVATION_RELEASE, "RESERVATION RELEASE" },
+	{ SPDK_OCSSD_OPC_VECTOR_RESET, "OCSSD / VECTOR RESET" },
+	{ SPDK_OCSSD_OPC_VECTOR_WRITE, "OCSSD / VECTOR WRITE" },
+	{ SPDK_OCSSD_OPC_VECTOR_READ, "OCSSD / VECTOR READ" },
+	{ SPDK_OCSSD_OPC_VECTOR_COPY, "OCSSD / VECTOR COPY" },
+	{ 0xFFFF, "IO COMMAND" }
+};
+
+static const struct nvme_string sgl_type[] = {
+	{ SPDK_NVME_SGL_TYPE_DATA_BLOCK, "DATA BLOCK" },
+	{ SPDK_NVME_SGL_TYPE_BIT_BUCKET, "BIT BUCKET" },
+	{ SPDK_NVME_SGL_TYPE_SEGMENT, "SEGMENT" },
+	{ SPDK_NVME_SGL_TYPE_LAST_SEGMENT, "LAST SEGMENT" },
+	{ SPDK_NVME_SGL_TYPE_TRANSPORT_DATA_BLOCK, "TRANSPORT DATA BLOCK" },
+	{ SPDK_NVME_SGL_TYPE_VENDOR_SPECIFIC, "VENDOR SPECIFIC" },
+	{ 0xFFFF, "RESERVED" }
+};
+
+static const struct nvme_string sgl_subtype[] = {
+	{ SPDK_NVME_SGL_SUBTYPE_ADDRESS, "ADDRESS" },
+	{ SPDK_NVME_SGL_SUBTYPE_OFFSET, "OFFSET" },
+	{ SPDK_NVME_SGL_SUBTYPE_TRANSPORT, "TRANSPORT" },
+	{ SPDK_NVME_SGL_SUBTYPE_INVALIDATE_KEY, "INVALIDATE KEY" },
+	{ 0xFFFF, "RESERVED" }
+};
+
+static const char *
+nvme_get_string(const struct nvme_string *strings, uint16_t value)
+{
+	const struct nvme_string *entry;
+
+	entry = strings;
+
+	while (entry->value != 0xFFFF) {
+		if (entry->value == value) {
+			return entry->str;
+		}
+		entry++;
+	}
+	return entry->str;
+}
+
+static void
+nvme_get_sgl_unkeyed(char *buf, size_t size, struct spdk_nvme_cmd *cmd)
+{
+	struct spdk_nvme_sgl_descriptor *sgl = &cmd->dptr.sgl1;
+
+	snprintf(buf, size, " len:0x%x", sgl->unkeyed.length);
+}
+
+static void
+nvme_get_sgl_keyed(char *buf, size_t size, struct spdk_nvme_cmd *cmd)
+{
+	struct spdk_nvme_sgl_descriptor *sgl = &cmd->dptr.sgl1;
+
+	snprintf(buf, size, " len:0x%x key:0x%x", sgl->keyed.length, sgl->keyed.key);
+}
+
+static void
+nvme_get_sgl(char *buf, size_t size, struct spdk_nvme_cmd *cmd)
+{
+	struct spdk_nvme_sgl_descriptor *sgl = &cmd->dptr.sgl1;
+	int c;
+
+	c = snprintf(buf, size, "SGL %s %s 0x%" PRIx64, nvme_get_string(sgl_type, sgl->generic.type),
+		     nvme_get_string(sgl_subtype, sgl->generic.subtype), sgl->address);
+	assert(c >= 0 && (size_t)c < size);
+
+	if (sgl->generic.type == SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK) {
+		nvme_get_sgl_unkeyed(buf + c, size - c, cmd);
+	}
+
+	if (sgl->generic.type == SPDK_NVME_SGL_TYPE_DATA_BLOCK) {
+		nvme_get_sgl_keyed(buf + c, size - c, cmd);
+	}
+}
+
+static void
+nvme_get_prp(char *buf, size_t size, struct spdk_nvme_cmd *cmd)
+{
+	snprintf(buf, size, "PRP1 0x%" PRIx64 " PRP2 0x%" PRIx64, cmd->dptr.prp.prp1, cmd->dptr.prp.prp2);
+}
+
+static void
+nvme_get_dptr(char *buf, size_t size, struct spdk_nvme_cmd *cmd)
+{
+	if (spdk_nvme_opc_get_data_transfer(cmd->opc) != SPDK_NVME_DATA_NONE) {
+		switch (cmd->psdt) {
+		case SPDK_NVME_PSDT_PRP:
+			nvme_get_prp(buf, size, cmd);
+			break;
+		case SPDK_NVME_PSDT_SGL_MPTR_CONTIG:
+		case SPDK_NVME_PSDT_SGL_MPTR_SGL:
+			nvme_get_sgl(buf, size, cmd);
+			break;
+		default:
+			;
+		}
+	}
+}
+
+static void
+nvme_admin_qpair_print_command(uint16_t qid, struct spdk_nvme_cmd *cmd)
+{
+	struct spdk_nvmf_capsule_cmd *fcmd = (void *)cmd;
+	char dptr[NVME_CMD_DPTR_STR_SIZE] = {'\0'};
+
+	assert(cmd != NULL);
+
+	nvme_get_dptr(dptr, sizeof(dptr), cmd);
+
+	switch ((int)cmd->opc) {
+	case SPDK_NVME_OPC_SET_FEATURES:
+	case SPDK_NVME_OPC_GET_FEATURES:
+		SPDK_NOTICELOG("%s %s cid:%d cdw10:%08x %s\n",
+			       nvme_get_string(admin_opcode, cmd->opc), nvme_get_string(feat_opcode,
+					       cmd->cdw10_bits.set_features.fid), cmd->cid, cmd->cdw10, dptr);
+		break;
+	case SPDK_NVME_OPC_FABRIC:
+		SPDK_NOTICELOG("%s %s qid:%d cid:%d %s\n",
+			       nvme_get_string(admin_opcode, cmd->opc), nvme_get_string(fabric_opcode, fcmd->fctype), qid,
+			       fcmd->cid, dptr);
+		break;
+	default:
+		SPDK_NOTICELOG("%s (%02x) qid:%d cid:%d nsid:%x cdw10:%08x cdw11:%08x %s\n",
+			       nvme_get_string(admin_opcode, cmd->opc), cmd->opc, qid, cmd->cid, cmd->nsid, cmd->cdw10,
+			       cmd->cdw11, dptr);
+	}
+}
+
+static void
+nvme_io_qpair_print_command(uint16_t qid, struct spdk_nvme_cmd *cmd)
+{
+	char dptr[NVME_CMD_DPTR_STR_SIZE] = {'\0'};
+
+	assert(cmd != NULL);
+
+	nvme_get_dptr(dptr, sizeof(dptr), cmd);
+
+	switch ((int)cmd->opc) {
+	case SPDK_NVME_OPC_WRITE:
+	case SPDK_NVME_OPC_READ:
+	case SPDK_NVME_OPC_WRITE_UNCORRECTABLE:
+	case SPDK_NVME_OPC_COMPARE:
+		SPDK_NOTICELOG("%s sqid:%d cid:%d nsid:%d "
+			       "lba:%llu len:%d %s\n",
+			       nvme_get_string(io_opcode, cmd->opc), qid, cmd->cid, cmd->nsid,
+			       ((unsigned long long)cmd->cdw11 << 32) + cmd->cdw10,
+			       (cmd->cdw12 & 0xFFFF) + 1, dptr);
+		break;
+	case SPDK_NVME_OPC_FLUSH:
+	case SPDK_NVME_OPC_DATASET_MANAGEMENT:
+		SPDK_NOTICELOG("%s sqid:%d cid:%d nsid:%d\n",
+			       nvme_get_string(io_opcode, cmd->opc), qid, cmd->cid, cmd->nsid);
+		break;
+	default:
+		SPDK_NOTICELOG("%s (%02x) sqid:%d cid:%d nsid:%d\n",
+			       nvme_get_string(io_opcode, cmd->opc), cmd->opc, qid, cmd->cid, cmd->nsid);
+		break;
+	}
+}
+
+void
+spdk_nvme_print_command(uint16_t qid, struct spdk_nvme_cmd *cmd)
+{
+	assert(cmd != NULL);
+
+	if (qid == 0 || cmd->opc == SPDK_NVME_OPC_FABRIC) {
+		nvme_admin_qpair_print_command(qid, cmd);
+	} else {
+		nvme_io_qpair_print_command(qid, cmd);
+	}
+}
+
+void
+spdk_nvme_qpair_print_command(struct spdk_nvme_qpair *qpair, struct spdk_nvme_cmd *cmd)
+{
+	assert(qpair != NULL);
+	assert(cmd != NULL);
+
+	spdk_nvme_print_command(qpair->id, cmd);
+}
+
+static const struct nvme_string generic_status[] = {
+	{ SPDK_NVME_SC_SUCCESS, "SUCCESS" },
+	{ SPDK_NVME_SC_INVALID_OPCODE, "INVALID OPCODE" },
+	{ SPDK_NVME_SC_INVALID_FIELD, "INVALID FIELD" },
+	{ SPDK_NVME_SC_COMMAND_ID_CONFLICT, "COMMAND ID CONFLICT" },
+	{ SPDK_NVME_SC_DATA_TRANSFER_ERROR, "DATA TRANSFER ERROR" },
+	{ SPDK_NVME_SC_ABORTED_POWER_LOSS, "ABORTED - POWER LOSS" },
+	{ SPDK_NVME_SC_INTERNAL_DEVICE_ERROR, "INTERNAL DEVICE ERROR" },
+	{ SPDK_NVME_SC_ABORTED_BY_REQUEST, "ABORTED - BY REQUEST" },
+	{ SPDK_NVME_SC_ABORTED_SQ_DELETION, "ABORTED - SQ DELETION" },
+	{ SPDK_NVME_SC_ABORTED_FAILED_FUSED, "ABORTED - FAILED FUSED" },
+	{ SPDK_NVME_SC_ABORTED_MISSING_FUSED, "ABORTED - MISSING FUSED" },
+	{ SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT, "INVALID NAMESPACE OR FORMAT" },
+	{ SPDK_NVME_SC_COMMAND_SEQUENCE_ERROR, "COMMAND SEQUENCE ERROR" },
+	{ SPDK_NVME_SC_INVALID_SGL_SEG_DESCRIPTOR, "INVALID SGL SEGMENT DESCRIPTOR" },
+	{ SPDK_NVME_SC_INVALID_NUM_SGL_DESCIRPTORS, "INVALID NUMBER OF SGL DESCRIPTORS" },
+	{ SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID, "DATA SGL LENGTH INVALID" },
+	{ SPDK_NVME_SC_METADATA_SGL_LENGTH_INVALID, "METADATA SGL LENGTH INVALID" },
+	{ SPDK_NVME_SC_SGL_DESCRIPTOR_TYPE_INVALID, "SGL DESCRIPTOR TYPE INVALID" },
+	{ SPDK_NVME_SC_INVALID_CONTROLLER_MEM_BUF, "INVALID CONTROLLER MEMORY BUFFER" },
+	{ SPDK_NVME_SC_INVALID_PRP_OFFSET, "INVALID PRP OFFSET" },
+	{ SPDK_NVME_SC_ATOMIC_WRITE_UNIT_EXCEEDED, "ATOMIC WRITE UNIT EXCEEDED" },
+	{ SPDK_NVME_SC_OPERATION_DENIED, "OPERATION DENIED" },
+	{ SPDK_NVME_SC_INVALID_SGL_OFFSET, "INVALID SGL OFFSET" },
+	{ SPDK_NVME_SC_HOSTID_INCONSISTENT_FORMAT, "HOSTID INCONSISTENT FORMAT" },
+	{ SPDK_NVME_SC_KEEP_ALIVE_EXPIRED, "KEEP ALIVE EXPIRED" },
+	{ SPDK_NVME_SC_KEEP_ALIVE_INVALID, "KEEP ALIVE INVALID" },
+	{ SPDK_NVME_SC_ABORTED_PREEMPT, "ABORTED - PREEMPT AND ABORT" },
+	{ SPDK_NVME_SC_SANITIZE_FAILED, "SANITIZE FAILED" },
+	{ SPDK_NVME_SC_SANITIZE_IN_PROGRESS, "SANITIZE IN PROGRESS" },
+	{ SPDK_NVME_SC_SGL_DATA_BLOCK_GRANULARITY_INVALID, "DATA BLOCK GRANULARITY INVALID" },
+	{ SPDK_NVME_SC_COMMAND_INVALID_IN_CMB, "COMMAND NOT SUPPORTED FOR QUEUE IN CMB" },
+	{ SPDK_NVME_SC_LBA_OUT_OF_RANGE, "LBA OUT OF RANGE" },
+	{ SPDK_NVME_SC_CAPACITY_EXCEEDED, "CAPACITY EXCEEDED" },
+	{ SPDK_NVME_SC_NAMESPACE_NOT_READY, "NAMESPACE NOT READY" },
+	{ SPDK_NVME_SC_RESERVATION_CONFLICT, "RESERVATION CONFLICT" },
+	{ SPDK_NVME_SC_FORMAT_IN_PROGRESS, "FORMAT IN PROGRESS" },
+	{ 0xFFFF, "GENERIC" }
+};
+
+static const struct nvme_string command_specific_status[] = {
+	{ SPDK_NVME_SC_COMPLETION_QUEUE_INVALID, "INVALID COMPLETION QUEUE" },
+	{ SPDK_NVME_SC_INVALID_QUEUE_IDENTIFIER, "INVALID QUEUE IDENTIFIER" },
+	{ SPDK_NVME_SC_INVALID_QUEUE_SIZE, "INVALID QUEUE SIZE" },
+	{ SPDK_NVME_SC_ABORT_COMMAND_LIMIT_EXCEEDED, "ABORT CMD LIMIT EXCEEDED" },
+	{ SPDK_NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED, "ASYNC LIMIT EXCEEDED" },
+	{ SPDK_NVME_SC_INVALID_FIRMWARE_SLOT, "INVALID FIRMWARE SLOT" },
+	{ SPDK_NVME_SC_INVALID_FIRMWARE_IMAGE, "INVALID FIRMWARE IMAGE" },
+	{ SPDK_NVME_SC_INVALID_INTERRUPT_VECTOR, "INVALID INTERRUPT VECTOR" },
+	{ SPDK_NVME_SC_INVALID_LOG_PAGE, "INVALID LOG PAGE" },
+	{ SPDK_NVME_SC_INVALID_FORMAT, "INVALID FORMAT" },
+	{ SPDK_NVME_SC_FIRMWARE_REQ_CONVENTIONAL_RESET, "FIRMWARE REQUIRES CONVENTIONAL RESET" },
+	{ SPDK_NVME_SC_INVALID_QUEUE_DELETION, "INVALID QUEUE DELETION" },
+	{ SPDK_NVME_SC_FEATURE_ID_NOT_SAVEABLE, "FEATURE ID NOT SAVEABLE" },
+	{ SPDK_NVME_SC_FEATURE_NOT_CHANGEABLE, "FEATURE NOT CHANGEABLE" },
+	{ SPDK_NVME_SC_FEATURE_NOT_NAMESPACE_SPECIFIC, "FEATURE NOT NAMESPACE SPECIFIC" },
+	{ SPDK_NVME_SC_FIRMWARE_REQ_NVM_RESET, "FIRMWARE REQUIRES NVM RESET" },
+	{ SPDK_NVME_SC_FIRMWARE_REQ_RESET, "FIRMWARE REQUIRES RESET" },
+	{ SPDK_NVME_SC_FIRMWARE_REQ_MAX_TIME_VIOLATION, "FIRMWARE REQUIRES MAX TIME VIOLATION" },
+	{ SPDK_NVME_SC_FIRMWARE_ACTIVATION_PROHIBITED, "FIRMWARE ACTIVATION PROHIBITED" },
+	{ SPDK_NVME_SC_OVERLAPPING_RANGE, "OVERLAPPING RANGE" },
+	{ SPDK_NVME_SC_NAMESPACE_INSUFFICIENT_CAPACITY, "NAMESPACE INSUFFICIENT CAPACITY" },
+	{ SPDK_NVME_SC_NAMESPACE_ID_UNAVAILABLE, "NAMESPACE ID UNAVAILABLE" },
+	{ SPDK_NVME_SC_NAMESPACE_ALREADY_ATTACHED, "NAMESPACE ALREADY ATTACHED" },
+	{ SPDK_NVME_SC_NAMESPACE_IS_PRIVATE, "NAMESPACE IS PRIVATE" },
+	{ SPDK_NVME_SC_NAMESPACE_NOT_ATTACHED, "NAMESPACE NOT ATTACHED" },
+	{ SPDK_NVME_SC_THINPROVISIONING_NOT_SUPPORTED, "THINPROVISIONING NOT SUPPORTED" },
+	{ SPDK_NVME_SC_CONTROLLER_LIST_INVALID, "CONTROLLER LIST INVALID" },
+	{ SPDK_NVME_SC_DEVICE_SELF_TEST_IN_PROGRESS, "DEVICE SELF-TEST IN PROGRESS" },
+	{ SPDK_NVME_SC_BOOT_PARTITION_WRITE_PROHIBITED, "BOOT PARTITION WRITE PROHIBITED" },
+	{ SPDK_NVME_SC_INVALID_CTRLR_ID, "INVALID CONTROLLER ID" },
+	{ SPDK_NVME_SC_INVALID_SECONDARY_CTRLR_STATE, "INVALID SECONDARY CONTROLLER STATE" },
+	{ SPDK_NVME_SC_INVALID_NUM_CTRLR_RESOURCES, "INVALID NUMBER OF CONTROLLER RESOURCES" },
+	{ SPDK_NVME_SC_INVALID_RESOURCE_ID, "INVALID RESOURCE IDENTIFIER" },
+	{ SPDK_NVME_SC_CONFLICTING_ATTRIBUTES, "CONFLICTING ATTRIBUTES" },
+	{ SPDK_NVME_SC_INVALID_PROTECTION_INFO, "INVALID PROTECTION INFO" },
+	{ SPDK_NVME_SC_ATTEMPTED_WRITE_TO_RO_RANGE, "WRITE TO RO RANGE" },
+	{ 0xFFFF, "COMMAND SPECIFIC" }
+};
+
+static const struct nvme_string media_error_status[] = {
+	{ SPDK_NVME_SC_WRITE_FAULTS, "WRITE FAULTS" },
+	{ SPDK_NVME_SC_UNRECOVERED_READ_ERROR, "UNRECOVERED READ ERROR" },
+	{ SPDK_NVME_SC_GUARD_CHECK_ERROR, "GUARD CHECK ERROR" },
+	{ SPDK_NVME_SC_APPLICATION_TAG_CHECK_ERROR, "APPLICATION TAG CHECK ERROR" },
+	{ SPDK_NVME_SC_REFERENCE_TAG_CHECK_ERROR, "REFERENCE TAG CHECK ERROR" },
+	{ SPDK_NVME_SC_COMPARE_FAILURE, "COMPARE FAILURE" },
+	{ SPDK_NVME_SC_ACCESS_DENIED, "ACCESS DENIED" },
+	{ SPDK_NVME_SC_DEALLOCATED_OR_UNWRITTEN_BLOCK, "DEALLOCATED OR UNWRITTEN BLOCK" },
+	{ SPDK_OCSSD_SC_OFFLINE_CHUNK, "RESET OFFLINE CHUNK" },
+	{ SPDK_OCSSD_SC_INVALID_RESET, "INVALID RESET" },
+	{ SPDK_OCSSD_SC_WRITE_FAIL_WRITE_NEXT_UNIT, "WRITE FAIL WRITE NEXT UNIT" },
+	{ SPDK_OCSSD_SC_WRITE_FAIL_CHUNK_EARLY_CLOSE, "WRITE FAIL CHUNK EARLY CLOSE" },
+	{ SPDK_OCSSD_SC_OUT_OF_ORDER_WRITE, "OUT OF ORDER WRITE" },
+	{ SPDK_OCSSD_SC_READ_HIGH_ECC, "READ HIGH ECC" },
+	{ 0xFFFF, "MEDIA ERROR" }
+};
+
+static const struct nvme_string path_status[] = {
+	{ SPDK_NVME_SC_INTERNAL_PATH_ERROR, "INTERNAL PATH ERROR" },
+	{ SPDK_NVME_SC_CONTROLLER_PATH_ERROR, "CONTROLLER PATH ERROR" },
+	{ SPDK_NVME_SC_HOST_PATH_ERROR, "HOST PATH ERROR" },
+	{ SPDK_NVME_SC_ABORTED_BY_HOST, "ABORTED BY HOST" },
+	{ 0xFFFF, "PATH ERROR" }
+};
+
+const char *
+spdk_nvme_cpl_get_status_string(const struct spdk_nvme_status *status)
+{
+	const struct nvme_string *entry;
+
+	switch (status->sct) {
+	case SPDK_NVME_SCT_GENERIC:
+		entry = generic_status;
+		break;
+	case SPDK_NVME_SCT_COMMAND_SPECIFIC:
+		entry = command_specific_status;
+		break;
+	case SPDK_NVME_SCT_MEDIA_ERROR:
+		entry = media_error_status;
+		break;
+	case SPDK_NVME_SCT_PATH:
+		entry = path_status;
+		break;
+	case SPDK_NVME_SCT_VENDOR_SPECIFIC:
+		return "VENDOR SPECIFIC";
+	default:
+		return "RESERVED";
+	}
+
+	return nvme_get_string(entry, status->sc);
+}
+
+void
+spdk_nvme_print_completion(uint16_t qid, struct spdk_nvme_cpl *cpl)
+{
+	assert(cpl != NULL);
+
+	/* Check that sqid matches qid. Note that sqid is reserved
+	 * for fabrics so don't print an error when sqid is 0. */
+	if (cpl->sqid != qid && cpl->sqid != 0) {
+		SPDK_ERRLOG("sqid %u doesn't match qid\n", cpl->sqid);
+	}
+
+	SPDK_NOTICELOG("%s (%02x/%02x) qid:%d cid:%d cdw0:%x sqhd:%04x p:%x m:%x dnr:%x\n",
+		       spdk_nvme_cpl_get_status_string(&cpl->status),
+		       cpl->status.sct, cpl->status.sc, qid, cpl->cid, cpl->cdw0,
+		       cpl->sqhd, cpl->status.p, cpl->status.m, cpl->status.dnr);
+}
+
+void
+spdk_nvme_qpair_print_completion(struct spdk_nvme_qpair *qpair, struct spdk_nvme_cpl *cpl)
+{
+	spdk_nvme_print_completion(qpair->id, cpl);
+}
+
+bool
+nvme_completion_is_retry(const struct spdk_nvme_cpl *cpl)
+{
+	/*
+	 * TODO: spec is not clear how commands that are aborted due
+	 *  to TLER will be marked.  So for now, it seems
+	 *  NAMESPACE_NOT_READY is the only case where we should
+	 *  look at the DNR bit.
+	 */
+	switch ((int)cpl->status.sct) {
+	case SPDK_NVME_SCT_GENERIC:
+		switch ((int)cpl->status.sc) {
+		case SPDK_NVME_SC_NAMESPACE_NOT_READY:
+		case SPDK_NVME_SC_FORMAT_IN_PROGRESS:
+			if (cpl->status.dnr) {
+				return false;
+			} else {
+				return true;
+			}
+		case SPDK_NVME_SC_INVALID_OPCODE:
+		case SPDK_NVME_SC_INVALID_FIELD:
+		case SPDK_NVME_SC_COMMAND_ID_CONFLICT:
+		case SPDK_NVME_SC_DATA_TRANSFER_ERROR:
+		case SPDK_NVME_SC_ABORTED_POWER_LOSS:
+		case SPDK_NVME_SC_INTERNAL_DEVICE_ERROR:
+		case SPDK_NVME_SC_ABORTED_BY_REQUEST:
+		case SPDK_NVME_SC_ABORTED_SQ_DELETION:
+		case SPDK_NVME_SC_ABORTED_FAILED_FUSED:
+		case SPDK_NVME_SC_ABORTED_MISSING_FUSED:
+		case SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT:
+		case SPDK_NVME_SC_COMMAND_SEQUENCE_ERROR:
+		case SPDK_NVME_SC_LBA_OUT_OF_RANGE:
+		case SPDK_NVME_SC_CAPACITY_EXCEEDED:
+		default:
+			return false;
+		}
+	case SPDK_NVME_SCT_PATH:
+		/*
+		 * Per NVMe TP 4028 (Path and Transport Error Enhancements), retries should be
+		 * based on the setting of the DNR bit for Internal Path Error
+		 */
+		switch ((int)cpl->status.sc) {
+		case SPDK_NVME_SC_INTERNAL_PATH_ERROR:
+			return !cpl->status.dnr;
+		default:
+			return false;
+		}
+	case SPDK_NVME_SCT_COMMAND_SPECIFIC:
+	case SPDK_NVME_SCT_MEDIA_ERROR:
+	case SPDK_NVME_SCT_VENDOR_SPECIFIC:
+	default:
+		return false;
+	}
+}
+
+static void
+nvme_qpair_manual_complete_request(struct spdk_nvme_qpair *qpair,
+				   struct nvme_request *req, uint32_t sct, uint32_t sc,
+				   uint32_t dnr, bool print_on_error)
+{
+	struct spdk_nvme_cpl	cpl;
+	bool			error;
+
+	memset(&cpl, 0, sizeof(cpl));
+	cpl.sqid = qpair->id;
+	cpl.status.sct = sct;
+	cpl.status.sc = sc;
+	cpl.status.dnr = dnr;
+
+	error = spdk_nvme_cpl_is_error(&cpl);
+
+	if (error && print_on_error && !qpair->ctrlr->opts.disable_error_logging) {
+		SPDK_NOTICELOG("Command completed manually:\n");
+		spdk_nvme_qpair_print_command(qpair, &req->cmd);
+		spdk_nvme_qpair_print_completion(qpair, &cpl);
+	}
+
+	nvme_complete_request(req->cb_fn, req->cb_arg, qpair, req, &cpl);
+	nvme_free_request(req);
+}
+
+static void
+_nvme_qpair_abort_queued_reqs(struct spdk_nvme_qpair *qpair, uint32_t dnr)
+{
+	struct nvme_request		*req;
+
+	while (!STAILQ_EMPTY(&qpair->queued_req)) {
+		req = STAILQ_FIRST(&qpair->queued_req);
+		STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq);
+		if (!qpair->ctrlr->opts.disable_error_logging) {
+			SPDK_ERRLOG("aborting queued i/o\n");
+		}
+		nvme_qpair_manual_complete_request(qpair, req, SPDK_NVME_SCT_GENERIC,
+						   SPDK_NVME_SC_ABORTED_BY_REQUEST, dnr, true);
+	}
+}
+
+/* The callback to a request may submit the next request which is queued and
+ * then the same callback may abort it immediately. This repetition may cause
+ * infinite recursive calls. Hence move aborting requests to another list here
+ * and abort them later at resubmission.
+ */
+static void
+_nvme_qpair_complete_abort_queued_reqs(struct spdk_nvme_qpair *qpair)
+{
+	struct nvme_request		*req;
+
+	while (!STAILQ_EMPTY(&qpair->aborting_queued_req)) {
+		req = STAILQ_FIRST(&qpair->aborting_queued_req);
+		STAILQ_REMOVE_HEAD(&qpair->aborting_queued_req, stailq);
+		nvme_qpair_manual_complete_request(qpair, req, SPDK_NVME_SCT_GENERIC,
+						   SPDK_NVME_SC_ABORTED_BY_REQUEST, 1, true);
+	}
+}
+
+uint32_t
+nvme_qpair_abort_queued_reqs(struct spdk_nvme_qpair *qpair, void *cmd_cb_arg)
+{
+	struct nvme_request	*req, *tmp;
+	uint32_t		aborting = 0;
+
+	STAILQ_FOREACH_SAFE(req, &qpair->queued_req, stailq, tmp) {
+		if (req->cb_arg == cmd_cb_arg) {
+			STAILQ_REMOVE(&qpair->queued_req, req, nvme_request, stailq);
+			STAILQ_INSERT_TAIL(&qpair->aborting_queued_req, req, stailq);
+			if (!qpair->ctrlr->opts.disable_error_logging) {
+				SPDK_ERRLOG("aborting queued i/o\n");
+			}
+			aborting++;
+		}
+	}
+
+	return aborting;
+}
+
+static inline bool
+nvme_qpair_check_enabled(struct spdk_nvme_qpair *qpair)
+{
+	struct nvme_request *req;
+
+	/*
+	 * Either during initial connect or reset, the qpair should follow the given state machine.
+	 * QPAIR_DISABLED->QPAIR_CONNECTING->QPAIR_CONNECTED->QPAIR_ENABLING->QPAIR_ENABLED. In the
+	 * reset case, once the qpair is properly connected, we need to abort any outstanding requests
+	 * from the old transport connection and encourage the application to retry them. We also need
+	 * to submit any queued requests that built up while we were in the connected or enabling state.
+	 */
+	if (nvme_qpair_get_state(qpair) == NVME_QPAIR_CONNECTED && !qpair->ctrlr->is_resetting) {
+		nvme_qpair_set_state(qpair, NVME_QPAIR_ENABLING);
+		/*
+		 * PCIe is special, for fabrics transports, we can abort requests before disconnect during reset
+		 * but we have historically not disconnected pcie qpairs during reset so we have to abort requests
+		 * here.
+		 */
+		if (qpair->ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) {
+			nvme_qpair_abort_reqs(qpair, 0);
+		}
+		nvme_qpair_set_state(qpair, NVME_QPAIR_ENABLED);
+		while (!STAILQ_EMPTY(&qpair->queued_req)) {
+			req = STAILQ_FIRST(&qpair->queued_req);
+			STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq);
+			if (nvme_qpair_resubmit_request(qpair, req)) {
+				break;
+			}
+		}
+	}
+
+	/*
+	 * When doing a reset, we must disconnect the qpair on the proper core.
+	 * Note, reset is the only case where we set the failure reason without
+	 * setting the qpair state since reset is done at the generic layer on the
+	 * controller thread and we can't disconnect I/O qpairs from the controller
+	 * thread.
+	 */
+	if (qpair->transport_failure_reason != SPDK_NVME_QPAIR_FAILURE_NONE &&
+	    nvme_qpair_get_state(qpair) == NVME_QPAIR_ENABLED) {
+		/* Don't disconnect PCIe qpairs. They are a special case for reset. */
+		if (qpair->ctrlr->trid.trtype != SPDK_NVME_TRANSPORT_PCIE) {
+			nvme_ctrlr_disconnect_qpair(qpair);
+		}
+		return false;
+	}
+
+	return nvme_qpair_get_state(qpair) == NVME_QPAIR_ENABLED;
+}
+
+void
+nvme_qpair_resubmit_requests(struct spdk_nvme_qpair *qpair, uint32_t num_requests)
+{
+	uint32_t i;
+	int resubmit_rc;
+	struct nvme_request *req;
+
+	for (i = 0; i < num_requests; i++) {
+		if (qpair->ctrlr->is_resetting) {
+			break;
+		}
+		if ((req = STAILQ_FIRST(&qpair->queued_req)) == NULL) {
+			break;
+		}
+		STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq);
+		resubmit_rc = nvme_qpair_resubmit_request(qpair, req);
+		if (spdk_unlikely(resubmit_rc != 0)) {
+			SPDK_ERRLOG("Unable to resubmit as many requests as we completed.\n");
+			break;
+		}
+	}
+
+	_nvme_qpair_complete_abort_queued_reqs(qpair);
+}
+
+int32_t
+spdk_nvme_qpair_process_completions(struct spdk_nvme_qpair *qpair, uint32_t max_completions)
+{
+	int32_t ret;
+	struct nvme_request *req, *tmp;
+
+	if (spdk_unlikely(qpair->ctrlr->is_failed)) {
+		if (qpair->ctrlr->is_removed) {
+			nvme_qpair_set_state(qpair, NVME_QPAIR_DESTROYING);
+			nvme_qpair_abort_reqs(qpair, 1 /* Do not retry */);
+		}
+		return -ENXIO;
+	}
+
+	if (spdk_unlikely(!nvme_qpair_check_enabled(qpair) &&
+			  !(nvme_qpair_get_state(qpair) == NVME_QPAIR_CONNECTING))) {
+		/*
+		 * qpair is not enabled, likely because a controller reset is
+		 *  in progress.
+		 */
+		return -ENXIO;
+	}
+
+	/* error injection for those queued error requests */
+	if (spdk_unlikely(!STAILQ_EMPTY(&qpair->err_req_head))) {
+		STAILQ_FOREACH_SAFE(req, &qpair->err_req_head, stailq, tmp) {
+			if (spdk_get_ticks() - req->submit_tick > req->timeout_tsc) {
+				STAILQ_REMOVE(&qpair->err_req_head, req, nvme_request, stailq);
+				nvme_qpair_manual_complete_request(qpair, req,
+								   req->cpl.status.sct,
+								   req->cpl.status.sc, 0, true);
+			}
+		}
+	}
+
+	qpair->in_completion_context = 1;
+	ret = nvme_transport_qpair_process_completions(qpair, max_completions);
+	if (ret < 0) {
+		SPDK_ERRLOG("CQ error, abort requests after transport retry counter exceeded\n");
+		if (nvme_qpair_is_admin_queue(qpair)) {
+			nvme_ctrlr_fail(qpair->ctrlr, false);
+		}
+	}
+	qpair->in_completion_context = 0;
+	if (qpair->delete_after_completion_context) {
+		/*
+		 * A request to delete this qpair was made in the context of this completion
+		 *  routine - so it is safe to delete it now.
+		 */
+		spdk_nvme_ctrlr_free_io_qpair(qpair);
+		return ret;
+	}
+
+	/*
+	 * At this point, ret must represent the number of completions we reaped.
+	 * submit as many queued requests as we completed.
+	 */
+	nvme_qpair_resubmit_requests(qpair, ret);
+
+	return ret;
+}
+
+spdk_nvme_qp_failure_reason
+spdk_nvme_qpair_get_failure_reason(struct spdk_nvme_qpair *qpair)
+{
+	return qpair->transport_failure_reason;
+}
+
+int
+nvme_qpair_init(struct spdk_nvme_qpair *qpair, uint16_t id,
+		struct spdk_nvme_ctrlr *ctrlr,
+		enum spdk_nvme_qprio qprio,
+		uint32_t num_requests)
+{
+	size_t req_size_padded;
+	uint32_t i;
+
+	qpair->id = id;
+	qpair->qprio = qprio;
+
+	qpair->in_completion_context = 0;
+	qpair->delete_after_completion_context = 0;
+	qpair->no_deletion_notification_needed = 0;
+
+	qpair->ctrlr = ctrlr;
+	qpair->trtype = ctrlr->trid.trtype;
+
+	STAILQ_INIT(&qpair->free_req);
+	STAILQ_INIT(&qpair->queued_req);
+	STAILQ_INIT(&qpair->aborting_queued_req);
+	TAILQ_INIT(&qpair->err_cmd_head);
+	STAILQ_INIT(&qpair->err_req_head);
+
+	req_size_padded = (sizeof(struct nvme_request) + 63) & ~(size_t)63;
+
+	qpair->req_buf = spdk_zmalloc(req_size_padded * num_requests, 64, NULL,
+				      SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_SHARE);
+	if (qpair->req_buf == NULL) {
+		SPDK_ERRLOG("no memory to allocate qpair(cntlid:0x%x sqid:%d) req_buf with %d request\n",
+			    ctrlr->cntlid, qpair->id, num_requests);
+		return -ENOMEM;
+	}
+
+	for (i = 0; i < num_requests; i++) {
+		struct nvme_request *req = qpair->req_buf + i * req_size_padded;
+
+		req->qpair = qpair;
+		STAILQ_INSERT_HEAD(&qpair->free_req, req, stailq);
+	}
+
+	return 0;
+}
+
+void
+nvme_qpair_complete_error_reqs(struct spdk_nvme_qpair *qpair)
+{
+	struct nvme_request		*req;
+
+	while (!STAILQ_EMPTY(&qpair->err_req_head)) {
+		req = STAILQ_FIRST(&qpair->err_req_head);
+		STAILQ_REMOVE_HEAD(&qpair->err_req_head, stailq);
+		nvme_qpair_manual_complete_request(qpair, req,
+						   req->cpl.status.sct,
+						   req->cpl.status.sc, 0, true);
+	}
+}
+
+void
+nvme_qpair_deinit(struct spdk_nvme_qpair *qpair)
+{
+	struct nvme_error_cmd *cmd, *entry;
+
+	_nvme_qpair_abort_queued_reqs(qpair, 1);
+	_nvme_qpair_complete_abort_queued_reqs(qpair);
+	nvme_qpair_complete_error_reqs(qpair);
+
+	TAILQ_FOREACH_SAFE(cmd, &qpair->err_cmd_head, link, entry) {
+		TAILQ_REMOVE(&qpair->err_cmd_head, cmd, link);
+		spdk_free(cmd);
+	}
+
+	spdk_free(qpair->req_buf);
+}
+
+static inline int
+_nvme_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req)
+{
+	int			rc = 0;
+	struct nvme_request	*child_req, *tmp;
+	struct nvme_error_cmd	*cmd;
+	struct spdk_nvme_ctrlr	*ctrlr = qpair->ctrlr;
+	bool			child_req_failed = false;
+
+	nvme_qpair_check_enabled(qpair);
+
+	if (req->num_children) {
+		/*
+		 * This is a split (parent) request. Submit all of the children but not the parent
+		 * request itself, since the parent is the original unsplit request.
+		 */
+		TAILQ_FOREACH_SAFE(child_req, &req->children, child_tailq, tmp) {
+			if (spdk_likely(!child_req_failed)) {
+				rc = nvme_qpair_submit_request(qpair, child_req);
+				if (spdk_unlikely(rc != 0)) {
+					child_req_failed = true;
+				}
+			} else { /* free remaining child_reqs since one child_req fails */
+				nvme_request_remove_child(req, child_req);
+				nvme_request_free_children(child_req);
+				nvme_free_request(child_req);
+			}
+		}
+
+		if (spdk_unlikely(child_req_failed)) {
+			/* part of children requests have been submitted,
+			 * return success since we must wait for those children to complete,
+			 * but set the parent request to failure.
+			 */
+			if (req->num_children) {
+				req->cpl.status.sct = SPDK_NVME_SCT_GENERIC;
+				req->cpl.status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
+				return 0;
+			}
+			goto error;
+		}
+
+		return rc;
+	}
+
+	/* queue those requests which matches with opcode in err_cmd list */
+	if (spdk_unlikely(!TAILQ_EMPTY(&qpair->err_cmd_head))) {
+		TAILQ_FOREACH(cmd, &qpair->err_cmd_head, link) {
+			if (!cmd->do_not_submit) {
+				continue;
+			}
+
+			if ((cmd->opc == req->cmd.opc) && cmd->err_count) {
+				/* add to error request list and set cpl */
+				req->timeout_tsc = cmd->timeout_tsc;
+				req->submit_tick = spdk_get_ticks();
+				req->cpl.status.sct = cmd->status.sct;
+				req->cpl.status.sc = cmd->status.sc;
+				STAILQ_INSERT_TAIL(&qpair->err_req_head, req, stailq);
+				cmd->err_count--;
+				return 0;
+			}
+		}
+	}
+
+	if (spdk_unlikely(ctrlr->is_failed)) {
+		rc = -ENXIO;
+		goto error;
+	}
+
+	/* assign submit_tick before submitting req to specific transport */
+	if (spdk_unlikely(ctrlr->timeout_enabled)) {
+		if (req->submit_tick == 0) { /* req submitted for the first time */
+			req->submit_tick = spdk_get_ticks();
+			req->timed_out = false;
+		}
+	} else {
+		req->submit_tick = 0;
+	}
+
+	/* Allow two cases:
+	 * 1. NVMe qpair is enabled.
+	 * 2. Always allow fabrics commands through - these get
+	 * the controller out of reset state.
+	 */
+	if (spdk_likely(nvme_qpair_get_state(qpair) == NVME_QPAIR_ENABLED) ||
+	    (req->cmd.opc == SPDK_NVME_OPC_FABRIC &&
+	     nvme_qpair_get_state(qpair) == NVME_QPAIR_CONNECTING)) {
+		rc = nvme_transport_qpair_submit_request(qpair, req);
+	} else {
+		/* The controller is being reset - queue this request and
+		 *  submit it later when the reset is completed.
+		 */
+		return -EAGAIN;
+	}
+
+	if (spdk_likely(rc == 0)) {
+		req->queued = false;
+		return 0;
+	}
+
+	if (rc == -EAGAIN) {
+		return -EAGAIN;
+	}
+
+error:
+	if (req->parent != NULL) {
+		nvme_request_remove_child(req->parent, req);
+	}
+
+	/* The request is from queued_req list we should trigger the callback from caller */
+	if (spdk_unlikely(req->queued)) {
+		nvme_qpair_manual_complete_request(qpair, req, SPDK_NVME_SCT_GENERIC,
+						   SPDK_NVME_SC_INTERNAL_DEVICE_ERROR, true, true);
+		return rc;
+	}
+
+	nvme_free_request(req);
+
+	return rc;
+}
+
+int
+nvme_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req)
+{
+	int rc;
+
+	/* This prevents us from entering an infinite loop when freeing queued I/O in disconnect. */
+	if (spdk_unlikely(nvme_qpair_get_state(qpair) == NVME_QPAIR_DISCONNECTING ||
+			  nvme_qpair_get_state(qpair) == NVME_QPAIR_DESTROYING)) {
+		if (req->parent != NULL) {
+			nvme_request_remove_child(req->parent, req);
+		}
+		nvme_free_request(req);
+		return -ENXIO;
+	}
+
+	if (spdk_unlikely(!STAILQ_EMPTY(&qpair->queued_req) && req->num_children == 0)) {
+		/*
+		 * requests that have no children should be sent to the transport after all
+		 * currently queued requests. Requests with chilren will be split and go back
+		 * through this path.
+		 */
+		STAILQ_INSERT_TAIL(&qpair->queued_req, req, stailq);
+		req->queued = true;
+		return 0;
+	}
+
+	rc = _nvme_qpair_submit_request(qpair, req);
+	if (rc == -EAGAIN) {
+		STAILQ_INSERT_TAIL(&qpair->queued_req, req, stailq);
+		req->queued = true;
+		rc = 0;
+	}
+
+	return rc;
+}
+
+static int
+nvme_qpair_resubmit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req)
+{
+	int rc;
+
+	/*
+	 * We should never have a request with children on the queue.
+	 * This is necessary to preserve the 1:1 relationship between
+	 * completions and resubmissions.
+	 */
+	assert(req->num_children == 0);
+	assert(req->queued);
+	rc = _nvme_qpair_submit_request(qpair, req);
+	if (spdk_unlikely(rc == -EAGAIN)) {
+		STAILQ_INSERT_HEAD(&qpair->queued_req, req, stailq);
+	}
+
+	return rc;
+}
+
+void
+nvme_qpair_abort_reqs(struct spdk_nvme_qpair *qpair, uint32_t dnr)
+{
+	nvme_qpair_complete_error_reqs(qpair);
+	_nvme_qpair_abort_queued_reqs(qpair, dnr);
+	_nvme_qpair_complete_abort_queued_reqs(qpair);
+	nvme_transport_qpair_abort_reqs(qpair, dnr);
+}
+
+int
+spdk_nvme_qpair_add_cmd_error_injection(struct spdk_nvme_ctrlr *ctrlr,
+					struct spdk_nvme_qpair *qpair,
+					uint8_t opc, bool do_not_submit,
+					uint64_t timeout_in_us,
+					uint32_t err_count,
+					uint8_t sct, uint8_t sc)
+{
+	struct nvme_error_cmd *entry, *cmd = NULL;
+
+	if (qpair == NULL) {
+		qpair = ctrlr->adminq;
+	}
+
+	TAILQ_FOREACH(entry, &qpair->err_cmd_head, link) {
+		if (entry->opc == opc) {
+			cmd = entry;
+			break;
+		}
+	}
+
+	if (cmd == NULL) {
+		cmd = spdk_zmalloc(sizeof(*cmd), 64, NULL, SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
+		if (!cmd) {
+			return -ENOMEM;
+		}
+		TAILQ_INSERT_TAIL(&qpair->err_cmd_head, cmd, link);
+	}
+
+	cmd->do_not_submit = do_not_submit;
+	cmd->err_count = err_count;
+	cmd->timeout_tsc = timeout_in_us * spdk_get_ticks_hz() / 1000000ULL;
+	cmd->opc = opc;
+	cmd->status.sct = sct;
+	cmd->status.sc = sc;
+
+	return 0;
+}
+
+void
+spdk_nvme_qpair_remove_cmd_error_injection(struct spdk_nvme_ctrlr *ctrlr,
+		struct spdk_nvme_qpair *qpair,
+		uint8_t opc)
+{
+	struct nvme_error_cmd *cmd, *entry;
+
+	if (qpair == NULL) {
+		qpair = ctrlr->adminq;
+	}
+
+	TAILQ_FOREACH_SAFE(cmd, &qpair->err_cmd_head, link, entry) {
+		if (cmd->opc == opc) {
+			TAILQ_REMOVE(&qpair->err_cmd_head, cmd, link);
+			spdk_free(cmd);
+			return;
+		}
+	}
+
+	return;
+}
diff --git a/src/spdk/lib/nvme/nvme_quirks.c b/src/spdk/lib/nvme/nvme_quirks.c
new file mode 100644
index 000000000..38c8f0eae
--- /dev/null
+++ b/src/spdk/lib/nvme/nvme_quirks.c
@@ -0,0 +1,155 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "nvme_internal.h"
+
+struct nvme_quirk {
+	struct spdk_pci_id	id;
+	uint64_t		flags;
+};
+
+static const struct nvme_quirk nvme_quirks[] = {
+	{	{SPDK_PCI_CLASS_NVME, SPDK_PCI_VID_INTEL, 0x0953, SPDK_PCI_ANY_ID, SPDK_PCI_ANY_ID},
+		NVME_INTEL_QUIRK_READ_LATENCY |
+		NVME_INTEL_QUIRK_WRITE_LATENCY |
+		NVME_INTEL_QUIRK_STRIPING |
+		NVME_QUIRK_READ_ZERO_AFTER_DEALLOCATE |
+		NVME_QUIRK_DELAY_BEFORE_INIT |
+		NVME_QUIRK_MINIMUM_IO_QUEUE_SIZE
+	},
+	{	{SPDK_PCI_CLASS_NVME, SPDK_PCI_VID_INTEL, 0x0A53, SPDK_PCI_ANY_ID, SPDK_PCI_ANY_ID},
+		NVME_INTEL_QUIRK_READ_LATENCY |
+		NVME_INTEL_QUIRK_WRITE_LATENCY |
+		NVME_INTEL_QUIRK_STRIPING |
+		NVME_QUIRK_READ_ZERO_AFTER_DEALLOCATE |
+		NVME_QUIRK_DELAY_BEFORE_INIT |
+		NVME_QUIRK_MINIMUM_IO_QUEUE_SIZE
+	},
+	{	{SPDK_PCI_CLASS_NVME, SPDK_PCI_VID_INTEL, 0x0A54, SPDK_PCI_ANY_ID, SPDK_PCI_ANY_ID},
+		NVME_INTEL_QUIRK_READ_LATENCY |
+		NVME_INTEL_QUIRK_WRITE_LATENCY |
+		NVME_INTEL_QUIRK_STRIPING |
+		NVME_QUIRK_READ_ZERO_AFTER_DEALLOCATE |
+		NVME_QUIRK_MINIMUM_IO_QUEUE_SIZE
+	},
+	{	{SPDK_PCI_CLASS_NVME, SPDK_PCI_VID_INTEL, 0x0A55, SPDK_PCI_ANY_ID, SPDK_PCI_ANY_ID},
+		NVME_INTEL_QUIRK_READ_LATENCY |
+		NVME_INTEL_QUIRK_WRITE_LATENCY |
+		NVME_INTEL_QUIRK_STRIPING |
+		NVME_QUIRK_READ_ZERO_AFTER_DEALLOCATE |
+		NVME_QUIRK_MINIMUM_IO_QUEUE_SIZE
+	},
+	{	{SPDK_PCI_CLASS_NVME, SPDK_PCI_VID_MEMBLAZE, 0x0540, SPDK_PCI_ANY_ID, SPDK_PCI_ANY_ID},
+		NVME_QUIRK_DELAY_BEFORE_CHK_RDY
+	},
+	{	{SPDK_PCI_CLASS_NVME, SPDK_PCI_VID_SAMSUNG, 0xa821, SPDK_PCI_ANY_ID, SPDK_PCI_ANY_ID},
+		NVME_QUIRK_DELAY_BEFORE_CHK_RDY
+	},
+	{	{SPDK_PCI_CLASS_NVME, SPDK_PCI_VID_SAMSUNG, 0xa822, SPDK_PCI_ANY_ID, SPDK_PCI_ANY_ID},
+		NVME_QUIRK_DELAY_BEFORE_CHK_RDY
+	},
+	{	{SPDK_PCI_CLASS_NVME, SPDK_PCI_VID_VIRTUALBOX, 0x4e56, SPDK_PCI_ANY_ID, SPDK_PCI_ANY_ID},
+		NVME_QUIRK_DELAY_AFTER_QUEUE_ALLOC
+	},
+	{	{SPDK_PCI_CLASS_NVME, SPDK_PCI_VID_INTEL, 0x5845, SPDK_PCI_ANY_ID, SPDK_PCI_ANY_ID},
+		NVME_QUIRK_IDENTIFY_CNS |
+		NVME_INTEL_QUIRK_NO_LOG_PAGES |
+		NVME_QUIRK_MAXIMUM_PCI_ACCESS_WIDTH
+	},
+	{	{SPDK_PCI_CLASS_NVME, SPDK_PCI_VID_CNEXLABS, 0x1f1f, SPDK_PCI_ANY_ID, SPDK_PCI_ANY_ID},
+		NVME_QUIRK_IDENTIFY_CNS |
+		NVME_QUIRK_OCSSD
+	},
+	{	{SPDK_PCI_CLASS_NVME, SPDK_PCI_VID_VMWARE, 0x07f0, SPDK_PCI_ANY_ID, SPDK_PCI_ANY_ID},
+		NVME_QUIRK_SHST_COMPLETE
+	},
+	{	{SPDK_PCI_CLASS_NVME, SPDK_PCI_VID_INTEL, 0x2700, SPDK_PCI_ANY_ID, SPDK_PCI_ANY_ID},
+		NVME_QUIRK_OACS_SECURITY
+	},
+	{	{0x000000, 0x0000, 0x0000, 0x0000, 0x0000}, 0}
+};
+
+/* Compare each field. SPDK_PCI_ANY_ID in s1 matches everything */
+static bool
+pci_id_match(const struct spdk_pci_id *s1, const struct spdk_pci_id *s2)
+{
+	if ((s1->class_id == SPDK_PCI_CLASS_ANY_ID || s1->class_id == s2->class_id) &&
+	    (s1->vendor_id == SPDK_PCI_ANY_ID || s1->vendor_id == s2->vendor_id) &&
+	    (s1->device_id == SPDK_PCI_ANY_ID || s1->device_id == s2->device_id) &&
+	    (s1->subvendor_id == SPDK_PCI_ANY_ID || s1->subvendor_id == s2->subvendor_id) &&
+	    (s1->subdevice_id == SPDK_PCI_ANY_ID || s1->subdevice_id == s2->subdevice_id)) {
+		return true;
+	}
+	return false;
+}
+
+uint64_t
+nvme_get_quirks(const struct spdk_pci_id *id)
+{
+	const struct nvme_quirk *quirk = nvme_quirks;
+
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "Searching for %04x:%04x [%04x:%04x]...\n",
+		      id->vendor_id, id->device_id,
+		      id->subvendor_id, id->subdevice_id);
+
+	while (quirk->id.vendor_id) {
+		if (pci_id_match(&quirk->id, id)) {
+			SPDK_DEBUGLOG(SPDK_LOG_NVME, "Matched quirk %04x:%04x [%04x:%04x]:\n",
+				      quirk->id.vendor_id, quirk->id.device_id,
+				      quirk->id.subvendor_id, quirk->id.subdevice_id);
+
+#define PRINT_QUIRK(quirk_flag) \
+			do { \
+				if (quirk->flags & (quirk_flag)) { \
+					SPDK_DEBUGLOG(SPDK_LOG_NVME, "Quirk enabled: %s\n", #quirk_flag); \
+				} \
+			} while (0)
+
+			PRINT_QUIRK(NVME_INTEL_QUIRK_READ_LATENCY);
+			PRINT_QUIRK(NVME_INTEL_QUIRK_WRITE_LATENCY);
+			PRINT_QUIRK(NVME_QUIRK_DELAY_BEFORE_CHK_RDY);
+			PRINT_QUIRK(NVME_INTEL_QUIRK_STRIPING);
+			PRINT_QUIRK(NVME_QUIRK_DELAY_AFTER_QUEUE_ALLOC);
+			PRINT_QUIRK(NVME_QUIRK_READ_ZERO_AFTER_DEALLOCATE);
+			PRINT_QUIRK(NVME_QUIRK_IDENTIFY_CNS);
+			PRINT_QUIRK(NVME_QUIRK_OCSSD);
+
+			return quirk->flags;
+		}
+		quirk++;
+	}
+
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "No quirks found.\n");
+
+	return 0;
+}
diff --git a/src/spdk/lib/nvme/nvme_rdma.c b/src/spdk/lib/nvme/nvme_rdma.c
new file mode 100644
index 000000000..84537c4a1
--- /dev/null
+++ b/src/spdk/lib/nvme/nvme_rdma.c
@@ -0,0 +1,2852 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation. All rights reserved.
+ *   Copyright (c) 2019, 2020 Mellanox Technologies LTD. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * NVMe over RDMA transport
+ */
+
+#include "spdk/stdinc.h"
+
+#include "spdk/assert.h"
+#include "spdk/log.h"
+#include "spdk/trace.h"
+#include "spdk/queue.h"
+#include "spdk/nvme.h"
+#include "spdk/nvmf_spec.h"
+#include "spdk/string.h"
+#include "spdk/endian.h"
+#include "spdk/likely.h"
+#include "spdk/config.h"
+
+#include "nvme_internal.h"
+#include "spdk_internal/rdma.h"
+
+#define NVME_RDMA_TIME_OUT_IN_MS 2000
+#define NVME_RDMA_RW_BUFFER_SIZE 131072
+
+/*
+ * NVME RDMA qpair Resource Defaults
+ */
+#define NVME_RDMA_DEFAULT_TX_SGE		2
+#define NVME_RDMA_DEFAULT_RX_SGE		1
+
+/* Max number of NVMe-oF SGL descriptors supported by the host */
+#define NVME_RDMA_MAX_SGL_DESCRIPTORS		16
+
+/* number of STAILQ entries for holding pending RDMA CM events. */
+#define NVME_RDMA_NUM_CM_EVENTS			256
+
+/* CM event processing timeout */
+#define NVME_RDMA_QPAIR_CM_EVENT_TIMEOUT_US	1000000
+
+/* The default size for a shared rdma completion queue. */
+#define DEFAULT_NVME_RDMA_CQ_SIZE		4096
+
+/*
+ * In the special case of a stale connection we don't expose a mechanism
+ * for the user to retry the connection so we need to handle it internally.
+ */
+#define NVME_RDMA_STALE_CONN_RETRY_MAX		5
+#define NVME_RDMA_STALE_CONN_RETRY_DELAY_US	10000
+
+/*
+ * Maximum value of transport_retry_count used by RDMA controller
+ */
+#define NVME_RDMA_CTRLR_MAX_TRANSPORT_RETRY_COUNT	7
+
+/*
+ * Maximum value of transport_ack_timeout used by RDMA controller
+ */
+#define NVME_RDMA_CTRLR_MAX_TRANSPORT_ACK_TIMEOUT	31
+
+/*
+ * Number of poller cycles to keep a pointer to destroyed qpairs
+ * in the poll group.
+ */
+#define NVME_RDMA_DESTROYED_QPAIR_EXPIRATION_CYCLES	50
+
+/*
+ * The max length of keyed SGL data block (3 bytes)
+ */
+#define NVME_RDMA_MAX_KEYED_SGL_LENGTH ((1u << 24u) - 1)
+
+#define WC_PER_QPAIR(queue_depth)	(queue_depth * 2)
+
+enum nvme_rdma_wr_type {
+	RDMA_WR_TYPE_RECV,
+	RDMA_WR_TYPE_SEND,
+};
+
+struct nvme_rdma_wr {
+	/* Using this instead of the enum allows this struct to only occupy one byte. */
+	uint8_t	type;
+};
+
+struct spdk_nvmf_cmd {
+	struct spdk_nvme_cmd cmd;
+	struct spdk_nvme_sgl_descriptor sgl[NVME_RDMA_MAX_SGL_DESCRIPTORS];
+};
+
+struct spdk_nvme_rdma_hooks g_nvme_hooks = {};
+
+/* Mapping from virtual address to ibv_mr pointer for a protection domain */
+struct spdk_nvme_rdma_mr_map {
+	struct ibv_pd				*pd;
+	struct spdk_mem_map			*map;
+	uint64_t				ref;
+	LIST_ENTRY(spdk_nvme_rdma_mr_map)	link;
+};
+
+/* STAILQ wrapper for cm events. */
+struct nvme_rdma_cm_event_entry {
+	struct rdma_cm_event			*evt;
+	STAILQ_ENTRY(nvme_rdma_cm_event_entry)	link;
+};
+
+/* NVMe RDMA transport extensions for spdk_nvme_ctrlr */
+struct nvme_rdma_ctrlr {
+	struct spdk_nvme_ctrlr			ctrlr;
+
+	struct ibv_pd				*pd;
+
+	uint16_t				max_sge;
+
+	struct rdma_event_channel		*cm_channel;
+
+	STAILQ_HEAD(, nvme_rdma_cm_event_entry)	pending_cm_events;
+
+	STAILQ_HEAD(, nvme_rdma_cm_event_entry)	free_cm_events;
+
+	struct nvme_rdma_cm_event_entry		*cm_events;
+};
+
+struct nvme_rdma_destroyed_qpair {
+	struct nvme_rdma_qpair			*destroyed_qpair_tracker;
+	uint32_t				completed_cycles;
+	STAILQ_ENTRY(nvme_rdma_destroyed_qpair)	link;
+};
+
+struct nvme_rdma_poller {
+	struct ibv_context		*device;
+	struct ibv_cq			*cq;
+	int				required_num_wc;
+	int				current_num_wc;
+	STAILQ_ENTRY(nvme_rdma_poller)	link;
+};
+
+struct nvme_rdma_poll_group {
+	struct spdk_nvme_transport_poll_group		group;
+	STAILQ_HEAD(, nvme_rdma_poller)			pollers;
+	int						num_pollers;
+	STAILQ_HEAD(, nvme_rdma_destroyed_qpair)	destroyed_qpairs;
+};
+
+struct spdk_nvme_send_wr_list {
+	struct ibv_send_wr	*first;
+	struct ibv_send_wr	*last;
+};
+
+struct spdk_nvme_recv_wr_list {
+	struct ibv_recv_wr	*first;
+	struct ibv_recv_wr	*last;
+};
+
+/* Memory regions */
+union nvme_rdma_mr {
+	struct ibv_mr	*mr;
+	uint64_t	key;
+};
+
+/* NVMe RDMA qpair extensions for spdk_nvme_qpair */
+struct nvme_rdma_qpair {
+	struct spdk_nvme_qpair			qpair;
+
+	struct spdk_rdma_qp			*rdma_qp;
+	struct rdma_cm_id			*cm_id;
+	struct ibv_cq				*cq;
+
+	struct	spdk_nvme_rdma_req		*rdma_reqs;
+
+	uint32_t				max_send_sge;
+
+	uint32_t				max_recv_sge;
+
+	uint16_t				num_entries;
+
+	bool					delay_cmd_submit;
+
+	bool					poll_group_disconnect_in_progress;
+
+	uint32_t				num_completions;
+
+	/* Parallel arrays of response buffers + response SGLs of size num_entries */
+	struct ibv_sge				*rsp_sgls;
+	struct spdk_nvme_rdma_rsp		*rsps;
+
+	struct ibv_recv_wr			*rsp_recv_wrs;
+
+	struct spdk_nvme_send_wr_list		sends_to_post;
+	struct spdk_nvme_recv_wr_list		recvs_to_post;
+
+	/* Memory region describing all rsps for this qpair */
+	union nvme_rdma_mr			rsp_mr;
+
+	/*
+	 * Array of num_entries NVMe commands registered as RDMA message buffers.
+	 * Indexed by rdma_req->id.
+	 */
+	struct spdk_nvmf_cmd			*cmds;
+
+	/* Memory region describing all cmds for this qpair */
+	union nvme_rdma_mr			cmd_mr;
+
+	struct spdk_nvme_rdma_mr_map		*mr_map;
+
+	TAILQ_HEAD(, spdk_nvme_rdma_req)	free_reqs;
+	TAILQ_HEAD(, spdk_nvme_rdma_req)	outstanding_reqs;
+
+	/* Counts of outstanding send and recv objects */
+	uint16_t				current_num_recvs;
+	uint16_t				current_num_sends;
+
+	/* Placed at the end of the struct since it is not used frequently */
+	struct rdma_cm_event			*evt;
+
+	/* Used by poll group to keep the qpair around until it is ready to remove it. */
+	bool					defer_deletion_to_pg;
+};
+
+enum NVME_RDMA_COMPLETION_FLAGS {
+	NVME_RDMA_SEND_COMPLETED = 1u << 0,
+	NVME_RDMA_RECV_COMPLETED = 1u << 1,
+};
+
+struct spdk_nvme_rdma_req {
+	uint16_t				id;
+	uint16_t				completion_flags: 2;
+	uint16_t				reserved: 14;
+	/* if completion of RDMA_RECV received before RDMA_SEND, we will complete nvme request
+	 * during processing of RDMA_SEND. To complete the request we must know the index
+	 * of nvme_cpl received in RDMA_RECV, so store it in this field */
+	uint16_t				rsp_idx;
+
+	struct nvme_rdma_wr			rdma_wr;
+
+	struct ibv_send_wr			send_wr;
+
+	struct nvme_request			*req;
+
+	struct ibv_sge				send_sgl[NVME_RDMA_DEFAULT_TX_SGE];
+
+	TAILQ_ENTRY(spdk_nvme_rdma_req)		link;
+};
+
+enum nvme_rdma_key_type {
+	NVME_RDMA_MR_RKEY,
+	NVME_RDMA_MR_LKEY
+};
+
+struct spdk_nvme_rdma_rsp {
+	struct spdk_nvme_cpl	cpl;
+	struct nvme_rdma_qpair	*rqpair;
+	uint16_t		idx;
+	struct nvme_rdma_wr	rdma_wr;
+};
+
+static const char *rdma_cm_event_str[] = {
+	"RDMA_CM_EVENT_ADDR_RESOLVED",
+	"RDMA_CM_EVENT_ADDR_ERROR",
+	"RDMA_CM_EVENT_ROUTE_RESOLVED",
+	"RDMA_CM_EVENT_ROUTE_ERROR",
+	"RDMA_CM_EVENT_CONNECT_REQUEST",
+	"RDMA_CM_EVENT_CONNECT_RESPONSE",
+	"RDMA_CM_EVENT_CONNECT_ERROR",
+	"RDMA_CM_EVENT_UNREACHABLE",
+	"RDMA_CM_EVENT_REJECTED",
+	"RDMA_CM_EVENT_ESTABLISHED",
+	"RDMA_CM_EVENT_DISCONNECTED",
+	"RDMA_CM_EVENT_DEVICE_REMOVAL",
+	"RDMA_CM_EVENT_MULTICAST_JOIN",
+	"RDMA_CM_EVENT_MULTICAST_ERROR",
+	"RDMA_CM_EVENT_ADDR_CHANGE",
+	"RDMA_CM_EVENT_TIMEWAIT_EXIT"
+};
+
+static LIST_HEAD(, spdk_nvme_rdma_mr_map) g_rdma_mr_maps = LIST_HEAD_INITIALIZER(&g_rdma_mr_maps);
+static pthread_mutex_t g_rdma_mr_maps_mutex = PTHREAD_MUTEX_INITIALIZER;
+struct nvme_rdma_qpair *nvme_rdma_poll_group_get_qpair_by_id(struct nvme_rdma_poll_group *group,
+		uint32_t qp_num);
+
+static inline void *
+nvme_rdma_calloc(size_t nmemb, size_t size)
+{
+	if (!g_nvme_hooks.get_rkey) {
+		return calloc(nmemb, size);
+	} else {
+		return spdk_zmalloc(nmemb * size, 0, NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA);
+	}
+}
+
+static inline void
+nvme_rdma_free(void *buf)
+{
+	if (!g_nvme_hooks.get_rkey) {
+		free(buf);
+	} else {
+		spdk_free(buf);
+	}
+}
+
+static int nvme_rdma_ctrlr_delete_io_qpair(struct spdk_nvme_ctrlr *ctrlr,
+		struct spdk_nvme_qpair *qpair);
+
+static inline struct nvme_rdma_qpair *
+nvme_rdma_qpair(struct spdk_nvme_qpair *qpair)
+{
+	assert(qpair->trtype == SPDK_NVME_TRANSPORT_RDMA);
+	return SPDK_CONTAINEROF(qpair, struct nvme_rdma_qpair, qpair);
+}
+
+static inline struct nvme_rdma_poll_group *
+nvme_rdma_poll_group(struct spdk_nvme_transport_poll_group *group)
+{
+	return (SPDK_CONTAINEROF(group, struct nvme_rdma_poll_group, group));
+}
+
+static inline struct nvme_rdma_ctrlr *
+nvme_rdma_ctrlr(struct spdk_nvme_ctrlr *ctrlr)
+{
+	assert(ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_RDMA);
+	return SPDK_CONTAINEROF(ctrlr, struct nvme_rdma_ctrlr, ctrlr);
+}
+
+static struct spdk_nvme_rdma_req *
+nvme_rdma_req_get(struct nvme_rdma_qpair *rqpair)
+{
+	struct spdk_nvme_rdma_req *rdma_req;
+
+	rdma_req = TAILQ_FIRST(&rqpair->free_reqs);
+	if (rdma_req) {
+		TAILQ_REMOVE(&rqpair->free_reqs, rdma_req, link);
+		TAILQ_INSERT_TAIL(&rqpair->outstanding_reqs, rdma_req, link);
+	}
+
+	return rdma_req;
+}
+
+static void
+nvme_rdma_req_put(struct nvme_rdma_qpair *rqpair, struct spdk_nvme_rdma_req *rdma_req)
+{
+	rdma_req->completion_flags = 0;
+	rdma_req->req = NULL;
+	TAILQ_INSERT_HEAD(&rqpair->free_reqs, rdma_req, link);
+}
+
+static void
+nvme_rdma_req_complete(struct spdk_nvme_rdma_req *rdma_req,
+		       struct spdk_nvme_cpl *rsp)
+{
+	struct nvme_request *req = rdma_req->req;
+	struct nvme_rdma_qpair *rqpair;
+
+	assert(req != NULL);
+
+	rqpair = nvme_rdma_qpair(req->qpair);
+	TAILQ_REMOVE(&rqpair->outstanding_reqs, rdma_req, link);
+
+	nvme_complete_request(req->cb_fn, req->cb_arg, req->qpair, req, rsp);
+	nvme_free_request(req);
+}
+
+static const char *
+nvme_rdma_cm_event_str_get(uint32_t event)
+{
+	if (event < SPDK_COUNTOF(rdma_cm_event_str)) {
+		return rdma_cm_event_str[event];
+	} else {
+		return "Undefined";
+	}
+}
+
+
+static int
+nvme_rdma_qpair_process_cm_event(struct nvme_rdma_qpair *rqpair)
+{
+	struct rdma_cm_event				*event = rqpair->evt;
+	struct spdk_nvmf_rdma_accept_private_data	*accept_data;
+	int						rc = 0;
+
+	if (event) {
+		switch (event->event) {
+		case RDMA_CM_EVENT_ADDR_RESOLVED:
+		case RDMA_CM_EVENT_ADDR_ERROR:
+		case RDMA_CM_EVENT_ROUTE_RESOLVED:
+		case RDMA_CM_EVENT_ROUTE_ERROR:
+			break;
+		case RDMA_CM_EVENT_CONNECT_REQUEST:
+			break;
+		case RDMA_CM_EVENT_CONNECT_ERROR:
+			break;
+		case RDMA_CM_EVENT_UNREACHABLE:
+		case RDMA_CM_EVENT_REJECTED:
+			break;
+		case RDMA_CM_EVENT_CONNECT_RESPONSE:
+			rc = spdk_rdma_qp_complete_connect(rqpair->rdma_qp);
+		/* fall through */
+		case RDMA_CM_EVENT_ESTABLISHED:
+			accept_data = (struct spdk_nvmf_rdma_accept_private_data *)event->param.conn.private_data;
+			if (accept_data == NULL) {
+				rc = -1;
+			} else {
+				SPDK_DEBUGLOG(SPDK_LOG_NVME, "Requested queue depth %d. Actually got queue depth %d.\n",
+					      rqpair->num_entries, accept_data->crqsize);
+				rqpair->num_entries = spdk_min(rqpair->num_entries, accept_data->crqsize);
+			}
+			break;
+		case RDMA_CM_EVENT_DISCONNECTED:
+			rqpair->qpair.transport_failure_reason = SPDK_NVME_QPAIR_FAILURE_REMOTE;
+			break;
+		case RDMA_CM_EVENT_DEVICE_REMOVAL:
+			rqpair->qpair.transport_failure_reason = SPDK_NVME_QPAIR_FAILURE_LOCAL;
+			break;
+		case RDMA_CM_EVENT_MULTICAST_JOIN:
+		case RDMA_CM_EVENT_MULTICAST_ERROR:
+			break;
+		case RDMA_CM_EVENT_ADDR_CHANGE:
+			rqpair->qpair.transport_failure_reason = SPDK_NVME_QPAIR_FAILURE_LOCAL;
+			break;
+		case RDMA_CM_EVENT_TIMEWAIT_EXIT:
+			break;
+		default:
+			SPDK_ERRLOG("Unexpected Acceptor Event [%d]\n", event->event);
+			break;
+		}
+		rqpair->evt = NULL;
+		rdma_ack_cm_event(event);
+	}
+
+	return rc;
+}
+
+/*
+ * This function must be called under the nvme controller's lock
+ * because it touches global controller variables. The lock is taken
+ * by the generic transport code before invoking a few of the functions
+ * in this file: nvme_rdma_ctrlr_connect_qpair, nvme_rdma_ctrlr_delete_io_qpair,
+ * and conditionally nvme_rdma_qpair_process_completions when it is calling
+ * completions on the admin qpair. When adding a new call to this function, please
+ * verify that it is in a situation where it falls under the lock.
+ */
+static int
+nvme_rdma_poll_events(struct nvme_rdma_ctrlr *rctrlr)
+{
+	struct nvme_rdma_cm_event_entry	*entry, *tmp;
+	struct nvme_rdma_qpair		*event_qpair;
+	struct rdma_cm_event		*event;
+	struct rdma_event_channel	*channel = rctrlr->cm_channel;
+
+	STAILQ_FOREACH_SAFE(entry, &rctrlr->pending_cm_events, link, tmp) {
+		event_qpair = nvme_rdma_qpair(entry->evt->id->context);
+		if (event_qpair->evt == NULL) {
+			event_qpair->evt = entry->evt;
+			STAILQ_REMOVE(&rctrlr->pending_cm_events, entry, nvme_rdma_cm_event_entry, link);
+			STAILQ_INSERT_HEAD(&rctrlr->free_cm_events, entry, link);
+		}
+	}
+
+	while (rdma_get_cm_event(channel, &event) == 0) {
+		event_qpair = nvme_rdma_qpair(event->id->context);
+		if (event_qpair->evt == NULL) {
+			event_qpair->evt = event;
+		} else {
+			assert(rctrlr == nvme_rdma_ctrlr(event_qpair->qpair.ctrlr));
+			entry = STAILQ_FIRST(&rctrlr->free_cm_events);
+			if (entry == NULL) {
+				rdma_ack_cm_event(event);
+				return -ENOMEM;
+			}
+			STAILQ_REMOVE(&rctrlr->free_cm_events, entry, nvme_rdma_cm_event_entry, link);
+			entry->evt = event;
+			STAILQ_INSERT_TAIL(&rctrlr->pending_cm_events, entry, link);
+		}
+	}
+
+	if (errno == EAGAIN || errno == EWOULDBLOCK) {
+		return 0;
+	} else {
+		return errno;
+	}
+}
+
+static int
+nvme_rdma_validate_cm_event(enum rdma_cm_event_type expected_evt_type,
+			    struct rdma_cm_event *reaped_evt)
+{
+	int rc = -EBADMSG;
+
+	if (expected_evt_type == reaped_evt->event) {
+		return 0;
+	}
+
+	switch (expected_evt_type) {
+	case RDMA_CM_EVENT_ESTABLISHED:
+		/*
+		 * There is an enum ib_cm_rej_reason in the kernel headers that sets 10 as
+		 * IB_CM_REJ_STALE_CONN. I can't find the corresponding userspace but we get
+		 * the same values here.
+		 */
+		if (reaped_evt->event == RDMA_CM_EVENT_REJECTED && reaped_evt->status == 10) {
+			rc = -ESTALE;
+		} else if (reaped_evt->event == RDMA_CM_EVENT_CONNECT_RESPONSE) {
+			/*
+			 *  If we are using a qpair which is not created using rdma cm API
+			 *  then we will receive RDMA_CM_EVENT_CONNECT_RESPONSE instead of
+			 *  RDMA_CM_EVENT_ESTABLISHED.
+			 */
+			return 0;
+		}
+		break;
+	default:
+		break;
+	}
+
+	SPDK_ERRLOG("Expected %s but received %s (%d) from CM event channel (status = %d)\n",
+		    nvme_rdma_cm_event_str_get(expected_evt_type),
+		    nvme_rdma_cm_event_str_get(reaped_evt->event), reaped_evt->event,
+		    reaped_evt->status);
+	return rc;
+}
+
+static int
+nvme_rdma_process_event(struct nvme_rdma_qpair *rqpair,
+			struct rdma_event_channel *channel,
+			enum rdma_cm_event_type evt)
+{
+	struct nvme_rdma_ctrlr	*rctrlr;
+	uint64_t timeout_ticks;
+	int	rc = 0, rc2;
+
+	if (rqpair->evt != NULL) {
+		rc = nvme_rdma_qpair_process_cm_event(rqpair);
+		if (rc) {
+			return rc;
+		}
+	}
+
+	timeout_ticks = (NVME_RDMA_QPAIR_CM_EVENT_TIMEOUT_US * spdk_get_ticks_hz()) / SPDK_SEC_TO_USEC +
+			spdk_get_ticks();
+	rctrlr = nvme_rdma_ctrlr(rqpair->qpair.ctrlr);
+	assert(rctrlr != NULL);
+
+	while (!rqpair->evt && spdk_get_ticks() < timeout_ticks && rc == 0) {
+		rc = nvme_rdma_poll_events(rctrlr);
+	}
+
+	if (rc) {
+		return rc;
+	}
+
+	if (rqpair->evt == NULL) {
+		return -EADDRNOTAVAIL;
+	}
+
+	rc = nvme_rdma_validate_cm_event(evt, rqpair->evt);
+
+	rc2 = nvme_rdma_qpair_process_cm_event(rqpair);
+	/* bad message takes precedence over the other error codes from processing the event. */
+	return rc == 0 ? rc2 : rc;
+}
+
+static int
+nvme_rdma_qpair_init(struct nvme_rdma_qpair *rqpair)
+{
+	int			rc;
+	struct spdk_rdma_qp_init_attr	attr = {};
+	struct ibv_device_attr	dev_attr;
+	struct nvme_rdma_ctrlr	*rctrlr;
+
+	rc = ibv_query_device(rqpair->cm_id->verbs, &dev_attr);
+	if (rc != 0) {
+		SPDK_ERRLOG("Failed to query RDMA device attributes.\n");
+		return -1;
+	}
+
+	if (rqpair->qpair.poll_group) {
+		assert(!rqpair->cq);
+		rc = nvme_poll_group_connect_qpair(&rqpair->qpair);
+		if (rc) {
+			SPDK_ERRLOG("Unable to activate the rdmaqpair.\n");
+			return -1;
+		}
+		assert(rqpair->cq);
+	} else {
+		rqpair->cq = ibv_create_cq(rqpair->cm_id->verbs, rqpair->num_entries * 2, rqpair, NULL, 0);
+		if (!rqpair->cq) {
+			SPDK_ERRLOG("Unable to create completion queue: errno %d: %s\n", errno, spdk_strerror(errno));
+			return -1;
+		}
+	}
+
+	rctrlr = nvme_rdma_ctrlr(rqpair->qpair.ctrlr);
+	if (g_nvme_hooks.get_ibv_pd) {
+		rctrlr->pd = g_nvme_hooks.get_ibv_pd(&rctrlr->ctrlr.trid, rqpair->cm_id->verbs);
+	} else {
+		rctrlr->pd = NULL;
+	}
+
+	attr.pd =		rctrlr->pd;
+	attr.send_cq		= rqpair->cq;
+	attr.recv_cq		= rqpair->cq;
+	attr.cap.max_send_wr	= rqpair->num_entries; /* SEND operations */
+	attr.cap.max_recv_wr	= rqpair->num_entries; /* RECV operations */
+	attr.cap.max_send_sge	= spdk_min(NVME_RDMA_DEFAULT_TX_SGE, dev_attr.max_sge);
+	attr.cap.max_recv_sge	= spdk_min(NVME_RDMA_DEFAULT_RX_SGE, dev_attr.max_sge);
+
+	rqpair->rdma_qp = spdk_rdma_qp_create(rqpair->cm_id, &attr);
+
+	if (!rqpair->rdma_qp) {
+		return -1;
+	}
+
+	/* ibv_create_qp will change the values in attr.cap. Make sure we store the proper value. */
+	rqpair->max_send_sge = spdk_min(NVME_RDMA_DEFAULT_TX_SGE, attr.cap.max_send_sge);
+	rqpair->max_recv_sge = spdk_min(NVME_RDMA_DEFAULT_RX_SGE, attr.cap.max_recv_sge);
+	rqpair->current_num_recvs = 0;
+	rqpair->current_num_sends = 0;
+
+	rctrlr->pd = rqpair->rdma_qp->qp->pd;
+
+	rqpair->cm_id->context = &rqpair->qpair;
+
+	return 0;
+}
+
+static inline int
+nvme_rdma_qpair_submit_sends(struct nvme_rdma_qpair *rqpair)
+{
+	struct ibv_send_wr *bad_send_wr;
+	int rc;
+
+	rc = spdk_rdma_qp_flush_send_wrs(rqpair->rdma_qp, &bad_send_wr);
+
+	if (spdk_unlikely(rc)) {
+		SPDK_ERRLOG("Failed to post WRs on send queue, errno %d (%s), bad_wr %p\n",
+			    rc, spdk_strerror(rc), bad_send_wr);
+		while (bad_send_wr != NULL) {
+			assert(rqpair->current_num_sends > 0);
+			rqpair->current_num_sends--;
+			bad_send_wr = bad_send_wr->next;
+		}
+		return rc;
+	}
+
+	return 0;
+}
+
+static inline int
+nvme_rdma_qpair_submit_recvs(struct nvme_rdma_qpair *rqpair)
+{
+	struct ibv_recv_wr *bad_recv_wr;
+	int rc = 0;
+
+	if (rqpair->recvs_to_post.first) {
+		rc = ibv_post_recv(rqpair->rdma_qp->qp, rqpair->recvs_to_post.first, &bad_recv_wr);
+		if (spdk_unlikely(rc)) {
+			SPDK_ERRLOG("Failed to post WRs on receive queue, errno %d (%s), bad_wr %p\n",
+				    rc, spdk_strerror(rc), bad_recv_wr);
+			while (bad_recv_wr != NULL) {
+				assert(rqpair->current_num_sends > 0);
+				rqpair->current_num_recvs--;
+				bad_recv_wr = bad_recv_wr->next;
+			}
+		}
+
+		rqpair->recvs_to_post.first = NULL;
+	}
+	return rc;
+}
+
+/* Append the given send wr structure to the qpair's outstanding sends list. */
+/* This function accepts only a single wr. */
+static inline int
+nvme_rdma_qpair_queue_send_wr(struct nvme_rdma_qpair *rqpair, struct ibv_send_wr *wr)
+{
+	assert(wr->next == NULL);
+
+	assert(rqpair->current_num_sends < rqpair->num_entries);
+
+	rqpair->current_num_sends++;
+	spdk_rdma_qp_queue_send_wrs(rqpair->rdma_qp, wr);
+
+	if (!rqpair->delay_cmd_submit) {
+		return nvme_rdma_qpair_submit_sends(rqpair);
+	}
+
+	return 0;
+}
+
+/* Append the given recv wr structure to the qpair's outstanding recvs list. */
+/* This function accepts only a single wr. */
+static inline int
+nvme_rdma_qpair_queue_recv_wr(struct nvme_rdma_qpair *rqpair, struct ibv_recv_wr *wr)
+{
+
+	assert(wr->next == NULL);
+	assert(rqpair->current_num_recvs < rqpair->num_entries);
+
+	rqpair->current_num_recvs++;
+	if (rqpair->recvs_to_post.first == NULL) {
+		rqpair->recvs_to_post.first = wr;
+	} else {
+		rqpair->recvs_to_post.last->next = wr;
+	}
+
+	rqpair->recvs_to_post.last = wr;
+
+	if (!rqpair->delay_cmd_submit) {
+		return nvme_rdma_qpair_submit_recvs(rqpair);
+	}
+
+	return 0;
+}
+
+#define nvme_rdma_trace_ibv_sge(sg_list) \
+	if (sg_list) { \
+		SPDK_DEBUGLOG(SPDK_LOG_NVME, "local addr %p length 0x%x lkey 0x%x\n", \
+			      (void *)(sg_list)->addr, (sg_list)->length, (sg_list)->lkey); \
+	}
+
+static int
+nvme_rdma_post_recv(struct nvme_rdma_qpair *rqpair, uint16_t rsp_idx)
+{
+	struct ibv_recv_wr *wr;
+
+	wr = &rqpair->rsp_recv_wrs[rsp_idx];
+	wr->next = NULL;
+	nvme_rdma_trace_ibv_sge(wr->sg_list);
+	return nvme_rdma_qpair_queue_recv_wr(rqpair, wr);
+}
+
+static int
+nvme_rdma_reg_mr(struct rdma_cm_id *cm_id, union nvme_rdma_mr *mr, void *mem, size_t length)
+{
+	if (!g_nvme_hooks.get_rkey) {
+		mr->mr = rdma_reg_msgs(cm_id, mem, length);
+		if (mr->mr == NULL) {
+			SPDK_ERRLOG("Unable to register mr: %s (%d)\n",
+				    spdk_strerror(errno), errno);
+			return -1;
+		}
+	} else {
+		mr->key = g_nvme_hooks.get_rkey(cm_id->pd, mem, length);
+	}
+
+	return 0;
+}
+
+static void
+nvme_rdma_dereg_mr(union nvme_rdma_mr *mr)
+{
+	if (!g_nvme_hooks.get_rkey) {
+		if (mr->mr && rdma_dereg_mr(mr->mr)) {
+			SPDK_ERRLOG("Unable to de-register mr\n");
+		}
+	} else {
+		if (mr->key) {
+			g_nvme_hooks.put_rkey(mr->key);
+		}
+	}
+	memset(mr, 0, sizeof(*mr));
+}
+
+static uint32_t
+nvme_rdma_mr_get_lkey(union nvme_rdma_mr *mr)
+{
+	uint32_t lkey;
+
+	if (!g_nvme_hooks.get_rkey) {
+		lkey = mr->mr->lkey;
+	} else {
+		lkey = *((uint64_t *) mr->key);
+	}
+
+	return lkey;
+}
+
+static void
+nvme_rdma_unregister_rsps(struct nvme_rdma_qpair *rqpair)
+{
+	nvme_rdma_dereg_mr(&rqpair->rsp_mr);
+}
+
+static void
+nvme_rdma_free_rsps(struct nvme_rdma_qpair *rqpair)
+{
+	nvme_rdma_free(rqpair->rsps);
+	rqpair->rsps = NULL;
+	nvme_rdma_free(rqpair->rsp_sgls);
+	rqpair->rsp_sgls = NULL;
+	nvme_rdma_free(rqpair->rsp_recv_wrs);
+	rqpair->rsp_recv_wrs = NULL;
+}
+
+static int
+nvme_rdma_alloc_rsps(struct nvme_rdma_qpair *rqpair)
+{
+	rqpair->rsps = NULL;
+	rqpair->rsp_recv_wrs = NULL;
+
+	rqpair->rsp_sgls = nvme_rdma_calloc(rqpair->num_entries, sizeof(*rqpair->rsp_sgls));
+	if (!rqpair->rsp_sgls) {
+		SPDK_ERRLOG("Failed to allocate rsp_sgls\n");
+		goto fail;
+	}
+
+	rqpair->rsp_recv_wrs = nvme_rdma_calloc(rqpair->num_entries, sizeof(*rqpair->rsp_recv_wrs));
+	if (!rqpair->rsp_recv_wrs) {
+		SPDK_ERRLOG("Failed to allocate rsp_recv_wrs\n");
+		goto fail;
+	}
+
+	rqpair->rsps = nvme_rdma_calloc(rqpair->num_entries, sizeof(*rqpair->rsps));
+	if (!rqpair->rsps) {
+		SPDK_ERRLOG("can not allocate rdma rsps\n");
+		goto fail;
+	}
+
+	return 0;
+fail:
+	nvme_rdma_free_rsps(rqpair);
+	return -ENOMEM;
+}
+
+static int
+nvme_rdma_register_rsps(struct nvme_rdma_qpair *rqpair)
+{
+	uint16_t i;
+	int rc;
+	uint32_t lkey;
+
+	rc = nvme_rdma_reg_mr(rqpair->cm_id, &rqpair->rsp_mr,
+			      rqpair->rsps, rqpair->num_entries * sizeof(*rqpair->rsps));
+
+	if (rc < 0) {
+		goto fail;
+	}
+
+	lkey = nvme_rdma_mr_get_lkey(&rqpair->rsp_mr);
+
+	for (i = 0; i < rqpair->num_entries; i++) {
+		struct ibv_sge *rsp_sgl = &rqpair->rsp_sgls[i];
+		struct spdk_nvme_rdma_rsp *rsp = &rqpair->rsps[i];
+
+		rsp->rqpair = rqpair;
+		rsp->rdma_wr.type = RDMA_WR_TYPE_RECV;
+		rsp->idx = i;
+		rsp_sgl->addr = (uint64_t)&rqpair->rsps[i];
+		rsp_sgl->length = sizeof(struct spdk_nvme_cpl);
+		rsp_sgl->lkey = lkey;
+
+		rqpair->rsp_recv_wrs[i].wr_id = (uint64_t)&rsp->rdma_wr;
+		rqpair->rsp_recv_wrs[i].next = NULL;
+		rqpair->rsp_recv_wrs[i].sg_list = rsp_sgl;
+		rqpair->rsp_recv_wrs[i].num_sge = 1;
+
+		rc = nvme_rdma_post_recv(rqpair, i);
+		if (rc) {
+			goto fail;
+		}
+	}
+
+	rc = nvme_rdma_qpair_submit_recvs(rqpair);
+	if (rc) {
+		goto fail;
+	}
+
+	return 0;
+
+fail:
+	nvme_rdma_unregister_rsps(rqpair);
+	return rc;
+}
+
+static void
+nvme_rdma_unregister_reqs(struct nvme_rdma_qpair *rqpair)
+{
+	nvme_rdma_dereg_mr(&rqpair->cmd_mr);
+}
+
+static void
+nvme_rdma_free_reqs(struct nvme_rdma_qpair *rqpair)
+{
+	if (!rqpair->rdma_reqs) {
+		return;
+	}
+
+	nvme_rdma_free(rqpair->cmds);
+	rqpair->cmds = NULL;
+
+	nvme_rdma_free(rqpair->rdma_reqs);
+	rqpair->rdma_reqs = NULL;
+}
+
+static int
+nvme_rdma_alloc_reqs(struct nvme_rdma_qpair *rqpair)
+{
+	uint16_t i;
+
+	rqpair->rdma_reqs = nvme_rdma_calloc(rqpair->num_entries, sizeof(struct spdk_nvme_rdma_req));
+	if (rqpair->rdma_reqs == NULL) {
+		SPDK_ERRLOG("Failed to allocate rdma_reqs\n");
+		goto fail;
+	}
+
+	rqpair->cmds = nvme_rdma_calloc(rqpair->num_entries, sizeof(*rqpair->cmds));
+	if (!rqpair->cmds) {
+		SPDK_ERRLOG("Failed to allocate RDMA cmds\n");
+		goto fail;
+	}
+
+
+	TAILQ_INIT(&rqpair->free_reqs);
+	TAILQ_INIT(&rqpair->outstanding_reqs);
+	for (i = 0; i < rqpair->num_entries; i++) {
+		struct spdk_nvme_rdma_req	*rdma_req;
+		struct spdk_nvmf_cmd		*cmd;
+
+		rdma_req = &rqpair->rdma_reqs[i];
+		rdma_req->rdma_wr.type = RDMA_WR_TYPE_SEND;
+		cmd = &rqpair->cmds[i];
+
+		rdma_req->id = i;
+
+		/* The first RDMA sgl element will always point
+		 * at this data structure. Depending on whether
+		 * an NVMe-oF SGL is required, the length of
+		 * this element may change. */
+		rdma_req->send_sgl[0].addr = (uint64_t)cmd;
+		rdma_req->send_wr.wr_id = (uint64_t)&rdma_req->rdma_wr;
+		rdma_req->send_wr.next = NULL;
+		rdma_req->send_wr.opcode = IBV_WR_SEND;
+		rdma_req->send_wr.send_flags = IBV_SEND_SIGNALED;
+		rdma_req->send_wr.sg_list = rdma_req->send_sgl;
+		rdma_req->send_wr.imm_data = 0;
+
+		TAILQ_INSERT_TAIL(&rqpair->free_reqs, rdma_req, link);
+	}
+
+	return 0;
+fail:
+	nvme_rdma_free_reqs(rqpair);
+	return -ENOMEM;
+}
+
+static int
+nvme_rdma_register_reqs(struct nvme_rdma_qpair *rqpair)
+{
+	int i;
+	int rc;
+	uint32_t lkey;
+
+	rc = nvme_rdma_reg_mr(rqpair->cm_id, &rqpair->cmd_mr,
+			      rqpair->cmds, rqpair->num_entries * sizeof(*rqpair->cmds));
+
+	if (rc < 0) {
+		goto fail;
+	}
+
+	lkey = nvme_rdma_mr_get_lkey(&rqpair->cmd_mr);
+
+	for (i = 0; i < rqpair->num_entries; i++) {
+		rqpair->rdma_reqs[i].send_sgl[0].lkey = lkey;
+	}
+
+	return 0;
+
+fail:
+	nvme_rdma_unregister_reqs(rqpair);
+	return -ENOMEM;
+}
+
+static int
+nvme_rdma_resolve_addr(struct nvme_rdma_qpair *rqpair,
+		       struct sockaddr *src_addr,
+		       struct sockaddr *dst_addr,
+		       struct rdma_event_channel *cm_channel)
+{
+	int ret;
+
+	ret = rdma_resolve_addr(rqpair->cm_id, src_addr, dst_addr,
+				NVME_RDMA_TIME_OUT_IN_MS);
+	if (ret) {
+		SPDK_ERRLOG("rdma_resolve_addr, %d\n", errno);
+		return ret;
+	}
+
+	ret = nvme_rdma_process_event(rqpair, cm_channel, RDMA_CM_EVENT_ADDR_RESOLVED);
+	if (ret) {
+		SPDK_ERRLOG("RDMA address resolution error\n");
+		return -1;
+	}
+
+	if (rqpair->qpair.ctrlr->opts.transport_ack_timeout != SPDK_NVME_TRANSPORT_ACK_TIMEOUT_DISABLED) {
+#ifdef SPDK_CONFIG_RDMA_SET_ACK_TIMEOUT
+		uint8_t timeout = rqpair->qpair.ctrlr->opts.transport_ack_timeout;
+		ret = rdma_set_option(rqpair->cm_id, RDMA_OPTION_ID,
+				      RDMA_OPTION_ID_ACK_TIMEOUT,
+				      &timeout, sizeof(timeout));
+		if (ret) {
+			SPDK_NOTICELOG("Can't apply RDMA_OPTION_ID_ACK_TIMEOUT %d, ret %d\n", timeout, ret);
+		}
+#else
+		SPDK_DEBUGLOG(SPDK_LOG_NVME, "transport_ack_timeout is not supported\n");
+#endif
+	}
+
+
+	ret = rdma_resolve_route(rqpair->cm_id, NVME_RDMA_TIME_OUT_IN_MS);
+	if (ret) {
+		SPDK_ERRLOG("rdma_resolve_route\n");
+		return ret;
+	}
+
+	ret = nvme_rdma_process_event(rqpair, cm_channel, RDMA_CM_EVENT_ROUTE_RESOLVED);
+	if (ret) {
+		SPDK_ERRLOG("RDMA route resolution error\n");
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+nvme_rdma_connect(struct nvme_rdma_qpair *rqpair)
+{
+	struct rdma_conn_param				param = {};
+	struct spdk_nvmf_rdma_request_private_data	request_data = {};
+	struct ibv_device_attr				attr;
+	int						ret;
+	struct spdk_nvme_ctrlr				*ctrlr;
+	struct nvme_rdma_ctrlr				*rctrlr;
+
+	ret = ibv_query_device(rqpair->cm_id->verbs, &attr);
+	if (ret != 0) {
+		SPDK_ERRLOG("Failed to query RDMA device attributes.\n");
+		return ret;
+	}
+
+	param.responder_resources = spdk_min(rqpair->num_entries, attr.max_qp_rd_atom);
+
+	ctrlr = rqpair->qpair.ctrlr;
+	if (!ctrlr) {
+		return -1;
+	}
+	rctrlr = nvme_rdma_ctrlr(ctrlr);
+	assert(rctrlr != NULL);
+
+	request_data.qid = rqpair->qpair.id;
+	request_data.hrqsize = rqpair->num_entries;
+	request_data.hsqsize = rqpair->num_entries - 1;
+	request_data.cntlid = ctrlr->cntlid;
+
+	param.private_data = &request_data;
+	param.private_data_len = sizeof(request_data);
+	param.retry_count = ctrlr->opts.transport_retry_count;
+	param.rnr_retry_count = 7;
+
+	/* Fields below are ignored by rdma cm if qpair has been
+	 * created using rdma cm API. */
+	param.srq = 0;
+	param.qp_num = rqpair->rdma_qp->qp->qp_num;
+
+	ret = rdma_connect(rqpair->cm_id, &param);
+	if (ret) {
+		SPDK_ERRLOG("nvme rdma connect error\n");
+		return ret;
+	}
+
+	ret = nvme_rdma_process_event(rqpair, rctrlr->cm_channel, RDMA_CM_EVENT_ESTABLISHED);
+	if (ret == -ESTALE) {
+		SPDK_NOTICELOG("Received a stale connection notice during connection.\n");
+		return -EAGAIN;
+	} else if (ret) {
+		SPDK_ERRLOG("RDMA connect error %d\n", ret);
+		return ret;
+	} else {
+		return 0;
+	}
+}
+
+static int
+nvme_rdma_parse_addr(struct sockaddr_storage *sa, int family, const char *addr, const char *service)
+{
+	struct addrinfo *res;
+	struct addrinfo hints;
+	int ret;
+
+	memset(&hints, 0, sizeof(hints));
+	hints.ai_family = family;
+	hints.ai_socktype = SOCK_STREAM;
+	hints.ai_protocol = 0;
+
+	ret = getaddrinfo(addr, service, &hints, &res);
+	if (ret) {
+		SPDK_ERRLOG("getaddrinfo failed: %s (%d)\n", gai_strerror(ret), ret);
+		return ret;
+	}
+
+	if (res->ai_addrlen > sizeof(*sa)) {
+		SPDK_ERRLOG("getaddrinfo() ai_addrlen %zu too large\n", (size_t)res->ai_addrlen);
+		ret = EINVAL;
+	} else {
+		memcpy(sa, res->ai_addr, res->ai_addrlen);
+	}
+
+	freeaddrinfo(res);
+	return ret;
+}
+
+static int
+nvme_rdma_mr_map_notify(void *cb_ctx, struct spdk_mem_map *map,
+			enum spdk_mem_map_notify_action action,
+			void *vaddr, size_t size)
+{
+	struct ibv_pd *pd = cb_ctx;
+	struct ibv_mr *mr;
+	int rc;
+
+	switch (action) {
+	case SPDK_MEM_MAP_NOTIFY_REGISTER:
+		if (!g_nvme_hooks.get_rkey) {
+			mr = ibv_reg_mr(pd, vaddr, size,
+					IBV_ACCESS_LOCAL_WRITE |
+					IBV_ACCESS_REMOTE_READ |
+					IBV_ACCESS_REMOTE_WRITE);
+			if (mr == NULL) {
+				SPDK_ERRLOG("ibv_reg_mr() failed\n");
+				return -EFAULT;
+			} else {
+				rc = spdk_mem_map_set_translation(map, (uint64_t)vaddr, size, (uint64_t)mr);
+			}
+		} else {
+			rc = spdk_mem_map_set_translation(map, (uint64_t)vaddr, size,
+							  g_nvme_hooks.get_rkey(pd, vaddr, size));
+		}
+		break;
+	case SPDK_MEM_MAP_NOTIFY_UNREGISTER:
+		if (!g_nvme_hooks.get_rkey) {
+			mr = (struct ibv_mr *)spdk_mem_map_translate(map, (uint64_t)vaddr, NULL);
+			if (mr) {
+				ibv_dereg_mr(mr);
+			}
+		}
+		rc = spdk_mem_map_clear_translation(map, (uint64_t)vaddr, size);
+		break;
+	default:
+		SPDK_UNREACHABLE();
+	}
+
+	return rc;
+}
+
+static int
+nvme_rdma_check_contiguous_entries(uint64_t addr_1, uint64_t addr_2)
+{
+	/* Two contiguous mappings will point to the same address which is the start of the RDMA MR. */
+	return addr_1 == addr_2;
+}
+
+static int
+nvme_rdma_register_mem(struct nvme_rdma_qpair *rqpair)
+{
+	struct ibv_pd *pd = rqpair->rdma_qp->qp->pd;
+	struct spdk_nvme_rdma_mr_map *mr_map;
+	const struct spdk_mem_map_ops nvme_rdma_map_ops = {
+		.notify_cb = nvme_rdma_mr_map_notify,
+		.are_contiguous = nvme_rdma_check_contiguous_entries
+	};
+
+	pthread_mutex_lock(&g_rdma_mr_maps_mutex);
+
+	/* Look up existing mem map registration for this pd */
+	LIST_FOREACH(mr_map, &g_rdma_mr_maps, link) {
+		if (mr_map->pd == pd) {
+			mr_map->ref++;
+			rqpair->mr_map = mr_map;
+			pthread_mutex_unlock(&g_rdma_mr_maps_mutex);
+			return 0;
+		}
+	}
+
+	mr_map = nvme_rdma_calloc(1, sizeof(*mr_map));
+	if (mr_map == NULL) {
+		SPDK_ERRLOG("Failed to allocate mr_map\n");
+		pthread_mutex_unlock(&g_rdma_mr_maps_mutex);
+		return -1;
+	}
+
+	mr_map->ref = 1;
+	mr_map->pd = pd;
+	mr_map->map = spdk_mem_map_alloc((uint64_t)NULL, &nvme_rdma_map_ops, pd);
+	if (mr_map->map == NULL) {
+		SPDK_ERRLOG("spdk_mem_map_alloc() failed\n");
+		nvme_rdma_free(mr_map);
+
+		pthread_mutex_unlock(&g_rdma_mr_maps_mutex);
+		return -1;
+	}
+
+	rqpair->mr_map = mr_map;
+	LIST_INSERT_HEAD(&g_rdma_mr_maps, mr_map, link);
+
+	pthread_mutex_unlock(&g_rdma_mr_maps_mutex);
+
+	return 0;
+}
+
+static void
+nvme_rdma_unregister_mem(struct nvme_rdma_qpair *rqpair)
+{
+	struct spdk_nvme_rdma_mr_map *mr_map;
+
+	mr_map = rqpair->mr_map;
+	rqpair->mr_map = NULL;
+
+	if (mr_map == NULL) {
+		return;
+	}
+
+	pthread_mutex_lock(&g_rdma_mr_maps_mutex);
+
+	assert(mr_map->ref > 0);
+	mr_map->ref--;
+	if (mr_map->ref == 0) {
+		LIST_REMOVE(mr_map, link);
+		spdk_mem_map_free(&mr_map->map);
+		nvme_rdma_free(mr_map);
+	}
+
+	pthread_mutex_unlock(&g_rdma_mr_maps_mutex);
+}
+
+static int
+_nvme_rdma_ctrlr_connect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)
+{
+	struct sockaddr_storage dst_addr;
+	struct sockaddr_storage src_addr;
+	bool src_addr_specified;
+	int rc;
+	struct nvme_rdma_ctrlr *rctrlr;
+	struct nvme_rdma_qpair *rqpair;
+	int family;
+
+	rqpair = nvme_rdma_qpair(qpair);
+	rctrlr = nvme_rdma_ctrlr(ctrlr);
+	assert(rctrlr != NULL);
+
+	switch (ctrlr->trid.adrfam) {
+	case SPDK_NVMF_ADRFAM_IPV4:
+		family = AF_INET;
+		break;
+	case SPDK_NVMF_ADRFAM_IPV6:
+		family = AF_INET6;
+		break;
+	default:
+		SPDK_ERRLOG("Unhandled ADRFAM %d\n", ctrlr->trid.adrfam);
+		return -1;
+	}
+
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "adrfam %d ai_family %d\n", ctrlr->trid.adrfam, family);
+
+	memset(&dst_addr, 0, sizeof(dst_addr));
+
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "trsvcid is %s\n", ctrlr->trid.trsvcid);
+	rc = nvme_rdma_parse_addr(&dst_addr, family, ctrlr->trid.traddr, ctrlr->trid.trsvcid);
+	if (rc != 0) {
+		SPDK_ERRLOG("dst_addr nvme_rdma_parse_addr() failed\n");
+		return -1;
+	}
+
+	if (ctrlr->opts.src_addr[0] || ctrlr->opts.src_svcid[0]) {
+		memset(&src_addr, 0, sizeof(src_addr));
+		rc = nvme_rdma_parse_addr(&src_addr, family, ctrlr->opts.src_addr, ctrlr->opts.src_svcid);
+		if (rc != 0) {
+			SPDK_ERRLOG("src_addr nvme_rdma_parse_addr() failed\n");
+			return -1;
+		}
+		src_addr_specified = true;
+	} else {
+		src_addr_specified = false;
+	}
+
+	rc = rdma_create_id(rctrlr->cm_channel, &rqpair->cm_id, rqpair, RDMA_PS_TCP);
+	if (rc < 0) {
+		SPDK_ERRLOG("rdma_create_id() failed\n");
+		return -1;
+	}
+
+	rc = nvme_rdma_resolve_addr(rqpair,
+				    src_addr_specified ? (struct sockaddr *)&src_addr : NULL,
+				    (struct sockaddr *)&dst_addr, rctrlr->cm_channel);
+	if (rc < 0) {
+		SPDK_ERRLOG("nvme_rdma_resolve_addr() failed\n");
+		return -1;
+	}
+
+	rc = nvme_rdma_qpair_init(rqpair);
+	if (rc < 0) {
+		SPDK_ERRLOG("nvme_rdma_qpair_init() failed\n");
+		return -1;
+	}
+
+	rc = nvme_rdma_connect(rqpair);
+	if (rc != 0) {
+		SPDK_ERRLOG("Unable to connect the rqpair\n");
+		return rc;
+	}
+
+	rc = nvme_rdma_register_reqs(rqpair);
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "rc =%d\n", rc);
+	if (rc) {
+		SPDK_ERRLOG("Unable to register rqpair RDMA requests\n");
+		return -1;
+	}
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "RDMA requests registered\n");
+
+	rc = nvme_rdma_register_rsps(rqpair);
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "rc =%d\n", rc);
+	if (rc < 0) {
+		SPDK_ERRLOG("Unable to register rqpair RDMA responses\n");
+		return -1;
+	}
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "RDMA responses registered\n");
+
+	rc = nvme_rdma_register_mem(rqpair);
+	if (rc < 0) {
+		SPDK_ERRLOG("Unable to register memory for RDMA\n");
+		return -1;
+	}
+
+	rc = nvme_fabric_qpair_connect(&rqpair->qpair, rqpair->num_entries);
+	if (rc < 0) {
+		rqpair->qpair.transport_failure_reason = SPDK_NVME_QPAIR_FAILURE_UNKNOWN;
+		SPDK_ERRLOG("Failed to send an NVMe-oF Fabric CONNECT command\n");
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+nvme_rdma_ctrlr_connect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)
+{
+	int rc;
+	int retry_count = 0;
+
+	rc = _nvme_rdma_ctrlr_connect_qpair(ctrlr, qpair);
+
+	/*
+	 * -EAGAIN represents the special case where the target side still thought it was connected.
+	 * Most NICs will fail the first connection attempt, and the NICs will clean up whatever
+	 * state they need to. After that, subsequent connection attempts will succeed.
+	 */
+	if (rc == -EAGAIN) {
+		SPDK_NOTICELOG("Detected stale connection on Target side for qpid: %d\n", qpair->id);
+		do {
+			nvme_delay(NVME_RDMA_STALE_CONN_RETRY_DELAY_US);
+			nvme_transport_ctrlr_disconnect_qpair(ctrlr, qpair);
+			rc = _nvme_rdma_ctrlr_connect_qpair(ctrlr, qpair);
+			retry_count++;
+		} while (rc == -EAGAIN && retry_count < NVME_RDMA_STALE_CONN_RETRY_MAX);
+	}
+
+	return rc;
+}
+
+/*
+ * Build SGL describing empty payload.
+ */
+static int
+nvme_rdma_build_null_request(struct spdk_nvme_rdma_req *rdma_req)
+{
+	struct nvme_request *req = rdma_req->req;
+
+	req->cmd.psdt = SPDK_NVME_PSDT_SGL_MPTR_CONTIG;
+
+	/* The first element of this SGL is pointing at an
+	 * spdk_nvmf_cmd object. For this particular command,
+	 * we only need the first 64 bytes corresponding to
+	 * the NVMe command. */
+	rdma_req->send_sgl[0].length = sizeof(struct spdk_nvme_cmd);
+
+	/* The RDMA SGL needs one element describing the NVMe command. */
+	rdma_req->send_wr.num_sge = 1;
+
+	req->cmd.dptr.sgl1.keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK;
+	req->cmd.dptr.sgl1.keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS;
+	req->cmd.dptr.sgl1.keyed.length = 0;
+	req->cmd.dptr.sgl1.keyed.key = 0;
+	req->cmd.dptr.sgl1.address = 0;
+
+	return 0;
+}
+
+static inline bool
+nvme_rdma_get_key(struct spdk_mem_map *map, void *payload, uint64_t size,
+		  enum nvme_rdma_key_type key_type, uint32_t *key)
+{
+	struct ibv_mr *mr;
+	uint64_t real_size = size;
+	uint32_t _key = 0;
+
+	if (!g_nvme_hooks.get_rkey) {
+		mr = (struct ibv_mr *)spdk_mem_map_translate(map, (uint64_t)payload, &real_size);
+
+		if (spdk_unlikely(!mr)) {
+			SPDK_ERRLOG("No translation for ptr %p, size %lu\n", payload, size);
+			return false;
+		}
+		switch (key_type) {
+		case NVME_RDMA_MR_RKEY:
+			_key = mr->rkey;
+			break;
+		case NVME_RDMA_MR_LKEY:
+			_key = mr->lkey;
+			break;
+		default:
+			SPDK_ERRLOG("Invalid key type %d\n", key_type);
+			assert(0);
+			return false;
+		}
+	} else {
+		_key = spdk_mem_map_translate(map, (uint64_t)payload, &real_size);
+	}
+
+	if (spdk_unlikely(real_size < size)) {
+		SPDK_ERRLOG("Data buffer split over multiple RDMA Memory Regions\n");
+		return false;
+	}
+
+	*key = _key;
+	return true;
+}
+
+/*
+ * Build inline SGL describing contiguous payload buffer.
+ */
+static int
+nvme_rdma_build_contig_inline_request(struct nvme_rdma_qpair *rqpair,
+				      struct spdk_nvme_rdma_req *rdma_req)
+{
+	struct nvme_request *req = rdma_req->req;
+	uint32_t lkey = 0;
+	void *payload;
+
+	payload = req->payload.contig_or_cb_arg + req->payload_offset;
+	assert(req->payload_size != 0);
+	assert(nvme_payload_type(&req->payload) == NVME_PAYLOAD_TYPE_CONTIG);
+
+	if (spdk_unlikely(!nvme_rdma_get_key(rqpair->mr_map->map, payload, req->payload_size,
+					     NVME_RDMA_MR_LKEY, &lkey))) {
+		return -1;
+	}
+
+	rdma_req->send_sgl[1].lkey = lkey;
+
+	/* The first element of this SGL is pointing at an
+	 * spdk_nvmf_cmd object. For this particular command,
+	 * we only need the first 64 bytes corresponding to
+	 * the NVMe command. */
+	rdma_req->send_sgl[0].length = sizeof(struct spdk_nvme_cmd);
+
+	rdma_req->send_sgl[1].addr = (uint64_t)payload;
+	rdma_req->send_sgl[1].length = (uint32_t)req->payload_size;
+
+	/* The RDMA SGL contains two elements. The first describes
+	 * the NVMe command and the second describes the data
+	 * payload. */
+	rdma_req->send_wr.num_sge = 2;
+
+	req->cmd.psdt = SPDK_NVME_PSDT_SGL_MPTR_CONTIG;
+	req->cmd.dptr.sgl1.unkeyed.type = SPDK_NVME_SGL_TYPE_DATA_BLOCK;
+	req->cmd.dptr.sgl1.unkeyed.subtype = SPDK_NVME_SGL_SUBTYPE_OFFSET;
+	req->cmd.dptr.sgl1.unkeyed.length = (uint32_t)req->payload_size;
+	/* Inline only supported for icdoff == 0 currently.  This function will
+	 * not get called for controllers with other values. */
+	req->cmd.dptr.sgl1.address = (uint64_t)0;
+
+	return 0;
+}
+
+/*
+ * Build SGL describing contiguous payload buffer.
+ */
+static int
+nvme_rdma_build_contig_request(struct nvme_rdma_qpair *rqpair,
+			       struct spdk_nvme_rdma_req *rdma_req)
+{
+	struct nvme_request *req = rdma_req->req;
+	void *payload = req->payload.contig_or_cb_arg + req->payload_offset;
+	uint32_t rkey = 0;
+
+	assert(req->payload_size != 0);
+	assert(nvme_payload_type(&req->payload) == NVME_PAYLOAD_TYPE_CONTIG);
+
+	if (spdk_unlikely(req->payload_size > NVME_RDMA_MAX_KEYED_SGL_LENGTH)) {
+		SPDK_ERRLOG("SGL length %u exceeds max keyed SGL block size %u\n",
+			    req->payload_size, NVME_RDMA_MAX_KEYED_SGL_LENGTH);
+		return -1;
+	}
+
+	if (spdk_unlikely(!nvme_rdma_get_key(rqpair->mr_map->map, payload, req->payload_size,
+					     NVME_RDMA_MR_RKEY, &rkey))) {
+		return -1;
+	}
+
+	req->cmd.dptr.sgl1.keyed.key = rkey;
+
+	/* The first element of this SGL is pointing at an
+	 * spdk_nvmf_cmd object. For this particular command,
+	 * we only need the first 64 bytes corresponding to
+	 * the NVMe command. */
+	rdma_req->send_sgl[0].length = sizeof(struct spdk_nvme_cmd);
+
+	/* The RDMA SGL needs one element describing the NVMe command. */
+	rdma_req->send_wr.num_sge = 1;
+
+	req->cmd.psdt = SPDK_NVME_PSDT_SGL_MPTR_CONTIG;
+	req->cmd.dptr.sgl1.keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK;
+	req->cmd.dptr.sgl1.keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS;
+	req->cmd.dptr.sgl1.keyed.length = req->payload_size;
+	req->cmd.dptr.sgl1.address = (uint64_t)payload;
+
+	return 0;
+}
+
+/*
+ * Build SGL describing scattered payload buffer.
+ */
+static int
+nvme_rdma_build_sgl_request(struct nvme_rdma_qpair *rqpair,
+			    struct spdk_nvme_rdma_req *rdma_req)
+{
+	struct nvme_request *req = rdma_req->req;
+	struct spdk_nvmf_cmd *cmd = &rqpair->cmds[rdma_req->id];
+	void *virt_addr;
+	uint32_t remaining_size;
+	uint32_t sge_length;
+	int rc, max_num_sgl, num_sgl_desc;
+	uint32_t rkey = 0;
+
+	assert(req->payload_size != 0);
+	assert(nvme_payload_type(&req->payload) == NVME_PAYLOAD_TYPE_SGL);
+	assert(req->payload.reset_sgl_fn != NULL);
+	assert(req->payload.next_sge_fn != NULL);
+	req->payload.reset_sgl_fn(req->payload.contig_or_cb_arg, req->payload_offset);
+
+	max_num_sgl = req->qpair->ctrlr->max_sges;
+
+	remaining_size = req->payload_size;
+	num_sgl_desc = 0;
+	do {
+		rc = req->payload.next_sge_fn(req->payload.contig_or_cb_arg, &virt_addr, &sge_length);
+		if (rc) {
+			return -1;
+		}
+
+		sge_length = spdk_min(remaining_size, sge_length);
+
+		if (spdk_unlikely(sge_length > NVME_RDMA_MAX_KEYED_SGL_LENGTH)) {
+			SPDK_ERRLOG("SGL length %u exceeds max keyed SGL block size %u\n",
+				    sge_length, NVME_RDMA_MAX_KEYED_SGL_LENGTH);
+			return -1;
+		}
+
+		if (spdk_unlikely(!nvme_rdma_get_key(rqpair->mr_map->map, virt_addr, sge_length,
+						     NVME_RDMA_MR_RKEY, &rkey))) {
+			return -1;
+		}
+
+		cmd->sgl[num_sgl_desc].keyed.key = rkey;
+		cmd->sgl[num_sgl_desc].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK;
+		cmd->sgl[num_sgl_desc].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS;
+		cmd->sgl[num_sgl_desc].keyed.length = sge_length;
+		cmd->sgl[num_sgl_desc].address = (uint64_t)virt_addr;
+
+		remaining_size -= sge_length;
+		num_sgl_desc++;
+	} while (remaining_size > 0 && num_sgl_desc < max_num_sgl);
+
+
+	/* Should be impossible if we did our sgl checks properly up the stack, but do a sanity check here. */
+	if (remaining_size > 0) {
+		return -1;
+	}
+
+	req->cmd.psdt = SPDK_NVME_PSDT_SGL_MPTR_CONTIG;
+
+	/* The RDMA SGL needs one element describing some portion
+	 * of the spdk_nvmf_cmd structure. */
+	rdma_req->send_wr.num_sge = 1;
+
+	/*
+	 * If only one SGL descriptor is required, it can be embedded directly in the command
+	 * as a data block descriptor.
+	 */
+	if (num_sgl_desc == 1) {
+		/* The first element of this SGL is pointing at an
+		 * spdk_nvmf_cmd object. For this particular command,
+		 * we only need the first 64 bytes corresponding to
+		 * the NVMe command. */
+		rdma_req->send_sgl[0].length = sizeof(struct spdk_nvme_cmd);
+
+		req->cmd.dptr.sgl1.keyed.type = cmd->sgl[0].keyed.type;
+		req->cmd.dptr.sgl1.keyed.subtype = cmd->sgl[0].keyed.subtype;
+		req->cmd.dptr.sgl1.keyed.length = cmd->sgl[0].keyed.length;
+		req->cmd.dptr.sgl1.keyed.key = cmd->sgl[0].keyed.key;
+		req->cmd.dptr.sgl1.address = cmd->sgl[0].address;
+	} else {
+		/*
+		 * Otherwise, The SGL descriptor embedded in the command must point to the list of
+		 * SGL descriptors used to describe the operation. In that case it is a last segment descriptor.
+		 */
+		rdma_req->send_sgl[0].length = sizeof(struct spdk_nvme_cmd) + sizeof(struct
+					       spdk_nvme_sgl_descriptor) * num_sgl_desc;
+
+		req->cmd.dptr.sgl1.unkeyed.type = SPDK_NVME_SGL_TYPE_LAST_SEGMENT;
+		req->cmd.dptr.sgl1.unkeyed.subtype = SPDK_NVME_SGL_SUBTYPE_OFFSET;
+		req->cmd.dptr.sgl1.unkeyed.length = num_sgl_desc * sizeof(struct spdk_nvme_sgl_descriptor);
+		req->cmd.dptr.sgl1.address = (uint64_t)0;
+	}
+
+	return 0;
+}
+
+/*
+ * Build inline SGL describing sgl payload buffer.
+ */
+static int
+nvme_rdma_build_sgl_inline_request(struct nvme_rdma_qpair *rqpair,
+				   struct spdk_nvme_rdma_req *rdma_req)
+{
+	struct nvme_request *req = rdma_req->req;
+	uint32_t lkey = 0;
+	uint32_t length;
+	void *virt_addr;
+	int rc;
+
+	assert(req->payload_size != 0);
+	assert(nvme_payload_type(&req->payload) == NVME_PAYLOAD_TYPE_SGL);
+	assert(req->payload.reset_sgl_fn != NULL);
+	assert(req->payload.next_sge_fn != NULL);
+	req->payload.reset_sgl_fn(req->payload.contig_or_cb_arg, req->payload_offset);
+
+	rc = req->payload.next_sge_fn(req->payload.contig_or_cb_arg, &virt_addr, &length);
+	if (rc) {
+		return -1;
+	}
+
+	if (length < req->payload_size) {
+		SPDK_DEBUGLOG(SPDK_LOG_NVME, "Inline SGL request split so sending separately.\n");
+		return nvme_rdma_build_sgl_request(rqpair, rdma_req);
+	}
+
+	if (length > req->payload_size) {
+		length = req->payload_size;
+	}
+
+	if (spdk_unlikely(!nvme_rdma_get_key(rqpair->mr_map->map, virt_addr, length,
+					     NVME_RDMA_MR_LKEY, &lkey))) {
+		return -1;
+	}
+
+	rdma_req->send_sgl[1].addr = (uint64_t)virt_addr;
+	rdma_req->send_sgl[1].length = length;
+	rdma_req->send_sgl[1].lkey = lkey;
+
+	rdma_req->send_wr.num_sge = 2;
+
+	/* The first element of this SGL is pointing at an
+	 * spdk_nvmf_cmd object. For this particular command,
+	 * we only need the first 64 bytes corresponding to
+	 * the NVMe command. */
+	rdma_req->send_sgl[0].length = sizeof(struct spdk_nvme_cmd);
+
+	req->cmd.psdt = SPDK_NVME_PSDT_SGL_MPTR_CONTIG;
+	req->cmd.dptr.sgl1.unkeyed.type = SPDK_NVME_SGL_TYPE_DATA_BLOCK;
+	req->cmd.dptr.sgl1.unkeyed.subtype = SPDK_NVME_SGL_SUBTYPE_OFFSET;
+	req->cmd.dptr.sgl1.unkeyed.length = (uint32_t)req->payload_size;
+	/* Inline only supported for icdoff == 0 currently.  This function will
+	 * not get called for controllers with other values. */
+	req->cmd.dptr.sgl1.address = (uint64_t)0;
+
+	return 0;
+}
+
+static int
+nvme_rdma_req_init(struct nvme_rdma_qpair *rqpair, struct nvme_request *req,
+		   struct spdk_nvme_rdma_req *rdma_req)
+{
+	struct spdk_nvme_ctrlr *ctrlr = rqpair->qpair.ctrlr;
+	enum nvme_payload_type payload_type;
+	bool icd_supported;
+	int rc;
+
+	assert(rdma_req->req == NULL);
+	rdma_req->req = req;
+	req->cmd.cid = rdma_req->id;
+	payload_type = nvme_payload_type(&req->payload);
+	/*
+	 * Check if icdoff is non zero, to avoid interop conflicts with
+	 * targets with non-zero icdoff.  Both SPDK and the Linux kernel
+	 * targets use icdoff = 0.  For targets with non-zero icdoff, we
+	 * will currently just not use inline data for now.
+	 */
+	icd_supported = spdk_nvme_opc_get_data_transfer(req->cmd.opc) == SPDK_NVME_DATA_HOST_TO_CONTROLLER
+			&& req->payload_size <= ctrlr->ioccsz_bytes && ctrlr->icdoff == 0;
+
+	if (req->payload_size == 0) {
+		rc = nvme_rdma_build_null_request(rdma_req);
+	} else if (payload_type == NVME_PAYLOAD_TYPE_CONTIG) {
+		if (icd_supported) {
+			rc = nvme_rdma_build_contig_inline_request(rqpair, rdma_req);
+		} else {
+			rc = nvme_rdma_build_contig_request(rqpair, rdma_req);
+		}
+	} else if (payload_type == NVME_PAYLOAD_TYPE_SGL) {
+		if (icd_supported) {
+			rc = nvme_rdma_build_sgl_inline_request(rqpair, rdma_req);
+		} else {
+			rc = nvme_rdma_build_sgl_request(rqpair, rdma_req);
+		}
+	} else {
+		rc = -1;
+	}
+
+	if (rc) {
+		rdma_req->req = NULL;
+		return rc;
+	}
+
+	memcpy(&rqpair->cmds[rdma_req->id], &req->cmd, sizeof(req->cmd));
+	return 0;
+}
+
+static struct spdk_nvme_qpair *
+nvme_rdma_ctrlr_create_qpair(struct spdk_nvme_ctrlr *ctrlr,
+			     uint16_t qid, uint32_t qsize,
+			     enum spdk_nvme_qprio qprio,
+			     uint32_t num_requests,
+			     bool delay_cmd_submit)
+{
+	struct nvme_rdma_qpair *rqpair;
+	struct spdk_nvme_qpair *qpair;
+	int rc;
+
+	rqpair = nvme_rdma_calloc(1, sizeof(struct nvme_rdma_qpair));
+	if (!rqpair) {
+		SPDK_ERRLOG("failed to get create rqpair\n");
+		return NULL;
+	}
+
+	rqpair->num_entries = qsize;
+	rqpair->delay_cmd_submit = delay_cmd_submit;
+	qpair = &rqpair->qpair;
+	rc = nvme_qpair_init(qpair, qid, ctrlr, qprio, num_requests);
+	if (rc != 0) {
+		return NULL;
+	}
+
+	rc = nvme_rdma_alloc_reqs(rqpair);
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "rc =%d\n", rc);
+	if (rc) {
+		SPDK_ERRLOG("Unable to allocate rqpair RDMA requests\n");
+		nvme_rdma_free(rqpair);
+		return NULL;
+	}
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "RDMA requests allocated\n");
+
+	rc = nvme_rdma_alloc_rsps(rqpair);
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "rc =%d\n", rc);
+	if (rc < 0) {
+		SPDK_ERRLOG("Unable to allocate rqpair RDMA responses\n");
+		nvme_rdma_free_reqs(rqpair);
+		nvme_rdma_free(rqpair);
+		return NULL;
+	}
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "RDMA responses allocated\n");
+
+	return qpair;
+}
+
+static void
+nvme_rdma_ctrlr_disconnect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)
+{
+	struct nvme_rdma_qpair *rqpair = nvme_rdma_qpair(qpair);
+	struct nvme_rdma_ctrlr *rctrlr = NULL;
+	struct nvme_rdma_cm_event_entry *entry, *tmp;
+
+	nvme_rdma_unregister_mem(rqpair);
+	nvme_rdma_unregister_reqs(rqpair);
+	nvme_rdma_unregister_rsps(rqpair);
+
+	if (rqpair->evt) {
+		rdma_ack_cm_event(rqpair->evt);
+		rqpair->evt = NULL;
+	}
+
+	/*
+	 * This works because we have the controller lock both in
+	 * this function and in the function where we add new events.
+	 */
+	if (qpair->ctrlr != NULL) {
+		rctrlr = nvme_rdma_ctrlr(qpair->ctrlr);
+		STAILQ_FOREACH_SAFE(entry, &rctrlr->pending_cm_events, link, tmp) {
+			if (nvme_rdma_qpair(entry->evt->id->context) == rqpair) {
+				STAILQ_REMOVE(&rctrlr->pending_cm_events, entry, nvme_rdma_cm_event_entry, link);
+				rdma_ack_cm_event(entry->evt);
+				STAILQ_INSERT_HEAD(&rctrlr->free_cm_events, entry, link);
+			}
+		}
+	}
+
+	if (rqpair->cm_id) {
+		if (rqpair->rdma_qp) {
+			spdk_rdma_qp_disconnect(rqpair->rdma_qp);
+			if (rctrlr != NULL) {
+				if (nvme_rdma_process_event(rqpair, rctrlr->cm_channel, RDMA_CM_EVENT_DISCONNECTED)) {
+					SPDK_DEBUGLOG(SPDK_LOG_NVME, "Target did not respond to qpair disconnect.\n");
+				}
+			}
+			spdk_rdma_qp_destroy(rqpair->rdma_qp);
+			rqpair->rdma_qp = NULL;
+		}
+
+		rdma_destroy_id(rqpair->cm_id);
+		rqpair->cm_id = NULL;
+	}
+
+	if (rqpair->cq) {
+		ibv_destroy_cq(rqpair->cq);
+		rqpair->cq = NULL;
+	}
+}
+
+static void nvme_rdma_qpair_abort_reqs(struct spdk_nvme_qpair *qpair, uint32_t dnr);
+
+static int
+nvme_rdma_ctrlr_delete_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)
+{
+	struct nvme_rdma_qpair *rqpair;
+
+	rqpair = nvme_rdma_qpair(qpair);
+	nvme_transport_ctrlr_disconnect_qpair(ctrlr, qpair);
+	if (rqpair->defer_deletion_to_pg) {
+		nvme_qpair_set_state(qpair, NVME_QPAIR_DESTROYING);
+		return 0;
+	}
+
+	nvme_rdma_qpair_abort_reqs(qpair, 1);
+	nvme_qpair_deinit(qpair);
+
+	nvme_rdma_free_reqs(rqpair);
+	nvme_rdma_free_rsps(rqpair);
+	nvme_rdma_free(rqpair);
+
+	return 0;
+}
+
+static struct spdk_nvme_qpair *
+nvme_rdma_ctrlr_create_io_qpair(struct spdk_nvme_ctrlr *ctrlr, uint16_t qid,
+				const struct spdk_nvme_io_qpair_opts *opts)
+{
+	return nvme_rdma_ctrlr_create_qpair(ctrlr, qid, opts->io_queue_size, opts->qprio,
+					    opts->io_queue_requests,
+					    opts->delay_cmd_submit);
+}
+
+static int
+nvme_rdma_ctrlr_enable(struct spdk_nvme_ctrlr *ctrlr)
+{
+	/* do nothing here */
+	return 0;
+}
+
+static int nvme_rdma_ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr);
+
+static struct spdk_nvme_ctrlr *nvme_rdma_ctrlr_construct(const struct spdk_nvme_transport_id *trid,
+		const struct spdk_nvme_ctrlr_opts *opts,
+		void *devhandle)
+{
+	struct nvme_rdma_ctrlr *rctrlr;
+	union spdk_nvme_cap_register cap;
+	union spdk_nvme_vs_register vs;
+	struct ibv_context **contexts;
+	struct ibv_device_attr dev_attr;
+	int i, flag, rc;
+
+	rctrlr = nvme_rdma_calloc(1, sizeof(struct nvme_rdma_ctrlr));
+	if (rctrlr == NULL) {
+		SPDK_ERRLOG("could not allocate ctrlr\n");
+		return NULL;
+	}
+
+	rctrlr->ctrlr.opts = *opts;
+	rctrlr->ctrlr.trid = *trid;
+
+	if (opts->transport_retry_count > NVME_RDMA_CTRLR_MAX_TRANSPORT_RETRY_COUNT) {
+		SPDK_NOTICELOG("transport_retry_count exceeds max value %d, use max value\n",
+			       NVME_RDMA_CTRLR_MAX_TRANSPORT_RETRY_COUNT);
+		rctrlr->ctrlr.opts.transport_retry_count = NVME_RDMA_CTRLR_MAX_TRANSPORT_RETRY_COUNT;
+	}
+
+	if (opts->transport_ack_timeout > NVME_RDMA_CTRLR_MAX_TRANSPORT_ACK_TIMEOUT) {
+		SPDK_NOTICELOG("transport_ack_timeout exceeds max value %d, use max value\n",
+			       NVME_RDMA_CTRLR_MAX_TRANSPORT_ACK_TIMEOUT);
+		rctrlr->ctrlr.opts.transport_ack_timeout = NVME_RDMA_CTRLR_MAX_TRANSPORT_ACK_TIMEOUT;
+	}
+
+	contexts = rdma_get_devices(NULL);
+	if (contexts == NULL) {
+		SPDK_ERRLOG("rdma_get_devices() failed: %s (%d)\n", spdk_strerror(errno), errno);
+		nvme_rdma_free(rctrlr);
+		return NULL;
+	}
+
+	i = 0;
+	rctrlr->max_sge = NVME_RDMA_MAX_SGL_DESCRIPTORS;
+
+	while (contexts[i] != NULL) {
+		rc = ibv_query_device(contexts[i], &dev_attr);
+		if (rc < 0) {
+			SPDK_ERRLOG("Failed to query RDMA device attributes.\n");
+			rdma_free_devices(contexts);
+			nvme_rdma_free(rctrlr);
+			return NULL;
+		}
+		rctrlr->max_sge = spdk_min(rctrlr->max_sge, (uint16_t)dev_attr.max_sge);
+		i++;
+	}
+
+	rdma_free_devices(contexts);
+
+	rc = nvme_ctrlr_construct(&rctrlr->ctrlr);
+	if (rc != 0) {
+		nvme_rdma_free(rctrlr);
+		return NULL;
+	}
+
+	STAILQ_INIT(&rctrlr->pending_cm_events);
+	STAILQ_INIT(&rctrlr->free_cm_events);
+	rctrlr->cm_events = nvme_rdma_calloc(NVME_RDMA_NUM_CM_EVENTS, sizeof(*rctrlr->cm_events));
+	if (rctrlr->cm_events == NULL) {
+		SPDK_ERRLOG("unable to allocat buffers to hold CM events.\n");
+		goto destruct_ctrlr;
+	}
+
+	for (i = 0; i < NVME_RDMA_NUM_CM_EVENTS; i++) {
+		STAILQ_INSERT_TAIL(&rctrlr->free_cm_events, &rctrlr->cm_events[i], link);
+	}
+
+	rctrlr->cm_channel = rdma_create_event_channel();
+	if (rctrlr->cm_channel == NULL) {
+		SPDK_ERRLOG("rdma_create_event_channel() failed\n");
+		goto destruct_ctrlr;
+	}
+
+	flag = fcntl(rctrlr->cm_channel->fd, F_GETFL);
+	if (fcntl(rctrlr->cm_channel->fd, F_SETFL, flag | O_NONBLOCK) < 0) {
+		SPDK_ERRLOG("Cannot set event channel to non blocking\n");
+		goto destruct_ctrlr;
+	}
+
+	rctrlr->ctrlr.adminq = nvme_rdma_ctrlr_create_qpair(&rctrlr->ctrlr, 0,
+			       rctrlr->ctrlr.opts.admin_queue_size, 0,
+			       rctrlr->ctrlr.opts.admin_queue_size, false);
+	if (!rctrlr->ctrlr.adminq) {
+		SPDK_ERRLOG("failed to create admin qpair\n");
+		goto destruct_ctrlr;
+	}
+
+	rc = nvme_transport_ctrlr_connect_qpair(&rctrlr->ctrlr, rctrlr->ctrlr.adminq);
+	if (rc < 0) {
+		SPDK_ERRLOG("failed to connect admin qpair\n");
+		goto destruct_ctrlr;
+	}
+
+	if (nvme_ctrlr_get_cap(&rctrlr->ctrlr, &cap)) {
+		SPDK_ERRLOG("get_cap() failed\n");
+		goto destruct_ctrlr;
+	}
+
+	if (nvme_ctrlr_get_vs(&rctrlr->ctrlr, &vs)) {
+		SPDK_ERRLOG("get_vs() failed\n");
+		goto destruct_ctrlr;
+	}
+
+	if (nvme_ctrlr_add_process(&rctrlr->ctrlr, 0) != 0) {
+		SPDK_ERRLOG("nvme_ctrlr_add_process() failed\n");
+		goto destruct_ctrlr;
+	}
+
+	nvme_ctrlr_init_cap(&rctrlr->ctrlr, &cap, &vs);
+
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "successfully initialized the nvmf ctrlr\n");
+	return &rctrlr->ctrlr;
+
+destruct_ctrlr:
+	nvme_ctrlr_destruct(&rctrlr->ctrlr);
+	return NULL;
+}
+
+static int
+nvme_rdma_ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr)
+{
+	struct nvme_rdma_ctrlr *rctrlr = nvme_rdma_ctrlr(ctrlr);
+	struct nvme_rdma_cm_event_entry *entry;
+
+	if (ctrlr->adminq) {
+		nvme_rdma_ctrlr_delete_io_qpair(ctrlr, ctrlr->adminq);
+	}
+
+	STAILQ_FOREACH(entry, &rctrlr->pending_cm_events, link) {
+		rdma_ack_cm_event(entry->evt);
+	}
+
+	STAILQ_INIT(&rctrlr->free_cm_events);
+	STAILQ_INIT(&rctrlr->pending_cm_events);
+	nvme_rdma_free(rctrlr->cm_events);
+
+	if (rctrlr->cm_channel) {
+		rdma_destroy_event_channel(rctrlr->cm_channel);
+		rctrlr->cm_channel = NULL;
+	}
+
+	nvme_ctrlr_destruct_finish(ctrlr);
+
+	nvme_rdma_free(rctrlr);
+
+	return 0;
+}
+
+static int
+nvme_rdma_qpair_submit_request(struct spdk_nvme_qpair *qpair,
+			       struct nvme_request *req)
+{
+	struct nvme_rdma_qpair *rqpair;
+	struct spdk_nvme_rdma_req *rdma_req;
+	struct ibv_send_wr *wr;
+
+	rqpair = nvme_rdma_qpair(qpair);
+	assert(rqpair != NULL);
+	assert(req != NULL);
+
+	rdma_req = nvme_rdma_req_get(rqpair);
+	if (!rdma_req) {
+		/* Inform the upper layer to try again later. */
+		return -EAGAIN;
+	}
+
+	if (nvme_rdma_req_init(rqpair, req, rdma_req)) {
+		SPDK_ERRLOG("nvme_rdma_req_init() failed\n");
+		TAILQ_REMOVE(&rqpair->outstanding_reqs, rdma_req, link);
+		nvme_rdma_req_put(rqpair, rdma_req);
+		return -1;
+	}
+
+	wr = &rdma_req->send_wr;
+	wr->next = NULL;
+	nvme_rdma_trace_ibv_sge(wr->sg_list);
+	return nvme_rdma_qpair_queue_send_wr(rqpair, wr);
+}
+
+static int
+nvme_rdma_qpair_reset(struct spdk_nvme_qpair *qpair)
+{
+	/* Currently, doing nothing here */
+	return 0;
+}
+
+static void
+nvme_rdma_qpair_abort_reqs(struct spdk_nvme_qpair *qpair, uint32_t dnr)
+{
+	struct spdk_nvme_rdma_req *rdma_req, *tmp;
+	struct spdk_nvme_cpl cpl;
+	struct nvme_rdma_qpair *rqpair = nvme_rdma_qpair(qpair);
+
+	cpl.status.sc = SPDK_NVME_SC_ABORTED_SQ_DELETION;
+	cpl.status.sct = SPDK_NVME_SCT_GENERIC;
+	cpl.status.dnr = dnr;
+
+	/*
+	 * We cannot abort requests at the RDMA layer without
+	 * unregistering them. If we do, we can still get error
+	 * free completions on the shared completion queue.
+	 */
+	if (nvme_qpair_get_state(qpair) > NVME_QPAIR_DISCONNECTING &&
+	    nvme_qpair_get_state(qpair) != NVME_QPAIR_DESTROYING) {
+		nvme_ctrlr_disconnect_qpair(qpair);
+	}
+
+	TAILQ_FOREACH_SAFE(rdma_req, &rqpair->outstanding_reqs, link, tmp) {
+		nvme_rdma_req_complete(rdma_req, &cpl);
+		nvme_rdma_req_put(rqpair, rdma_req);
+	}
+}
+
+static void
+nvme_rdma_qpair_check_timeout(struct spdk_nvme_qpair *qpair)
+{
+	uint64_t t02;
+	struct spdk_nvme_rdma_req *rdma_req, *tmp;
+	struct nvme_rdma_qpair *rqpair = nvme_rdma_qpair(qpair);
+	struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
+	struct spdk_nvme_ctrlr_process *active_proc;
+
+	/* Don't check timeouts during controller initialization. */
+	if (ctrlr->state != NVME_CTRLR_STATE_READY) {
+		return;
+	}
+
+	if (nvme_qpair_is_admin_queue(qpair)) {
+		active_proc = nvme_ctrlr_get_current_process(ctrlr);
+	} else {
+		active_proc = qpair->active_proc;
+	}
+
+	/* Only check timeouts if the current process has a timeout callback. */
+	if (active_proc == NULL || active_proc->timeout_cb_fn == NULL) {
+		return;
+	}
+
+	t02 = spdk_get_ticks();
+	TAILQ_FOREACH_SAFE(rdma_req, &rqpair->outstanding_reqs, link, tmp) {
+		assert(rdma_req->req != NULL);
+
+		if (nvme_request_check_timeout(rdma_req->req, rdma_req->id, active_proc, t02)) {
+			/*
+			 * The requests are in order, so as soon as one has not timed out,
+			 * stop iterating.
+			 */
+			break;
+		}
+	}
+}
+
+static inline int
+nvme_rdma_request_ready(struct nvme_rdma_qpair *rqpair, struct spdk_nvme_rdma_req *rdma_req)
+{
+	nvme_rdma_req_complete(rdma_req, &rqpair->rsps[rdma_req->rsp_idx].cpl);
+	nvme_rdma_req_put(rqpair, rdma_req);
+	return nvme_rdma_post_recv(rqpair, rdma_req->rsp_idx);
+}
+
+#define MAX_COMPLETIONS_PER_POLL 128
+
+static void
+nvme_rdma_fail_qpair(struct spdk_nvme_qpair *qpair, int failure_reason)
+{
+	if (failure_reason == IBV_WC_RETRY_EXC_ERR) {
+		qpair->transport_failure_reason = SPDK_NVME_QPAIR_FAILURE_REMOTE;
+	} else if (qpair->transport_failure_reason == SPDK_NVME_QPAIR_FAILURE_NONE) {
+		qpair->transport_failure_reason = SPDK_NVME_QPAIR_FAILURE_UNKNOWN;
+	}
+
+	nvme_ctrlr_disconnect_qpair(qpair);
+}
+
+static void
+nvme_rdma_conditional_fail_qpair(struct nvme_rdma_qpair *rqpair, struct nvme_rdma_poll_group *group)
+{
+	struct nvme_rdma_destroyed_qpair	*qpair_tracker;
+
+	assert(rqpair);
+	if (group) {
+		STAILQ_FOREACH(qpair_tracker, &group->destroyed_qpairs, link) {
+			if (qpair_tracker->destroyed_qpair_tracker == rqpair) {
+				return;
+			}
+		}
+	}
+	nvme_rdma_fail_qpair(&rqpair->qpair, 0);
+}
+
+static int
+nvme_rdma_cq_process_completions(struct ibv_cq *cq, uint32_t batch_size,
+				 struct nvme_rdma_poll_group *group,
+				 struct nvme_rdma_qpair *rdma_qpair)
+{
+	struct ibv_wc			wc[MAX_COMPLETIONS_PER_POLL];
+	struct nvme_rdma_qpair		*rqpair;
+	struct spdk_nvme_rdma_req	*rdma_req;
+	struct spdk_nvme_rdma_rsp	*rdma_rsp;
+	struct nvme_rdma_wr		*rdma_wr;
+	uint32_t			reaped = 0;
+	int				completion_rc = 0;
+	int				rc, i;
+
+	rc = ibv_poll_cq(cq, batch_size, wc);
+	if (rc < 0) {
+		SPDK_ERRLOG("Error polling CQ! (%d): %s\n",
+			    errno, spdk_strerror(errno));
+		return -ECANCELED;
+	} else if (rc == 0) {
+		return 0;
+	}
+
+	for (i = 0; i < rc; i++) {
+		rdma_wr = (struct nvme_rdma_wr *)wc[i].wr_id;
+		switch (rdma_wr->type) {
+		case RDMA_WR_TYPE_RECV:
+			rdma_rsp = SPDK_CONTAINEROF(rdma_wr, struct spdk_nvme_rdma_rsp, rdma_wr);
+			rqpair = rdma_rsp->rqpair;
+			assert(rqpair->current_num_recvs > 0);
+			rqpair->current_num_recvs--;
+
+			if (wc[i].status) {
+				SPDK_ERRLOG("CQ error on Queue Pair %p, Response Index %lu (%d): %s\n",
+					    rqpair, wc[i].wr_id, wc[i].status, ibv_wc_status_str(wc[i].status));
+				nvme_rdma_conditional_fail_qpair(rqpair, group);
+				completion_rc = -ENXIO;
+				continue;
+			}
+
+			SPDK_DEBUGLOG(SPDK_LOG_NVME, "CQ recv completion\n");
+
+			if (wc[i].byte_len < sizeof(struct spdk_nvme_cpl)) {
+				SPDK_ERRLOG("recv length %u less than expected response size\n", wc[i].byte_len);
+				nvme_rdma_conditional_fail_qpair(rqpair, group);
+				completion_rc = -ENXIO;
+				continue;
+			}
+			rdma_req = &rqpair->rdma_reqs[rdma_rsp->cpl.cid];
+			rdma_req->completion_flags |= NVME_RDMA_RECV_COMPLETED;
+			rdma_req->rsp_idx = rdma_rsp->idx;
+
+			if ((rdma_req->completion_flags & NVME_RDMA_SEND_COMPLETED) != 0) {
+				if (spdk_unlikely(nvme_rdma_request_ready(rqpair, rdma_req))) {
+					SPDK_ERRLOG("Unable to re-post rx descriptor\n");
+					nvme_rdma_conditional_fail_qpair(rqpair, group);
+					completion_rc = -ENXIO;
+					continue;
+				}
+				reaped++;
+				rqpair->num_completions++;
+			}
+			break;
+
+		case RDMA_WR_TYPE_SEND:
+			rdma_req = SPDK_CONTAINEROF(rdma_wr, struct spdk_nvme_rdma_req, rdma_wr);
+
+			/* If we are flushing I/O */
+			if (wc[i].status) {
+				rqpair = rdma_req->req ? nvme_rdma_qpair(rdma_req->req->qpair) : NULL;
+				if (!rqpair) {
+					rqpair = rdma_qpair != NULL ? rdma_qpair : nvme_rdma_poll_group_get_qpair_by_id(group,
+							wc[i].qp_num);
+				}
+				assert(rqpair);
+				assert(rqpair->current_num_sends > 0);
+				rqpair->current_num_sends--;
+				nvme_rdma_conditional_fail_qpair(rqpair, group);
+				SPDK_ERRLOG("CQ error on Queue Pair %p, Response Index %lu (%d): %s\n",
+					    rqpair, wc[i].wr_id, wc[i].status, ibv_wc_status_str(wc[i].status));
+				completion_rc = -ENXIO;
+				continue;
+			}
+
+			rqpair = nvme_rdma_qpair(rdma_req->req->qpair);
+			rdma_req->completion_flags |= NVME_RDMA_SEND_COMPLETED;
+			rqpair->current_num_sends--;
+
+			if ((rdma_req->completion_flags & NVME_RDMA_RECV_COMPLETED) != 0) {
+				if (spdk_unlikely(nvme_rdma_request_ready(rqpair, rdma_req))) {
+					SPDK_ERRLOG("Unable to re-post rx descriptor\n");
+					nvme_rdma_conditional_fail_qpair(rqpair, group);
+					completion_rc = -ENXIO;
+					continue;
+				}
+				reaped++;
+				rqpair->num_completions++;
+			}
+			break;
+
+		default:
+			SPDK_ERRLOG("Received an unexpected opcode on the CQ: %d\n", rdma_wr->type);
+			return -ECANCELED;
+		}
+	}
+
+	if (completion_rc) {
+		return completion_rc;
+	}
+
+	return reaped;
+}
+
+static void
+dummy_disconnected_qpair_cb(struct spdk_nvme_qpair *qpair, void *poll_group_ctx)
+{
+
+}
+
+static int
+nvme_rdma_qpair_process_completions(struct spdk_nvme_qpair *qpair,
+				    uint32_t max_completions)
+{
+	struct nvme_rdma_qpair		*rqpair = nvme_rdma_qpair(qpair);
+	int				rc = 0, batch_size;
+	struct ibv_cq			*cq;
+	struct nvme_rdma_ctrlr		*rctrlr;
+
+	/*
+	 * This is used during the connection phase. It's possible that we are still reaping error completions
+	 * from other qpairs so we need to call the poll group function. Also, it's more correct since the cq
+	 * is shared.
+	 */
+	if (qpair->poll_group != NULL) {
+		return spdk_nvme_poll_group_process_completions(qpair->poll_group->group, max_completions,
+				dummy_disconnected_qpair_cb);
+	}
+
+	if (max_completions == 0) {
+		max_completions = rqpair->num_entries;
+	} else {
+		max_completions = spdk_min(max_completions, rqpair->num_entries);
+	}
+
+	if (nvme_qpair_is_admin_queue(&rqpair->qpair)) {
+		rctrlr = nvme_rdma_ctrlr(rqpair->qpair.ctrlr);
+		nvme_rdma_poll_events(rctrlr);
+	}
+	nvme_rdma_qpair_process_cm_event(rqpair);
+
+	if (spdk_unlikely(qpair->transport_failure_reason != SPDK_NVME_QPAIR_FAILURE_NONE)) {
+		nvme_rdma_fail_qpair(qpair, 0);
+		return -ENXIO;
+	}
+
+	cq = rqpair->cq;
+
+	rqpair->num_completions = 0;
+	do {
+		batch_size = spdk_min((max_completions - rqpair->num_completions), MAX_COMPLETIONS_PER_POLL);
+		rc = nvme_rdma_cq_process_completions(cq, batch_size, NULL, rqpair);
+
+		if (rc == 0) {
+			break;
+			/* Handle the case where we fail to poll the cq. */
+		} else if (rc == -ECANCELED) {
+			nvme_rdma_fail_qpair(qpair, 0);
+			return -ENXIO;
+		} else if (rc == -ENXIO) {
+			return rc;
+		}
+	} while (rqpair->num_completions < max_completions);
+
+	if (spdk_unlikely(nvme_rdma_qpair_submit_sends(rqpair) ||
+			  nvme_rdma_qpair_submit_recvs(rqpair))) {
+		nvme_rdma_fail_qpair(qpair, 0);
+		return -ENXIO;
+	}
+
+	if (spdk_unlikely(rqpair->qpair.ctrlr->timeout_enabled)) {
+		nvme_rdma_qpair_check_timeout(qpair);
+	}
+
+	return rqpair->num_completions;
+}
+
+static uint32_t
+nvme_rdma_ctrlr_get_max_xfer_size(struct spdk_nvme_ctrlr *ctrlr)
+{
+	/* max_mr_size by ibv_query_device indicates the largest value that we can
+	 * set for a registered memory region.  It is independent from the actual
+	 * I/O size and is very likely to be larger than 2 MiB which is the
+	 * granularity we currently register memory regions.  Hence return
+	 * UINT32_MAX here and let the generic layer use the controller data to
+	 * moderate this value.
+	 */
+	return UINT32_MAX;
+}
+
+static uint16_t
+nvme_rdma_ctrlr_get_max_sges(struct spdk_nvme_ctrlr *ctrlr)
+{
+	struct nvme_rdma_ctrlr *rctrlr = nvme_rdma_ctrlr(ctrlr);
+
+	return rctrlr->max_sge;
+}
+
+static int
+nvme_rdma_qpair_iterate_requests(struct spdk_nvme_qpair *qpair,
+				 int (*iter_fn)(struct nvme_request *req, void *arg),
+				 void *arg)
+{
+	struct nvme_rdma_qpair *rqpair = nvme_rdma_qpair(qpair);
+	struct spdk_nvme_rdma_req *rdma_req, *tmp;
+	int rc;
+
+	assert(iter_fn != NULL);
+
+	TAILQ_FOREACH_SAFE(rdma_req, &rqpair->outstanding_reqs, link, tmp) {
+		assert(rdma_req->req != NULL);
+
+		rc = iter_fn(rdma_req->req, arg);
+		if (rc != 0) {
+			return rc;
+		}
+	}
+
+	return 0;
+}
+
+static void
+nvme_rdma_admin_qpair_abort_aers(struct spdk_nvme_qpair *qpair)
+{
+	struct spdk_nvme_rdma_req *rdma_req, *tmp;
+	struct spdk_nvme_cpl cpl;
+	struct nvme_rdma_qpair *rqpair = nvme_rdma_qpair(qpair);
+
+	cpl.status.sc = SPDK_NVME_SC_ABORTED_SQ_DELETION;
+	cpl.status.sct = SPDK_NVME_SCT_GENERIC;
+
+	TAILQ_FOREACH_SAFE(rdma_req, &rqpair->outstanding_reqs, link, tmp) {
+		assert(rdma_req->req != NULL);
+
+		if (rdma_req->req->cmd.opc != SPDK_NVME_OPC_ASYNC_EVENT_REQUEST) {
+			continue;
+		}
+
+		nvme_rdma_req_complete(rdma_req, &cpl);
+		nvme_rdma_req_put(rqpair, rdma_req);
+	}
+}
+
+static int
+nvme_rdma_poller_create(struct nvme_rdma_poll_group *group, struct ibv_context *ctx)
+{
+	struct nvme_rdma_poller *poller;
+
+	poller = calloc(1, sizeof(*poller));
+	if (poller == NULL) {
+		SPDK_ERRLOG("Unable to allocate poller.\n");
+		return -ENOMEM;
+	}
+
+	poller->device = ctx;
+	poller->cq = ibv_create_cq(poller->device, DEFAULT_NVME_RDMA_CQ_SIZE, group, NULL, 0);
+
+	if (poller->cq == NULL) {
+		free(poller);
+		return -EINVAL;
+	}
+
+	STAILQ_INSERT_HEAD(&group->pollers, poller, link);
+	group->num_pollers++;
+	poller->current_num_wc = DEFAULT_NVME_RDMA_CQ_SIZE;
+	poller->required_num_wc = 0;
+	return 0;
+}
+
+static void
+nvme_rdma_poll_group_free_pollers(struct nvme_rdma_poll_group *group)
+{
+	struct nvme_rdma_poller	*poller, *tmp_poller;
+
+	STAILQ_FOREACH_SAFE(poller, &group->pollers, link, tmp_poller) {
+		if (poller->cq) {
+			ibv_destroy_cq(poller->cq);
+		}
+		STAILQ_REMOVE(&group->pollers, poller, nvme_rdma_poller, link);
+		free(poller);
+	}
+}
+
+static struct spdk_nvme_transport_poll_group *
+nvme_rdma_poll_group_create(void)
+{
+	struct nvme_rdma_poll_group	*group;
+	struct ibv_context		**contexts;
+	int i = 0;
+
+	group = calloc(1, sizeof(*group));
+	if (group == NULL) {
+		SPDK_ERRLOG("Unable to allocate poll group.\n");
+		return NULL;
+	}
+
+	STAILQ_INIT(&group->pollers);
+
+	contexts = rdma_get_devices(NULL);
+	if (contexts == NULL) {
+		SPDK_ERRLOG("rdma_get_devices() failed: %s (%d)\n", spdk_strerror(errno), errno);
+		free(group);
+		return NULL;
+	}
+
+	while (contexts[i] != NULL) {
+		if (nvme_rdma_poller_create(group, contexts[i])) {
+			nvme_rdma_poll_group_free_pollers(group);
+			free(group);
+			rdma_free_devices(contexts);
+			return NULL;
+		}
+		i++;
+	}
+
+	rdma_free_devices(contexts);
+	STAILQ_INIT(&group->destroyed_qpairs);
+	return &group->group;
+}
+
+struct nvme_rdma_qpair *
+nvme_rdma_poll_group_get_qpair_by_id(struct nvme_rdma_poll_group *group, uint32_t qp_num)
+{
+	struct spdk_nvme_qpair *qpair;
+	struct nvme_rdma_destroyed_qpair *rqpair_tracker;
+	struct nvme_rdma_qpair *rqpair;
+
+	STAILQ_FOREACH(qpair, &group->group.disconnected_qpairs, poll_group_stailq) {
+		rqpair = nvme_rdma_qpair(qpair);
+		if (rqpair->rdma_qp->qp->qp_num == qp_num) {
+			return rqpair;
+		}
+	}
+
+	STAILQ_FOREACH(qpair, &group->group.connected_qpairs, poll_group_stailq) {
+		rqpair = nvme_rdma_qpair(qpair);
+		if (rqpair->rdma_qp->qp->qp_num == qp_num) {
+			return rqpair;
+		}
+	}
+
+	STAILQ_FOREACH(rqpair_tracker, &group->destroyed_qpairs, link) {
+		rqpair = rqpair_tracker->destroyed_qpair_tracker;
+		if (rqpair->rdma_qp->qp->qp_num == qp_num) {
+			return rqpair;
+		}
+	}
+
+	return NULL;
+}
+
+static int
+nvme_rdma_resize_cq(struct nvme_rdma_qpair *rqpair, struct nvme_rdma_poller *poller)
+{
+	int	current_num_wc, required_num_wc;
+
+	required_num_wc = poller->required_num_wc + WC_PER_QPAIR(rqpair->num_entries);
+	current_num_wc = poller->current_num_wc;
+	if (current_num_wc < required_num_wc) {
+		current_num_wc = spdk_max(current_num_wc * 2, required_num_wc);
+	}
+
+	if (poller->current_num_wc != current_num_wc) {
+		SPDK_DEBUGLOG(SPDK_LOG_NVME, "Resize RDMA CQ from %d to %d\n", poller->current_num_wc,
+			      current_num_wc);
+		if (ibv_resize_cq(poller->cq, current_num_wc)) {
+			SPDK_ERRLOG("RDMA CQ resize failed: errno %d: %s\n", errno, spdk_strerror(errno));
+			return -1;
+		}
+
+		poller->current_num_wc = current_num_wc;
+	}
+
+	poller->required_num_wc = required_num_wc;
+	return 0;
+}
+
+static int
+nvme_rdma_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair)
+{
+	struct nvme_rdma_qpair		*rqpair = nvme_rdma_qpair(qpair);
+	struct nvme_rdma_poll_group	*group = nvme_rdma_poll_group(qpair->poll_group);
+	struct nvme_rdma_poller		*poller;
+
+	assert(rqpair->cq == NULL);
+
+	STAILQ_FOREACH(poller, &group->pollers, link) {
+		if (poller->device == rqpair->cm_id->verbs) {
+			if (nvme_rdma_resize_cq(rqpair, poller)) {
+				return -EPROTO;
+			}
+			rqpair->cq = poller->cq;
+			break;
+		}
+	}
+
+	if (rqpair->cq == NULL) {
+		SPDK_ERRLOG("Unable to find a cq for qpair %p on poll group %p\n", qpair, qpair->poll_group);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+nvme_rdma_poll_group_disconnect_qpair(struct spdk_nvme_qpair *qpair)
+{
+	struct nvme_rdma_qpair			*rqpair = nvme_rdma_qpair(qpair);
+	struct nvme_rdma_poll_group		*group;
+	struct nvme_rdma_destroyed_qpair	*destroyed_qpair;
+	enum nvme_qpair_state			state;
+
+	if (rqpair->poll_group_disconnect_in_progress) {
+		return -EINPROGRESS;
+	}
+
+	rqpair->poll_group_disconnect_in_progress = true;
+	state = nvme_qpair_get_state(qpair);
+	group = nvme_rdma_poll_group(qpair->poll_group);
+	rqpair->cq = NULL;
+
+	/*
+	 * We want to guard against an endless recursive loop while making
+	 * sure the qpair is disconnected before we disconnect it from the qpair.
+	 */
+	if (state > NVME_QPAIR_DISCONNECTING && state != NVME_QPAIR_DESTROYING) {
+		nvme_ctrlr_disconnect_qpair(qpair);
+	}
+
+	/*
+	 * If this fails, the system is in serious trouble,
+	 * just let the qpair get cleaned up immediately.
+	 */
+	destroyed_qpair = calloc(1, sizeof(*destroyed_qpair));
+	if (destroyed_qpair == NULL) {
+		return 0;
+	}
+
+	destroyed_qpair->destroyed_qpair_tracker = rqpair;
+	destroyed_qpair->completed_cycles = 0;
+	STAILQ_INSERT_TAIL(&group->destroyed_qpairs, destroyed_qpair, link);
+
+	rqpair->defer_deletion_to_pg = true;
+
+	rqpair->poll_group_disconnect_in_progress = false;
+	return 0;
+}
+
+static int
+nvme_rdma_poll_group_add(struct spdk_nvme_transport_poll_group *tgroup,
+			 struct spdk_nvme_qpair *qpair)
+{
+	return 0;
+}
+
+static int
+nvme_rdma_poll_group_remove(struct spdk_nvme_transport_poll_group *tgroup,
+			    struct spdk_nvme_qpair *qpair)
+{
+	if (qpair->poll_group_tailq_head == &tgroup->connected_qpairs) {
+		return nvme_poll_group_disconnect_qpair(qpair);
+	}
+
+	return 0;
+}
+
+static void
+nvme_rdma_poll_group_delete_qpair(struct nvme_rdma_poll_group *group,
+				  struct nvme_rdma_destroyed_qpair *qpair_tracker)
+{
+	struct nvme_rdma_qpair *rqpair = qpair_tracker->destroyed_qpair_tracker;
+
+	rqpair->defer_deletion_to_pg = false;
+	if (nvme_qpair_get_state(&rqpair->qpair) == NVME_QPAIR_DESTROYING) {
+		nvme_rdma_ctrlr_delete_io_qpair(rqpair->qpair.ctrlr, &rqpair->qpair);
+	}
+	STAILQ_REMOVE(&group->destroyed_qpairs, qpair_tracker, nvme_rdma_destroyed_qpair, link);
+	free(qpair_tracker);
+}
+
+static int64_t
+nvme_rdma_poll_group_process_completions(struct spdk_nvme_transport_poll_group *tgroup,
+		uint32_t completions_per_qpair, spdk_nvme_disconnected_qpair_cb disconnected_qpair_cb)
+{
+	struct spdk_nvme_qpair			*qpair, *tmp_qpair;
+	struct nvme_rdma_destroyed_qpair	*qpair_tracker, *tmp_qpair_tracker;
+	struct nvme_rdma_qpair			*rqpair;
+	struct nvme_rdma_poll_group		*group;
+	struct nvme_rdma_poller			*poller;
+	int					num_qpairs = 0, batch_size, rc;
+	int64_t					total_completions = 0;
+	uint64_t				completions_allowed = 0;
+	uint64_t				completions_per_poller = 0;
+	uint64_t				poller_completions = 0;
+
+
+	if (completions_per_qpair == 0) {
+		completions_per_qpair = MAX_COMPLETIONS_PER_POLL;
+	}
+
+	group = nvme_rdma_poll_group(tgroup);
+	STAILQ_FOREACH_SAFE(qpair, &tgroup->disconnected_qpairs, poll_group_stailq, tmp_qpair) {
+		disconnected_qpair_cb(qpair, tgroup->group->ctx);
+	}
+
+	STAILQ_FOREACH_SAFE(qpair, &tgroup->connected_qpairs, poll_group_stailq, tmp_qpair) {
+		rqpair = nvme_rdma_qpair(qpair);
+		rqpair->num_completions = 0;
+		nvme_rdma_qpair_process_cm_event(rqpair);
+
+		if (spdk_unlikely(qpair->transport_failure_reason != SPDK_NVME_QPAIR_FAILURE_NONE)) {
+			nvme_rdma_fail_qpair(qpair, 0);
+			disconnected_qpair_cb(qpair, tgroup->group->ctx);
+			continue;
+		}
+		num_qpairs++;
+	}
+
+	completions_allowed = completions_per_qpair * num_qpairs;
+	completions_per_poller = spdk_max(completions_allowed / group->num_pollers, 1);
+
+	STAILQ_FOREACH(poller, &group->pollers, link) {
+		poller_completions = 0;
+		do {
+			batch_size = spdk_min((completions_per_poller - poller_completions), MAX_COMPLETIONS_PER_POLL);
+			rc = nvme_rdma_cq_process_completions(poller->cq, batch_size, group, NULL);
+			if (rc <= 0) {
+				if (rc == -ECANCELED) {
+					return -EIO;
+				}
+				break;
+			}
+
+			poller_completions += rc;
+		} while (poller_completions < completions_per_poller);
+		total_completions += poller_completions;
+	}
+
+	STAILQ_FOREACH_SAFE(qpair, &tgroup->connected_qpairs, poll_group_stailq, tmp_qpair) {
+		rqpair = nvme_rdma_qpair(qpair);
+		if (spdk_unlikely(qpair->ctrlr->timeout_enabled)) {
+			nvme_rdma_qpair_check_timeout(qpair);
+		}
+
+		nvme_rdma_qpair_submit_sends(rqpair);
+		nvme_rdma_qpair_submit_recvs(rqpair);
+		nvme_qpair_resubmit_requests(&rqpair->qpair, rqpair->num_completions);
+	}
+
+	/*
+	 * Once a qpair is disconnected, we can still get flushed completions for those disconnected qpairs.
+	 * For most pieces of hardware, those requests will complete immediately. However, there are certain
+	 * cases where flushed requests will linger. Default is to destroy qpair after all completions are freed,
+	 * but have a fallback for other cases where we don't get all of our completions back.
+	 */
+	STAILQ_FOREACH_SAFE(qpair_tracker, &group->destroyed_qpairs, link, tmp_qpair_tracker) {
+		qpair_tracker->completed_cycles++;
+		rqpair = qpair_tracker->destroyed_qpair_tracker;
+		if ((rqpair->current_num_sends == 0 && rqpair->current_num_recvs == 0) ||
+		    qpair_tracker->completed_cycles > NVME_RDMA_DESTROYED_QPAIR_EXPIRATION_CYCLES) {
+			nvme_rdma_poll_group_delete_qpair(group, qpair_tracker);
+		}
+	}
+
+	return total_completions;
+}
+
+static int
+nvme_rdma_poll_group_destroy(struct spdk_nvme_transport_poll_group *tgroup)
+{
+	struct nvme_rdma_poll_group		*group = nvme_rdma_poll_group(tgroup);
+	struct nvme_rdma_destroyed_qpair	*qpair_tracker, *tmp_qpair_tracker;
+	struct nvme_rdma_qpair			*rqpair;
+
+	if (!STAILQ_EMPTY(&tgroup->connected_qpairs) || !STAILQ_EMPTY(&tgroup->disconnected_qpairs)) {
+		return -EBUSY;
+	}
+
+	STAILQ_FOREACH_SAFE(qpair_tracker, &group->destroyed_qpairs, link, tmp_qpair_tracker) {
+		rqpair = qpair_tracker->destroyed_qpair_tracker;
+		if (nvme_qpair_get_state(&rqpair->qpair) == NVME_QPAIR_DESTROYING) {
+			rqpair->defer_deletion_to_pg = false;
+			nvme_rdma_ctrlr_delete_io_qpair(rqpair->qpair.ctrlr, &rqpair->qpair);
+		}
+
+		STAILQ_REMOVE(&group->destroyed_qpairs, qpair_tracker, nvme_rdma_destroyed_qpair, link);
+		free(qpair_tracker);
+	}
+
+	nvme_rdma_poll_group_free_pollers(group);
+	free(group);
+
+	return 0;
+}
+
+void
+spdk_nvme_rdma_init_hooks(struct spdk_nvme_rdma_hooks *hooks)
+{
+	g_nvme_hooks = *hooks;
+}
+
+const struct spdk_nvme_transport_ops rdma_ops = {
+	.name = "RDMA",
+	.type = SPDK_NVME_TRANSPORT_RDMA,
+	.ctrlr_construct = nvme_rdma_ctrlr_construct,
+	.ctrlr_scan = nvme_fabric_ctrlr_scan,
+	.ctrlr_destruct = nvme_rdma_ctrlr_destruct,
+	.ctrlr_enable = nvme_rdma_ctrlr_enable,
+
+	.ctrlr_set_reg_4 = nvme_fabric_ctrlr_set_reg_4,
+	.ctrlr_set_reg_8 = nvme_fabric_ctrlr_set_reg_8,
+	.ctrlr_get_reg_4 = nvme_fabric_ctrlr_get_reg_4,
+	.ctrlr_get_reg_8 = nvme_fabric_ctrlr_get_reg_8,
+
+	.ctrlr_get_max_xfer_size = nvme_rdma_ctrlr_get_max_xfer_size,
+	.ctrlr_get_max_sges = nvme_rdma_ctrlr_get_max_sges,
+
+	.ctrlr_create_io_qpair = nvme_rdma_ctrlr_create_io_qpair,
+	.ctrlr_delete_io_qpair = nvme_rdma_ctrlr_delete_io_qpair,
+	.ctrlr_connect_qpair = nvme_rdma_ctrlr_connect_qpair,
+	.ctrlr_disconnect_qpair = nvme_rdma_ctrlr_disconnect_qpair,
+
+	.qpair_abort_reqs = nvme_rdma_qpair_abort_reqs,
+	.qpair_reset = nvme_rdma_qpair_reset,
+	.qpair_submit_request = nvme_rdma_qpair_submit_request,
+	.qpair_process_completions = nvme_rdma_qpair_process_completions,
+	.qpair_iterate_requests = nvme_rdma_qpair_iterate_requests,
+	.admin_qpair_abort_aers = nvme_rdma_admin_qpair_abort_aers,
+
+	.poll_group_create = nvme_rdma_poll_group_create,
+	.poll_group_connect_qpair = nvme_rdma_poll_group_connect_qpair,
+	.poll_group_disconnect_qpair = nvme_rdma_poll_group_disconnect_qpair,
+	.poll_group_add = nvme_rdma_poll_group_add,
+	.poll_group_remove = nvme_rdma_poll_group_remove,
+	.poll_group_process_completions = nvme_rdma_poll_group_process_completions,
+	.poll_group_destroy = nvme_rdma_poll_group_destroy,
+
+};
+
+SPDK_NVME_TRANSPORT_REGISTER(rdma, &rdma_ops);
diff --git a/src/spdk/lib/nvme/nvme_tcp.c b/src/spdk/lib/nvme/nvme_tcp.c
new file mode 100644
index 000000000..98e8c6827
--- /dev/null
+++ b/src/spdk/lib/nvme/nvme_tcp.c
@@ -0,0 +1,1973 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation. All rights reserved.
+ *   Copyright (c) 2020 Mellanox Technologies LTD. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * NVMe/TCP transport
+ */
+
+#include "nvme_internal.h"
+
+#include "spdk/endian.h"
+#include "spdk/likely.h"
+#include "spdk/string.h"
+#include "spdk/stdinc.h"
+#include "spdk/crc32.h"
+#include "spdk/endian.h"
+#include "spdk/assert.h"
+#include "spdk/string.h"
+#include "spdk/thread.h"
+#include "spdk/trace.h"
+#include "spdk/util.h"
+
+#include "spdk_internal/nvme_tcp.h"
+
+#define NVME_TCP_RW_BUFFER_SIZE 131072
+#define NVME_TCP_TIME_OUT_IN_SECONDS 2
+
+#define NVME_TCP_HPDA_DEFAULT			0
+#define NVME_TCP_MAX_R2T_DEFAULT		1
+#define NVME_TCP_PDU_H2C_MIN_DATA_SIZE		4096
+#define NVME_TCP_IN_CAPSULE_DATA_MAX_SIZE	8192
+
+/* NVMe TCP transport extensions for spdk_nvme_ctrlr */
+struct nvme_tcp_ctrlr {
+	struct spdk_nvme_ctrlr			ctrlr;
+};
+
+struct nvme_tcp_poll_group {
+	struct spdk_nvme_transport_poll_group group;
+	struct spdk_sock_group *sock_group;
+	uint32_t completions_per_qpair;
+	int64_t num_completions;
+};
+
+/* NVMe TCP qpair extensions for spdk_nvme_qpair */
+struct nvme_tcp_qpair {
+	struct spdk_nvme_qpair			qpair;
+	struct spdk_sock			*sock;
+
+	TAILQ_HEAD(, nvme_tcp_req)		free_reqs;
+	TAILQ_HEAD(, nvme_tcp_req)		outstanding_reqs;
+
+	TAILQ_HEAD(, nvme_tcp_pdu)		send_queue;
+	struct nvme_tcp_pdu			recv_pdu;
+	struct nvme_tcp_pdu			send_pdu; /* only for error pdu and init pdu */
+	struct nvme_tcp_pdu			*send_pdus; /* Used by tcp_reqs */
+	enum nvme_tcp_pdu_recv_state		recv_state;
+
+	struct nvme_tcp_req			*tcp_reqs;
+
+	uint16_t				num_entries;
+
+	bool					host_hdgst_enable;
+	bool					host_ddgst_enable;
+
+	/** Specifies the maximum number of PDU-Data bytes per H2C Data Transfer PDU */
+	uint32_t				maxh2cdata;
+
+	uint32_t				maxr2t;
+
+	/* 0 based value, which is used to guide the padding */
+	uint8_t					cpda;
+
+	enum nvme_tcp_qpair_state		state;
+};
+
+enum nvme_tcp_req_state {
+	NVME_TCP_REQ_FREE,
+	NVME_TCP_REQ_ACTIVE,
+	NVME_TCP_REQ_ACTIVE_R2T,
+};
+
+struct nvme_tcp_req {
+	struct nvme_request			*req;
+	enum nvme_tcp_req_state			state;
+	uint16_t				cid;
+	uint16_t				ttag;
+	uint32_t				datao;
+	uint32_t				r2tl_remain;
+	uint32_t				active_r2ts;
+	bool					in_capsule_data;
+	/* It is used to track whether the req can be safely freed */
+	struct {
+		uint8_t				send_ack : 1;
+		uint8_t				data_recv : 1;
+		uint8_t				r2t_recv : 1;
+		uint8_t				reserved : 5;
+	} ordering;
+	struct nvme_tcp_pdu			*send_pdu;
+	struct iovec				iov[NVME_TCP_MAX_SGL_DESCRIPTORS];
+	uint32_t				iovcnt;
+	struct nvme_tcp_qpair			*tqpair;
+	TAILQ_ENTRY(nvme_tcp_req)		link;
+};
+
+static void nvme_tcp_send_h2c_data(struct nvme_tcp_req *tcp_req);
+
+static inline struct nvme_tcp_qpair *
+nvme_tcp_qpair(struct spdk_nvme_qpair *qpair)
+{
+	assert(qpair->trtype == SPDK_NVME_TRANSPORT_TCP);
+	return SPDK_CONTAINEROF(qpair, struct nvme_tcp_qpair, qpair);
+}
+
+static inline struct nvme_tcp_poll_group *
+nvme_tcp_poll_group(struct spdk_nvme_transport_poll_group *group)
+{
+	return SPDK_CONTAINEROF(group, struct nvme_tcp_poll_group, group);
+}
+
+static inline struct nvme_tcp_ctrlr *
+nvme_tcp_ctrlr(struct spdk_nvme_ctrlr *ctrlr)
+{
+	assert(ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_TCP);
+	return SPDK_CONTAINEROF(ctrlr, struct nvme_tcp_ctrlr, ctrlr);
+}
+
+static struct nvme_tcp_req *
+nvme_tcp_req_get(struct nvme_tcp_qpair *tqpair)
+{
+	struct nvme_tcp_req *tcp_req;
+
+	tcp_req = TAILQ_FIRST(&tqpair->free_reqs);
+	if (!tcp_req) {
+		return NULL;
+	}
+
+	assert(tcp_req->state == NVME_TCP_REQ_FREE);
+	tcp_req->state = NVME_TCP_REQ_ACTIVE;
+	TAILQ_REMOVE(&tqpair->free_reqs, tcp_req, link);
+	tcp_req->datao = 0;
+	tcp_req->req = NULL;
+	tcp_req->in_capsule_data = false;
+	tcp_req->r2tl_remain = 0;
+	tcp_req->active_r2ts = 0;
+	tcp_req->iovcnt = 0;
+	tcp_req->ordering.send_ack = 0;
+	tcp_req->ordering.data_recv = 0;
+	tcp_req->ordering.r2t_recv = 0;
+	memset(tcp_req->send_pdu, 0, sizeof(struct nvme_tcp_pdu));
+	TAILQ_INSERT_TAIL(&tqpair->outstanding_reqs, tcp_req, link);
+
+	return tcp_req;
+}
+
+static void
+nvme_tcp_req_put(struct nvme_tcp_qpair *tqpair, struct nvme_tcp_req *tcp_req)
+{
+	assert(tcp_req->state != NVME_TCP_REQ_FREE);
+	tcp_req->state = NVME_TCP_REQ_FREE;
+	TAILQ_INSERT_HEAD(&tqpair->free_reqs, tcp_req, link);
+}
+
+static int
+nvme_tcp_parse_addr(struct sockaddr_storage *sa, int family, const char *addr, const char *service)
+{
+	struct addrinfo *res;
+	struct addrinfo hints;
+	int ret;
+
+	memset(&hints, 0, sizeof(hints));
+	hints.ai_family = family;
+	hints.ai_socktype = SOCK_STREAM;
+	hints.ai_protocol = 0;
+
+	ret = getaddrinfo(addr, service, &hints, &res);
+	if (ret) {
+		SPDK_ERRLOG("getaddrinfo failed: %s (%d)\n", gai_strerror(ret), ret);
+		return ret;
+	}
+
+	if (res->ai_addrlen > sizeof(*sa)) {
+		SPDK_ERRLOG("getaddrinfo() ai_addrlen %zu too large\n", (size_t)res->ai_addrlen);
+		ret = EINVAL;
+	} else {
+		memcpy(sa, res->ai_addr, res->ai_addrlen);
+	}
+
+	freeaddrinfo(res);
+	return ret;
+}
+
+static void
+nvme_tcp_free_reqs(struct nvme_tcp_qpair *tqpair)
+{
+	free(tqpair->tcp_reqs);
+	tqpair->tcp_reqs = NULL;
+
+	spdk_free(tqpair->send_pdus);
+	tqpair->send_pdus = NULL;
+}
+
+static int
+nvme_tcp_alloc_reqs(struct nvme_tcp_qpair *tqpair)
+{
+	uint16_t i;
+	struct nvme_tcp_req	*tcp_req;
+
+	tqpair->tcp_reqs = calloc(tqpair->num_entries, sizeof(struct nvme_tcp_req));
+	if (tqpair->tcp_reqs == NULL) {
+		SPDK_ERRLOG("Failed to allocate tcp_reqs on tqpair=%p\n", tqpair);
+		goto fail;
+	}
+
+	tqpair->send_pdus = spdk_zmalloc(tqpair->num_entries * sizeof(struct nvme_tcp_pdu),
+					 0x1000, NULL,
+					 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA);
+
+	if (tqpair->send_pdus == NULL) {
+		SPDK_ERRLOG("Failed to allocate send_pdus on tqpair=%p\n", tqpair);
+		goto fail;
+	}
+
+	TAILQ_INIT(&tqpair->send_queue);
+	TAILQ_INIT(&tqpair->free_reqs);
+	TAILQ_INIT(&tqpair->outstanding_reqs);
+	for (i = 0; i < tqpair->num_entries; i++) {
+		tcp_req = &tqpair->tcp_reqs[i];
+		tcp_req->cid = i;
+		tcp_req->tqpair = tqpair;
+		tcp_req->send_pdu = &tqpair->send_pdus[i];
+		TAILQ_INSERT_TAIL(&tqpair->free_reqs, tcp_req, link);
+	}
+
+	return 0;
+fail:
+	nvme_tcp_free_reqs(tqpair);
+	return -ENOMEM;
+}
+
+static void
+nvme_tcp_ctrlr_disconnect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)
+{
+	struct nvme_tcp_qpair *tqpair = nvme_tcp_qpair(qpair);
+	struct nvme_tcp_pdu *pdu;
+
+	spdk_sock_close(&tqpair->sock);
+
+	/* clear the send_queue */
+	while (!TAILQ_EMPTY(&tqpair->send_queue)) {
+		pdu = TAILQ_FIRST(&tqpair->send_queue);
+		/* Remove the pdu from the send_queue to prevent the wrong sending out
+		 * in the next round connection
+		 */
+		TAILQ_REMOVE(&tqpair->send_queue, pdu, tailq);
+	}
+}
+
+static void nvme_tcp_qpair_abort_reqs(struct spdk_nvme_qpair *qpair, uint32_t dnr);
+
+static int
+nvme_tcp_ctrlr_delete_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)
+{
+	struct nvme_tcp_qpair *tqpair;
+
+	if (!qpair) {
+		return -1;
+	}
+
+	nvme_transport_ctrlr_disconnect_qpair(ctrlr, qpair);
+	nvme_tcp_qpair_abort_reqs(qpair, 1);
+	nvme_qpair_deinit(qpair);
+	tqpair = nvme_tcp_qpair(qpair);
+	nvme_tcp_free_reqs(tqpair);
+	free(tqpair);
+
+	return 0;
+}
+
+static int
+nvme_tcp_ctrlr_enable(struct spdk_nvme_ctrlr *ctrlr)
+{
+	return 0;
+}
+
+static int
+nvme_tcp_ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr)
+{
+	struct nvme_tcp_ctrlr *tctrlr = nvme_tcp_ctrlr(ctrlr);
+
+	if (ctrlr->adminq) {
+		nvme_tcp_ctrlr_delete_io_qpair(ctrlr, ctrlr->adminq);
+	}
+
+	nvme_ctrlr_destruct_finish(ctrlr);
+
+	free(tctrlr);
+
+	return 0;
+}
+
+static void
+_pdu_write_done(void *cb_arg, int err)
+{
+	struct nvme_tcp_pdu *pdu = cb_arg;
+	struct nvme_tcp_qpair *tqpair = pdu->qpair;
+
+	TAILQ_REMOVE(&tqpair->send_queue, pdu, tailq);
+
+	if (err != 0) {
+		nvme_transport_ctrlr_disconnect_qpair(tqpair->qpair.ctrlr, &tqpair->qpair);
+		return;
+	}
+
+	assert(pdu->cb_fn != NULL);
+	pdu->cb_fn(pdu->cb_arg);
+}
+
+static int
+nvme_tcp_qpair_write_pdu(struct nvme_tcp_qpair *tqpair,
+			 struct nvme_tcp_pdu *pdu,
+			 nvme_tcp_qpair_xfer_complete_cb cb_fn,
+			 void *cb_arg)
+{
+	int hlen;
+	uint32_t crc32c;
+	uint32_t mapped_length = 0;
+
+	hlen = pdu->hdr.common.hlen;
+
+	/* Header Digest */
+	if (g_nvme_tcp_hdgst[pdu->hdr.common.pdu_type] && tqpair->host_hdgst_enable) {
+		crc32c = nvme_tcp_pdu_calc_header_digest(pdu);
+		MAKE_DIGEST_WORD((uint8_t *)pdu->hdr.raw + hlen, crc32c);
+	}
+
+	/* Data Digest */
+	if (pdu->data_len > 0 && g_nvme_tcp_ddgst[pdu->hdr.common.pdu_type] && tqpair->host_ddgst_enable) {
+		crc32c = nvme_tcp_pdu_calc_data_digest(pdu);
+		MAKE_DIGEST_WORD(pdu->data_digest, crc32c);
+	}
+
+	pdu->cb_fn = cb_fn;
+	pdu->cb_arg = cb_arg;
+
+	pdu->sock_req.iovcnt = nvme_tcp_build_iovs(pdu->iov, NVME_TCP_MAX_SGL_DESCRIPTORS, pdu,
+			       tqpair->host_hdgst_enable, tqpair->host_ddgst_enable,
+			       &mapped_length);
+	pdu->qpair = tqpair;
+	pdu->sock_req.cb_fn = _pdu_write_done;
+	pdu->sock_req.cb_arg = pdu;
+	TAILQ_INSERT_TAIL(&tqpair->send_queue, pdu, tailq);
+	spdk_sock_writev_async(tqpair->sock, &pdu->sock_req);
+
+	return 0;
+}
+
+/*
+ * Build SGL describing contiguous payload buffer.
+ */
+static int
+nvme_tcp_build_contig_request(struct nvme_tcp_qpair *tqpair, struct nvme_tcp_req *tcp_req)
+{
+	struct nvme_request *req = tcp_req->req;
+
+	tcp_req->iov[0].iov_base = req->payload.contig_or_cb_arg + req->payload_offset;
+	tcp_req->iov[0].iov_len = req->payload_size;
+	tcp_req->iovcnt = 1;
+
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "enter\n");
+
+	assert(nvme_payload_type(&req->payload) == NVME_PAYLOAD_TYPE_CONTIG);
+
+	return 0;
+}
+
+/*
+ * Build SGL describing scattered payload buffer.
+ */
+static int
+nvme_tcp_build_sgl_request(struct nvme_tcp_qpair *tqpair, struct nvme_tcp_req *tcp_req)
+{
+	int rc;
+	uint32_t length, remaining_size, iovcnt = 0, max_num_sgl;
+	struct nvme_request *req = tcp_req->req;
+
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "enter\n");
+
+	assert(req->payload_size != 0);
+	assert(nvme_payload_type(&req->payload) == NVME_PAYLOAD_TYPE_SGL);
+	assert(req->payload.reset_sgl_fn != NULL);
+	assert(req->payload.next_sge_fn != NULL);
+	req->payload.reset_sgl_fn(req->payload.contig_or_cb_arg, req->payload_offset);
+
+	max_num_sgl = spdk_min(req->qpair->ctrlr->max_sges, NVME_TCP_MAX_SGL_DESCRIPTORS);
+	remaining_size = req->payload_size;
+
+	do {
+		rc = req->payload.next_sge_fn(req->payload.contig_or_cb_arg, &tcp_req->iov[iovcnt].iov_base,
+					      &length);
+		if (rc) {
+			return -1;
+		}
+
+		length = spdk_min(length, remaining_size);
+		tcp_req->iov[iovcnt].iov_len = length;
+		remaining_size -= length;
+		iovcnt++;
+	} while (remaining_size > 0 && iovcnt < max_num_sgl);
+
+
+	/* Should be impossible if we did our sgl checks properly up the stack, but do a sanity check here. */
+	if (remaining_size > 0) {
+		SPDK_ERRLOG("Failed to construct tcp_req=%p, and the iovcnt=%u, remaining_size=%u\n",
+			    tcp_req, iovcnt, remaining_size);
+		return -1;
+	}
+
+	tcp_req->iovcnt = iovcnt;
+
+	return 0;
+}
+
+static int
+nvme_tcp_req_init(struct nvme_tcp_qpair *tqpair, struct nvme_request *req,
+		  struct nvme_tcp_req *tcp_req)
+{
+	struct spdk_nvme_ctrlr *ctrlr = tqpair->qpair.ctrlr;
+	int rc = 0;
+	enum spdk_nvme_data_transfer xfer;
+	uint32_t max_incapsule_data_size;
+
+	tcp_req->req = req;
+	req->cmd.cid = tcp_req->cid;
+	req->cmd.psdt = SPDK_NVME_PSDT_SGL_MPTR_CONTIG;
+	req->cmd.dptr.sgl1.unkeyed.type = SPDK_NVME_SGL_TYPE_TRANSPORT_DATA_BLOCK;
+	req->cmd.dptr.sgl1.unkeyed.subtype = SPDK_NVME_SGL_SUBTYPE_TRANSPORT;
+	req->cmd.dptr.sgl1.unkeyed.length = req->payload_size;
+
+	if (nvme_payload_type(&req->payload) == NVME_PAYLOAD_TYPE_CONTIG) {
+		rc = nvme_tcp_build_contig_request(tqpair, tcp_req);
+	} else if (nvme_payload_type(&req->payload) == NVME_PAYLOAD_TYPE_SGL) {
+		rc = nvme_tcp_build_sgl_request(tqpair, tcp_req);
+	} else {
+		rc = -1;
+	}
+
+	if (rc) {
+		return rc;
+	}
+
+	if (req->cmd.opc == SPDK_NVME_OPC_FABRIC) {
+		struct spdk_nvmf_capsule_cmd *nvmf_cmd = (struct spdk_nvmf_capsule_cmd *)&req->cmd;
+
+		xfer = spdk_nvme_opc_get_data_transfer(nvmf_cmd->fctype);
+	} else {
+		xfer = spdk_nvme_opc_get_data_transfer(req->cmd.opc);
+	}
+	if (xfer == SPDK_NVME_DATA_HOST_TO_CONTROLLER) {
+		max_incapsule_data_size = ctrlr->ioccsz_bytes;
+		if ((req->cmd.opc == SPDK_NVME_OPC_FABRIC) || nvme_qpair_is_admin_queue(&tqpair->qpair)) {
+			max_incapsule_data_size = spdk_min(max_incapsule_data_size, NVME_TCP_IN_CAPSULE_DATA_MAX_SIZE);
+		}
+
+		if (req->payload_size <= max_incapsule_data_size) {
+			req->cmd.dptr.sgl1.unkeyed.type = SPDK_NVME_SGL_TYPE_DATA_BLOCK;
+			req->cmd.dptr.sgl1.unkeyed.subtype = SPDK_NVME_SGL_SUBTYPE_OFFSET;
+			req->cmd.dptr.sgl1.address = 0;
+			tcp_req->in_capsule_data = true;
+		}
+	}
+
+	return 0;
+}
+
+static inline void
+nvme_tcp_req_put_safe(struct nvme_tcp_req *tcp_req)
+{
+	if (tcp_req->ordering.send_ack && tcp_req->ordering.data_recv) {
+		assert(tcp_req->state == NVME_TCP_REQ_ACTIVE);
+		assert(tcp_req->tqpair != NULL);
+		nvme_tcp_req_put(tcp_req->tqpair, tcp_req);
+	}
+}
+
+static void
+nvme_tcp_qpair_cmd_send_complete(void *cb_arg)
+{
+	struct nvme_tcp_req *tcp_req = cb_arg;
+
+	tcp_req->ordering.send_ack = 1;
+	/* Handle the r2t case */
+	if (spdk_unlikely(tcp_req->ordering.r2t_recv)) {
+		nvme_tcp_send_h2c_data(tcp_req);
+	} else {
+		nvme_tcp_req_put_safe(tcp_req);
+	}
+}
+
+static int
+nvme_tcp_qpair_capsule_cmd_send(struct nvme_tcp_qpair *tqpair,
+				struct nvme_tcp_req *tcp_req)
+{
+	struct nvme_tcp_pdu *pdu;
+	struct spdk_nvme_tcp_cmd *capsule_cmd;
+	uint32_t plen = 0, alignment;
+	uint8_t pdo;
+
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "enter\n");
+	pdu = tcp_req->send_pdu;
+
+	capsule_cmd = &pdu->hdr.capsule_cmd;
+	capsule_cmd->common.pdu_type = SPDK_NVME_TCP_PDU_TYPE_CAPSULE_CMD;
+	plen = capsule_cmd->common.hlen = sizeof(*capsule_cmd);
+	capsule_cmd->ccsqe = tcp_req->req->cmd;
+
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "capsule_cmd cid=%u on tqpair(%p)\n", tcp_req->req->cmd.cid, tqpair);
+
+	if (tqpair->host_hdgst_enable) {
+		SPDK_DEBUGLOG(SPDK_LOG_NVME, "Header digest is enabled for capsule command on tcp_req=%p\n",
+			      tcp_req);
+		capsule_cmd->common.flags |= SPDK_NVME_TCP_CH_FLAGS_HDGSTF;
+		plen += SPDK_NVME_TCP_DIGEST_LEN;
+	}
+
+	if ((tcp_req->req->payload_size == 0) || !tcp_req->in_capsule_data) {
+		goto end;
+	}
+
+	pdo = plen;
+	pdu->padding_len = 0;
+	if (tqpair->cpda) {
+		alignment = (tqpair->cpda + 1) << 2;
+		if (alignment > plen) {
+			pdu->padding_len = alignment - plen;
+			pdo = alignment;
+			plen = alignment;
+		}
+	}
+
+	capsule_cmd->common.pdo = pdo;
+	plen += tcp_req->req->payload_size;
+	if (tqpair->host_ddgst_enable) {
+		capsule_cmd->common.flags |= SPDK_NVME_TCP_CH_FLAGS_DDGSTF;
+		plen += SPDK_NVME_TCP_DIGEST_LEN;
+	}
+
+	tcp_req->datao = 0;
+	nvme_tcp_pdu_set_data_buf(pdu, tcp_req->iov, tcp_req->iovcnt,
+				  0, tcp_req->req->payload_size);
+end:
+	capsule_cmd->common.plen = plen;
+	return nvme_tcp_qpair_write_pdu(tqpair, pdu, nvme_tcp_qpair_cmd_send_complete, tcp_req);
+
+}
+
+static int
+nvme_tcp_qpair_submit_request(struct spdk_nvme_qpair *qpair,
+			      struct nvme_request *req)
+{
+	struct nvme_tcp_qpair *tqpair;
+	struct nvme_tcp_req *tcp_req;
+
+	tqpair = nvme_tcp_qpair(qpair);
+	assert(tqpair != NULL);
+	assert(req != NULL);
+
+	tcp_req = nvme_tcp_req_get(tqpair);
+	if (!tcp_req) {
+		/* Inform the upper layer to try again later. */
+		return -EAGAIN;
+	}
+
+	if (nvme_tcp_req_init(tqpair, req, tcp_req)) {
+		SPDK_ERRLOG("nvme_tcp_req_init() failed\n");
+		TAILQ_REMOVE(&tcp_req->tqpair->outstanding_reqs, tcp_req, link);
+		nvme_tcp_req_put(tqpair, tcp_req);
+		return -1;
+	}
+
+	return nvme_tcp_qpair_capsule_cmd_send(tqpair, tcp_req);
+}
+
+static int
+nvme_tcp_qpair_reset(struct spdk_nvme_qpair *qpair)
+{
+	return 0;
+}
+
+static void
+nvme_tcp_req_complete(struct nvme_tcp_req *tcp_req,
+		      struct spdk_nvme_cpl *rsp)
+{
+	struct nvme_request *req;
+
+	assert(tcp_req->req != NULL);
+	req = tcp_req->req;
+
+	TAILQ_REMOVE(&tcp_req->tqpair->outstanding_reqs, tcp_req, link);
+	nvme_complete_request(req->cb_fn, req->cb_arg, req->qpair, req, rsp);
+	nvme_free_request(req);
+}
+
+static void
+nvme_tcp_qpair_abort_reqs(struct spdk_nvme_qpair *qpair, uint32_t dnr)
+{
+	struct nvme_tcp_req *tcp_req, *tmp;
+	struct spdk_nvme_cpl cpl;
+	struct nvme_tcp_qpair *tqpair = nvme_tcp_qpair(qpair);
+
+	cpl.status.sc = SPDK_NVME_SC_ABORTED_SQ_DELETION;
+	cpl.status.sct = SPDK_NVME_SCT_GENERIC;
+	cpl.status.dnr = dnr;
+
+	TAILQ_FOREACH_SAFE(tcp_req, &tqpair->outstanding_reqs, link, tmp) {
+		nvme_tcp_req_complete(tcp_req, &cpl);
+		nvme_tcp_req_put(tqpair, tcp_req);
+	}
+}
+
+static void
+nvme_tcp_qpair_set_recv_state(struct nvme_tcp_qpair *tqpair,
+			      enum nvme_tcp_pdu_recv_state state)
+{
+	if (tqpair->recv_state == state) {
+		SPDK_ERRLOG("The recv state of tqpair=%p is same with the state(%d) to be set\n",
+			    tqpair, state);
+		return;
+	}
+
+	tqpair->recv_state = state;
+	switch (state) {
+	case NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY:
+	case NVME_TCP_PDU_RECV_STATE_ERROR:
+		memset(&tqpair->recv_pdu, 0, sizeof(struct nvme_tcp_pdu));
+		break;
+	case NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_CH:
+	case NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PSH:
+	case NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PAYLOAD:
+	default:
+		break;
+	}
+}
+
+static void
+nvme_tcp_qpair_send_h2c_term_req_complete(void *cb_arg)
+{
+	struct nvme_tcp_qpair *tqpair = cb_arg;
+
+	tqpair->state = NVME_TCP_QPAIR_STATE_EXITING;
+}
+
+static void
+nvme_tcp_qpair_send_h2c_term_req(struct nvme_tcp_qpair *tqpair, struct nvme_tcp_pdu *pdu,
+				 enum spdk_nvme_tcp_term_req_fes fes, uint32_t error_offset)
+{
+	struct nvme_tcp_pdu *rsp_pdu;
+	struct spdk_nvme_tcp_term_req_hdr *h2c_term_req;
+	uint32_t h2c_term_req_hdr_len = sizeof(*h2c_term_req);
+	uint8_t copy_len;
+
+	rsp_pdu = &tqpair->send_pdu;
+	memset(rsp_pdu, 0, sizeof(*rsp_pdu));
+	h2c_term_req = &rsp_pdu->hdr.term_req;
+	h2c_term_req->common.pdu_type = SPDK_NVME_TCP_PDU_TYPE_H2C_TERM_REQ;
+	h2c_term_req->common.hlen = h2c_term_req_hdr_len;
+
+	if ((fes == SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD) ||
+	    (fes == SPDK_NVME_TCP_TERM_REQ_FES_INVALID_DATA_UNSUPPORTED_PARAMETER)) {
+		DSET32(&h2c_term_req->fei, error_offset);
+	}
+
+	copy_len = pdu->hdr.common.hlen;
+	if (copy_len > SPDK_NVME_TCP_TERM_REQ_ERROR_DATA_MAX_SIZE) {
+		copy_len = SPDK_NVME_TCP_TERM_REQ_ERROR_DATA_MAX_SIZE;
+	}
+
+	/* Copy the error info into the buffer */
+	memcpy((uint8_t *)rsp_pdu->hdr.raw + h2c_term_req_hdr_len, pdu->hdr.raw, copy_len);
+	nvme_tcp_pdu_set_data(rsp_pdu, (uint8_t *)rsp_pdu->hdr.raw + h2c_term_req_hdr_len, copy_len);
+
+	/* Contain the header len of the wrong received pdu */
+	h2c_term_req->common.plen = h2c_term_req->common.hlen + copy_len;
+	nvme_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_ERROR);
+	nvme_tcp_qpair_write_pdu(tqpair, rsp_pdu, nvme_tcp_qpair_send_h2c_term_req_complete, NULL);
+
+}
+
+static void
+nvme_tcp_pdu_ch_handle(struct nvme_tcp_qpair *tqpair)
+{
+	struct nvme_tcp_pdu *pdu;
+	uint32_t error_offset = 0;
+	enum spdk_nvme_tcp_term_req_fes fes;
+	uint32_t expected_hlen, hd_len = 0;
+	bool plen_error = false;
+
+	pdu = &tqpair->recv_pdu;
+
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "pdu type = %d\n", pdu->hdr.common.pdu_type);
+	if (pdu->hdr.common.pdu_type == SPDK_NVME_TCP_PDU_TYPE_IC_RESP) {
+		if (tqpair->state != NVME_TCP_QPAIR_STATE_INVALID) {
+			SPDK_ERRLOG("Already received IC_RESP PDU, and we should reject this pdu=%p\n", pdu);
+			fes = SPDK_NVME_TCP_TERM_REQ_FES_PDU_SEQUENCE_ERROR;
+			goto err;
+		}
+		expected_hlen = sizeof(struct spdk_nvme_tcp_ic_resp);
+		if (pdu->hdr.common.plen != expected_hlen) {
+			plen_error = true;
+		}
+	} else {
+		if (tqpair->state != NVME_TCP_QPAIR_STATE_RUNNING) {
+			SPDK_ERRLOG("The TCP/IP tqpair connection is not negotitated\n");
+			fes = SPDK_NVME_TCP_TERM_REQ_FES_PDU_SEQUENCE_ERROR;
+			goto err;
+		}
+
+		switch (pdu->hdr.common.pdu_type) {
+		case SPDK_NVME_TCP_PDU_TYPE_CAPSULE_RESP:
+			expected_hlen = sizeof(struct spdk_nvme_tcp_rsp);
+			if (pdu->hdr.common.flags & SPDK_NVME_TCP_CH_FLAGS_HDGSTF) {
+				hd_len = SPDK_NVME_TCP_DIGEST_LEN;
+			}
+
+			if (pdu->hdr.common.plen != (expected_hlen + hd_len)) {
+				plen_error = true;
+			}
+			break;
+		case SPDK_NVME_TCP_PDU_TYPE_C2H_DATA:
+			expected_hlen = sizeof(struct spdk_nvme_tcp_c2h_data_hdr);
+			if (pdu->hdr.common.plen < pdu->hdr.common.pdo) {
+				plen_error = true;
+			}
+			break;
+		case SPDK_NVME_TCP_PDU_TYPE_C2H_TERM_REQ:
+			expected_hlen = sizeof(struct spdk_nvme_tcp_term_req_hdr);
+			if ((pdu->hdr.common.plen <= expected_hlen) ||
+			    (pdu->hdr.common.plen > SPDK_NVME_TCP_TERM_REQ_PDU_MAX_SIZE)) {
+				plen_error = true;
+			}
+			break;
+		case SPDK_NVME_TCP_PDU_TYPE_R2T:
+			expected_hlen = sizeof(struct spdk_nvme_tcp_r2t_hdr);
+			if (pdu->hdr.common.flags & SPDK_NVME_TCP_CH_FLAGS_HDGSTF) {
+				hd_len = SPDK_NVME_TCP_DIGEST_LEN;
+			}
+
+			if (pdu->hdr.common.plen != (expected_hlen + hd_len)) {
+				plen_error = true;
+			}
+			break;
+
+		default:
+			SPDK_ERRLOG("Unexpected PDU type 0x%02x\n", tqpair->recv_pdu.hdr.common.pdu_type);
+			fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD;
+			error_offset = offsetof(struct spdk_nvme_tcp_common_pdu_hdr, pdu_type);
+			goto err;
+		}
+	}
+
+	if (pdu->hdr.common.hlen != expected_hlen) {
+		SPDK_ERRLOG("Expected PDU header length %u, got %u\n",
+			    expected_hlen, pdu->hdr.common.hlen);
+		fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD;
+		error_offset = offsetof(struct spdk_nvme_tcp_common_pdu_hdr, hlen);
+		goto err;
+
+	} else if (plen_error) {
+		fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD;
+		error_offset = offsetof(struct spdk_nvme_tcp_common_pdu_hdr, plen);
+		goto err;
+	} else {
+		nvme_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PSH);
+		nvme_tcp_pdu_calc_psh_len(&tqpair->recv_pdu, tqpair->host_hdgst_enable);
+		return;
+	}
+err:
+	nvme_tcp_qpair_send_h2c_term_req(tqpair, pdu, fes, error_offset);
+}
+
+static struct nvme_tcp_req *
+get_nvme_active_req_by_cid(struct nvme_tcp_qpair *tqpair, uint32_t cid)
+{
+	assert(tqpair != NULL);
+	if ((cid >= tqpair->num_entries) || (tqpair->tcp_reqs[cid].state == NVME_TCP_REQ_FREE)) {
+		return NULL;
+	}
+
+	return &tqpair->tcp_reqs[cid];
+}
+
+static void
+nvme_tcp_c2h_data_payload_handle(struct nvme_tcp_qpair *tqpair,
+				 struct nvme_tcp_pdu *pdu, uint32_t *reaped)
+{
+	struct nvme_tcp_req *tcp_req;
+	struct spdk_nvme_tcp_c2h_data_hdr *c2h_data;
+	struct spdk_nvme_cpl cpl = {};
+	uint8_t flags;
+
+	tcp_req = pdu->req;
+	assert(tcp_req != NULL);
+
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "enter\n");
+	c2h_data = &pdu->hdr.c2h_data;
+	tcp_req->datao += pdu->data_len;
+	flags = c2h_data->common.flags;
+
+	nvme_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY);
+	if (flags & SPDK_NVME_TCP_C2H_DATA_FLAGS_SUCCESS) {
+		if (tcp_req->datao == tcp_req->req->payload_size) {
+			cpl.status.p = 0;
+		} else {
+			cpl.status.p = 1;
+		}
+
+		cpl.cid = tcp_req->cid;
+		cpl.sqid = tqpair->qpair.id;
+		nvme_tcp_req_complete(tcp_req, &cpl);
+		if (tcp_req->ordering.send_ack) {
+			(*reaped)++;
+		}
+
+		tcp_req->ordering.data_recv = 1;
+		nvme_tcp_req_put_safe(tcp_req);
+	}
+}
+
+static const char *spdk_nvme_tcp_term_req_fes_str[] = {
+	"Invalid PDU Header Field",
+	"PDU Sequence Error",
+	"Header Digest Error",
+	"Data Transfer Out of Range",
+	"Data Transfer Limit Exceeded",
+	"Unsupported parameter",
+};
+
+static void
+nvme_tcp_c2h_term_req_dump(struct spdk_nvme_tcp_term_req_hdr *c2h_term_req)
+{
+	SPDK_ERRLOG("Error info of pdu(%p): %s\n", c2h_term_req,
+		    spdk_nvme_tcp_term_req_fes_str[c2h_term_req->fes]);
+	if ((c2h_term_req->fes == SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD) ||
+	    (c2h_term_req->fes == SPDK_NVME_TCP_TERM_REQ_FES_INVALID_DATA_UNSUPPORTED_PARAMETER)) {
+		SPDK_DEBUGLOG(SPDK_LOG_NVME, "The offset from the start of the PDU header is %u\n",
+			      DGET32(c2h_term_req->fei));
+	}
+	/* we may also need to dump some other info here */
+}
+
+static void
+nvme_tcp_c2h_term_req_payload_handle(struct nvme_tcp_qpair *tqpair,
+				     struct nvme_tcp_pdu *pdu)
+{
+	nvme_tcp_c2h_term_req_dump(&pdu->hdr.term_req);
+	nvme_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_ERROR);
+}
+
+static void
+nvme_tcp_pdu_payload_handle(struct nvme_tcp_qpair *tqpair,
+			    uint32_t *reaped)
+{
+	int rc = 0;
+	struct nvme_tcp_pdu *pdu;
+	uint32_t crc32c, error_offset = 0;
+	enum spdk_nvme_tcp_term_req_fes fes;
+
+	assert(tqpair->recv_state == NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PAYLOAD);
+	pdu = &tqpair->recv_pdu;
+
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "enter\n");
+
+	/* check data digest if need */
+	if (pdu->ddgst_enable) {
+		crc32c = nvme_tcp_pdu_calc_data_digest(pdu);
+		rc = MATCH_DIGEST_WORD(pdu->data_digest, crc32c);
+		if (rc == 0) {
+			SPDK_ERRLOG("data digest error on tqpair=(%p) with pdu=%p\n", tqpair, pdu);
+			fes = SPDK_NVME_TCP_TERM_REQ_FES_HDGST_ERROR;
+			nvme_tcp_qpair_send_h2c_term_req(tqpair, pdu, fes, error_offset);
+			return;
+		}
+	}
+
+	switch (pdu->hdr.common.pdu_type) {
+	case SPDK_NVME_TCP_PDU_TYPE_C2H_DATA:
+		nvme_tcp_c2h_data_payload_handle(tqpair, pdu, reaped);
+		break;
+
+	case SPDK_NVME_TCP_PDU_TYPE_C2H_TERM_REQ:
+		nvme_tcp_c2h_term_req_payload_handle(tqpair, pdu);
+		break;
+
+	default:
+		/* The code should not go to here */
+		SPDK_ERRLOG("The code should not go to here\n");
+		break;
+	}
+}
+
+static void
+nvme_tcp_send_icreq_complete(void *cb_arg)
+{
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "Complete the icreq send for tqpair=%p\n",
+		      (struct nvme_tcp_qpair *)cb_arg);
+}
+
+static void
+nvme_tcp_icresp_handle(struct nvme_tcp_qpair *tqpair,
+		       struct nvme_tcp_pdu *pdu)
+{
+	struct spdk_nvme_tcp_ic_resp *ic_resp = &pdu->hdr.ic_resp;
+	uint32_t error_offset = 0;
+	enum spdk_nvme_tcp_term_req_fes fes;
+	int recv_buf_size;
+
+	/* Only PFV 0 is defined currently */
+	if (ic_resp->pfv != 0) {
+		SPDK_ERRLOG("Expected ICResp PFV %u, got %u\n", 0u, ic_resp->pfv);
+		fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD;
+		error_offset = offsetof(struct spdk_nvme_tcp_ic_resp, pfv);
+		goto end;
+	}
+
+	if (ic_resp->maxh2cdata < NVME_TCP_PDU_H2C_MIN_DATA_SIZE) {
+		SPDK_ERRLOG("Expected ICResp maxh2cdata >=%u, got %u\n", NVME_TCP_PDU_H2C_MIN_DATA_SIZE,
+			    ic_resp->maxh2cdata);
+		fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD;
+		error_offset = offsetof(struct spdk_nvme_tcp_ic_resp, maxh2cdata);
+		goto end;
+	}
+	tqpair->maxh2cdata = ic_resp->maxh2cdata;
+
+	if (ic_resp->cpda > SPDK_NVME_TCP_CPDA_MAX) {
+		SPDK_ERRLOG("Expected ICResp cpda <=%u, got %u\n", SPDK_NVME_TCP_CPDA_MAX, ic_resp->cpda);
+		fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD;
+		error_offset = offsetof(struct spdk_nvme_tcp_ic_resp, cpda);
+		goto end;
+	}
+	tqpair->cpda = ic_resp->cpda;
+
+	tqpair->host_hdgst_enable = ic_resp->dgst.bits.hdgst_enable ? true : false;
+	tqpair->host_ddgst_enable = ic_resp->dgst.bits.ddgst_enable ? true : false;
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "host_hdgst_enable: %u\n", tqpair->host_hdgst_enable);
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "host_ddgst_enable: %u\n", tqpair->host_ddgst_enable);
+
+	/* Now that we know whether digests are enabled, properly size the receive buffer to
+	 * handle several incoming 4K read commands according to SPDK_NVMF_TCP_RECV_BUF_SIZE_FACTOR
+	 * parameter. */
+	recv_buf_size = 0x1000 + sizeof(struct spdk_nvme_tcp_c2h_data_hdr);
+
+	if (tqpair->host_hdgst_enable) {
+		recv_buf_size += SPDK_NVME_TCP_DIGEST_LEN;
+	}
+
+	if (tqpair->host_ddgst_enable) {
+		recv_buf_size += SPDK_NVME_TCP_DIGEST_LEN;
+	}
+
+	if (spdk_sock_set_recvbuf(tqpair->sock, recv_buf_size * SPDK_NVMF_TCP_RECV_BUF_SIZE_FACTOR) < 0) {
+		SPDK_WARNLOG("Unable to allocate enough memory for receive buffer on tqpair=%p with size=%d\n",
+			     tqpair,
+			     recv_buf_size);
+		/* Not fatal. */
+	}
+
+	tqpair->state = NVME_TCP_QPAIR_STATE_RUNNING;
+	nvme_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY);
+	return;
+end:
+	nvme_tcp_qpair_send_h2c_term_req(tqpair, pdu, fes, error_offset);
+	return;
+}
+
+static void
+nvme_tcp_capsule_resp_hdr_handle(struct nvme_tcp_qpair *tqpair, struct nvme_tcp_pdu *pdu,
+				 uint32_t *reaped)
+{
+	struct nvme_tcp_req *tcp_req;
+	struct spdk_nvme_tcp_rsp *capsule_resp = &pdu->hdr.capsule_resp;
+	uint32_t cid, error_offset = 0;
+	enum spdk_nvme_tcp_term_req_fes fes;
+	struct spdk_nvme_cpl cpl;
+
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "enter\n");
+	cpl = capsule_resp->rccqe;
+	cid = cpl.cid;
+
+	/* Recv the pdu again */
+	nvme_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY);
+
+	tcp_req = get_nvme_active_req_by_cid(tqpair, cid);
+	if (!tcp_req) {
+		SPDK_ERRLOG("no tcp_req is found with cid=%u for tqpair=%p\n", cid, tqpair);
+		fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD;
+		error_offset = offsetof(struct spdk_nvme_tcp_rsp, rccqe);
+		goto end;
+
+	}
+
+	nvme_tcp_req_complete(tcp_req, &cpl);
+	if (tcp_req->ordering.send_ack) {
+		(*reaped)++;
+	}
+
+	tcp_req->ordering.data_recv = 1;
+	nvme_tcp_req_put_safe(tcp_req);
+
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "complete tcp_req(%p) on tqpair=%p\n", tcp_req, tqpair);
+
+	return;
+
+end:
+	nvme_tcp_qpair_send_h2c_term_req(tqpair, pdu, fes, error_offset);
+	return;
+}
+
+static void
+nvme_tcp_c2h_term_req_hdr_handle(struct nvme_tcp_qpair *tqpair,
+				 struct nvme_tcp_pdu *pdu)
+{
+	struct spdk_nvme_tcp_term_req_hdr *c2h_term_req = &pdu->hdr.term_req;
+	uint32_t error_offset = 0;
+	enum spdk_nvme_tcp_term_req_fes fes;
+
+	if (c2h_term_req->fes > SPDK_NVME_TCP_TERM_REQ_FES_INVALID_DATA_UNSUPPORTED_PARAMETER) {
+		SPDK_ERRLOG("Fatal Error Stauts(FES) is unknown for c2h_term_req pdu=%p\n", pdu);
+		fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD;
+		error_offset = offsetof(struct spdk_nvme_tcp_term_req_hdr, fes);
+		goto end;
+	}
+
+	/* set the data buffer */
+	nvme_tcp_pdu_set_data(pdu, (uint8_t *)pdu->hdr.raw + c2h_term_req->common.hlen,
+			      c2h_term_req->common.plen - c2h_term_req->common.hlen);
+	nvme_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PAYLOAD);
+	return;
+end:
+	nvme_tcp_qpair_send_h2c_term_req(tqpair, pdu, fes, error_offset);
+	return;
+}
+
+static void
+nvme_tcp_c2h_data_hdr_handle(struct nvme_tcp_qpair *tqpair, struct nvme_tcp_pdu *pdu)
+{
+	struct nvme_tcp_req *tcp_req;
+	struct spdk_nvme_tcp_c2h_data_hdr *c2h_data = &pdu->hdr.c2h_data;
+	uint32_t error_offset = 0;
+	enum spdk_nvme_tcp_term_req_fes fes;
+
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "enter\n");
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "c2h_data info on tqpair(%p): datao=%u, datal=%u, cccid=%d\n",
+		      tqpair, c2h_data->datao, c2h_data->datal, c2h_data->cccid);
+	tcp_req = get_nvme_active_req_by_cid(tqpair, c2h_data->cccid);
+	if (!tcp_req) {
+		SPDK_ERRLOG("no tcp_req found for c2hdata cid=%d\n", c2h_data->cccid);
+		fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD;
+		error_offset = offsetof(struct spdk_nvme_tcp_c2h_data_hdr, cccid);
+		goto end;
+
+	}
+
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "tcp_req(%p) on tqpair(%p): datao=%u, payload_size=%u\n",
+		      tcp_req, tqpair, tcp_req->datao, tcp_req->req->payload_size);
+
+	if (c2h_data->datal > tcp_req->req->payload_size) {
+		SPDK_ERRLOG("Invalid datal for tcp_req(%p), datal(%u) exceeds payload_size(%u)\n",
+			    tcp_req, c2h_data->datal, tcp_req->req->payload_size);
+		fes = SPDK_NVME_TCP_TERM_REQ_FES_DATA_TRANSFER_OUT_OF_RANGE;
+		goto end;
+	}
+
+	if (tcp_req->datao != c2h_data->datao) {
+		SPDK_ERRLOG("Invalid datao for tcp_req(%p), received datal(%u) != datao(%u) in tcp_req\n",
+			    tcp_req, c2h_data->datao, tcp_req->datao);
+		fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD;
+		error_offset = offsetof(struct spdk_nvme_tcp_c2h_data_hdr, datao);
+		goto end;
+	}
+
+	if ((c2h_data->datao + c2h_data->datal) > tcp_req->req->payload_size) {
+		SPDK_ERRLOG("Invalid data range for tcp_req(%p), received (datao(%u) + datal(%u)) > datao(%u) in tcp_req\n",
+			    tcp_req, c2h_data->datao, c2h_data->datal, tcp_req->req->payload_size);
+		fes = SPDK_NVME_TCP_TERM_REQ_FES_DATA_TRANSFER_OUT_OF_RANGE;
+		error_offset = offsetof(struct spdk_nvme_tcp_c2h_data_hdr, datal);
+		goto end;
+
+	}
+
+	nvme_tcp_pdu_set_data_buf(pdu, tcp_req->iov, tcp_req->iovcnt,
+				  c2h_data->datao, c2h_data->datal);
+	pdu->req = tcp_req;
+
+	nvme_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PAYLOAD);
+	return;
+
+end:
+	nvme_tcp_qpair_send_h2c_term_req(tqpair, pdu, fes, error_offset);
+	return;
+}
+
+static void
+nvme_tcp_qpair_h2c_data_send_complete(void *cb_arg)
+{
+	struct nvme_tcp_req *tcp_req = cb_arg;
+
+	assert(tcp_req != NULL);
+
+	tcp_req->ordering.send_ack = 1;
+	if (tcp_req->r2tl_remain) {
+		nvme_tcp_send_h2c_data(tcp_req);
+	} else {
+		assert(tcp_req->active_r2ts > 0);
+		tcp_req->active_r2ts--;
+		tcp_req->state = NVME_TCP_REQ_ACTIVE;
+		/* Need also call this function to free the resource */
+		nvme_tcp_req_put_safe(tcp_req);
+	}
+}
+
+static void
+nvme_tcp_send_h2c_data(struct nvme_tcp_req *tcp_req)
+{
+	struct nvme_tcp_qpair *tqpair = nvme_tcp_qpair(tcp_req->req->qpair);
+	struct nvme_tcp_pdu *rsp_pdu;
+	struct spdk_nvme_tcp_h2c_data_hdr *h2c_data;
+	uint32_t plen, pdo, alignment;
+
+	/* Reinit the send_ack and r2t_recv bits */
+	tcp_req->ordering.send_ack = 0;
+	tcp_req->ordering.r2t_recv = 0;
+	rsp_pdu = tcp_req->send_pdu;
+	memset(rsp_pdu, 0, sizeof(*rsp_pdu));
+	h2c_data = &rsp_pdu->hdr.h2c_data;
+
+	h2c_data->common.pdu_type = SPDK_NVME_TCP_PDU_TYPE_H2C_DATA;
+	plen = h2c_data->common.hlen = sizeof(*h2c_data);
+	h2c_data->cccid = tcp_req->cid;
+	h2c_data->ttag = tcp_req->ttag;
+	h2c_data->datao = tcp_req->datao;
+
+	h2c_data->datal = spdk_min(tcp_req->r2tl_remain, tqpair->maxh2cdata);
+	nvme_tcp_pdu_set_data_buf(rsp_pdu, tcp_req->iov, tcp_req->iovcnt,
+				  h2c_data->datao, h2c_data->datal);
+	tcp_req->r2tl_remain -= h2c_data->datal;
+
+	if (tqpair->host_hdgst_enable) {
+		h2c_data->common.flags |= SPDK_NVME_TCP_CH_FLAGS_HDGSTF;
+		plen += SPDK_NVME_TCP_DIGEST_LEN;
+	}
+
+	rsp_pdu->padding_len = 0;
+	pdo = plen;
+	if (tqpair->cpda) {
+		alignment = (tqpair->cpda + 1) << 2;
+		if (alignment > plen) {
+			rsp_pdu->padding_len = alignment - plen;
+			pdo = plen = alignment;
+		}
+	}
+
+	h2c_data->common.pdo = pdo;
+	plen += h2c_data->datal;
+	if (tqpair->host_ddgst_enable) {
+		h2c_data->common.flags |= SPDK_NVME_TCP_CH_FLAGS_DDGSTF;
+		plen += SPDK_NVME_TCP_DIGEST_LEN;
+	}
+
+	h2c_data->common.plen = plen;
+	tcp_req->datao += h2c_data->datal;
+	if (!tcp_req->r2tl_remain) {
+		h2c_data->common.flags |= SPDK_NVME_TCP_H2C_DATA_FLAGS_LAST_PDU;
+	}
+
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "h2c_data info: datao=%u, datal=%u, pdu_len=%u for tqpair=%p\n",
+		      h2c_data->datao, h2c_data->datal, h2c_data->common.plen, tqpair);
+
+	nvme_tcp_qpair_write_pdu(tqpair, rsp_pdu, nvme_tcp_qpair_h2c_data_send_complete, tcp_req);
+}
+
+static void
+nvme_tcp_r2t_hdr_handle(struct nvme_tcp_qpair *tqpair, struct nvme_tcp_pdu *pdu)
+{
+	struct nvme_tcp_req *tcp_req;
+	struct spdk_nvme_tcp_r2t_hdr *r2t = &pdu->hdr.r2t;
+	uint32_t cid, error_offset = 0;
+	enum spdk_nvme_tcp_term_req_fes fes;
+
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "enter\n");
+	cid = r2t->cccid;
+	tcp_req = get_nvme_active_req_by_cid(tqpair, cid);
+	if (!tcp_req) {
+		SPDK_ERRLOG("Cannot find tcp_req for tqpair=%p\n", tqpair);
+		fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD;
+		error_offset = offsetof(struct spdk_nvme_tcp_r2t_hdr, cccid);
+		goto end;
+	}
+
+	tcp_req->ordering.r2t_recv = 1;
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "r2t info: r2to=%u, r2tl=%u for tqpair=%p\n", r2t->r2to, r2t->r2tl,
+		      tqpair);
+
+	if (tcp_req->state == NVME_TCP_REQ_ACTIVE) {
+		assert(tcp_req->active_r2ts == 0);
+		tcp_req->state = NVME_TCP_REQ_ACTIVE_R2T;
+	}
+
+	tcp_req->active_r2ts++;
+	if (tcp_req->active_r2ts > tqpair->maxr2t) {
+		fes = SPDK_NVME_TCP_TERM_REQ_FES_R2T_LIMIT_EXCEEDED;
+		SPDK_ERRLOG("Invalid R2T: it exceeds the R2T maixmal=%u for tqpair=%p\n", tqpair->maxr2t, tqpair);
+		goto end;
+	}
+
+	if (tcp_req->datao != r2t->r2to) {
+		fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD;
+		error_offset = offsetof(struct spdk_nvme_tcp_r2t_hdr, r2to);
+		goto end;
+
+	}
+
+	if ((r2t->r2tl + r2t->r2to) > tcp_req->req->payload_size) {
+		SPDK_ERRLOG("Invalid R2T info for tcp_req=%p: (r2to(%u) + r2tl(%u)) exceeds payload_size(%u)\n",
+			    tcp_req, r2t->r2to, r2t->r2tl, tqpair->maxh2cdata);
+		fes = SPDK_NVME_TCP_TERM_REQ_FES_DATA_TRANSFER_OUT_OF_RANGE;
+		error_offset = offsetof(struct spdk_nvme_tcp_r2t_hdr, r2tl);
+		goto end;
+
+	}
+
+	tcp_req->ttag = r2t->ttag;
+	tcp_req->r2tl_remain = r2t->r2tl;
+	nvme_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY);
+
+	if (spdk_likely(tcp_req->ordering.send_ack)) {
+		nvme_tcp_send_h2c_data(tcp_req);
+	}
+	return;
+
+end:
+	nvme_tcp_qpair_send_h2c_term_req(tqpair, pdu, fes, error_offset);
+	return;
+
+}
+
+static void
+nvme_tcp_pdu_psh_handle(struct nvme_tcp_qpair *tqpair, uint32_t *reaped)
+{
+	struct nvme_tcp_pdu *pdu;
+	int rc;
+	uint32_t crc32c, error_offset = 0;
+	enum spdk_nvme_tcp_term_req_fes fes;
+
+	assert(tqpair->recv_state == NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PSH);
+	pdu = &tqpair->recv_pdu;
+
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "enter: pdu type =%u\n", pdu->hdr.common.pdu_type);
+	/* check header digest if needed */
+	if (pdu->has_hdgst) {
+		crc32c = nvme_tcp_pdu_calc_header_digest(pdu);
+		rc = MATCH_DIGEST_WORD((uint8_t *)pdu->hdr.raw + pdu->hdr.common.hlen, crc32c);
+		if (rc == 0) {
+			SPDK_ERRLOG("header digest error on tqpair=(%p) with pdu=%p\n", tqpair, pdu);
+			fes = SPDK_NVME_TCP_TERM_REQ_FES_HDGST_ERROR;
+			nvme_tcp_qpair_send_h2c_term_req(tqpair, pdu, fes, error_offset);
+			return;
+
+		}
+	}
+
+	switch (pdu->hdr.common.pdu_type) {
+	case SPDK_NVME_TCP_PDU_TYPE_IC_RESP:
+		nvme_tcp_icresp_handle(tqpair, pdu);
+		break;
+	case SPDK_NVME_TCP_PDU_TYPE_CAPSULE_RESP:
+		nvme_tcp_capsule_resp_hdr_handle(tqpair, pdu, reaped);
+		break;
+	case SPDK_NVME_TCP_PDU_TYPE_C2H_DATA:
+		nvme_tcp_c2h_data_hdr_handle(tqpair, pdu);
+		break;
+
+	case SPDK_NVME_TCP_PDU_TYPE_C2H_TERM_REQ:
+		nvme_tcp_c2h_term_req_hdr_handle(tqpair, pdu);
+		break;
+	case SPDK_NVME_TCP_PDU_TYPE_R2T:
+		nvme_tcp_r2t_hdr_handle(tqpair, pdu);
+		break;
+
+	default:
+		SPDK_ERRLOG("Unexpected PDU type 0x%02x\n", tqpair->recv_pdu.hdr.common.pdu_type);
+		fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD;
+		error_offset = 1;
+		nvme_tcp_qpair_send_h2c_term_req(tqpair, pdu, fes, error_offset);
+		break;
+	}
+
+}
+
+static int
+nvme_tcp_read_pdu(struct nvme_tcp_qpair *tqpair, uint32_t *reaped)
+{
+	int rc = 0;
+	struct nvme_tcp_pdu *pdu;
+	uint32_t data_len;
+	enum nvme_tcp_pdu_recv_state prev_state;
+
+	/* The loop here is to allow for several back-to-back state changes. */
+	do {
+		prev_state = tqpair->recv_state;
+		switch (tqpair->recv_state) {
+		/* If in a new state */
+		case NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY:
+			nvme_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_CH);
+			break;
+		/* common header */
+		case NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_CH:
+			pdu = &tqpair->recv_pdu;
+			if (pdu->ch_valid_bytes < sizeof(struct spdk_nvme_tcp_common_pdu_hdr)) {
+				rc = nvme_tcp_read_data(tqpair->sock,
+							sizeof(struct spdk_nvme_tcp_common_pdu_hdr) - pdu->ch_valid_bytes,
+							(uint8_t *)&pdu->hdr.common + pdu->ch_valid_bytes);
+				if (rc < 0) {
+					nvme_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_ERROR);
+					break;
+				}
+				pdu->ch_valid_bytes += rc;
+				if (pdu->ch_valid_bytes < sizeof(struct spdk_nvme_tcp_common_pdu_hdr)) {
+					return NVME_TCP_PDU_IN_PROGRESS;
+				}
+			}
+
+			/* The command header of this PDU has now been read from the socket. */
+			nvme_tcp_pdu_ch_handle(tqpair);
+			break;
+		/* Wait for the pdu specific header  */
+		case NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PSH:
+			pdu = &tqpair->recv_pdu;
+			rc = nvme_tcp_read_data(tqpair->sock,
+						pdu->psh_len - pdu->psh_valid_bytes,
+						(uint8_t *)&pdu->hdr.raw + sizeof(struct spdk_nvme_tcp_common_pdu_hdr) + pdu->psh_valid_bytes);
+			if (rc < 0) {
+				nvme_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_ERROR);
+				break;
+			}
+
+			pdu->psh_valid_bytes += rc;
+			if (pdu->psh_valid_bytes < pdu->psh_len) {
+				return NVME_TCP_PDU_IN_PROGRESS;
+			}
+
+			/* All header(ch, psh, head digist) of this PDU has now been read from the socket. */
+			nvme_tcp_pdu_psh_handle(tqpair, reaped);
+			break;
+		case NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PAYLOAD:
+			pdu = &tqpair->recv_pdu;
+			/* check whether the data is valid, if not we just return */
+			if (!pdu->data_len) {
+				return NVME_TCP_PDU_IN_PROGRESS;
+			}
+
+			data_len = pdu->data_len;
+			/* data digest */
+			if (spdk_unlikely((pdu->hdr.common.pdu_type == SPDK_NVME_TCP_PDU_TYPE_C2H_DATA) &&
+					  tqpair->host_ddgst_enable)) {
+				data_len += SPDK_NVME_TCP_DIGEST_LEN;
+				pdu->ddgst_enable = true;
+			}
+
+			rc = nvme_tcp_read_payload_data(tqpair->sock, pdu);
+			if (rc < 0) {
+				nvme_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_ERROR);
+				break;
+			}
+
+			pdu->readv_offset += rc;
+			if (pdu->readv_offset < data_len) {
+				return NVME_TCP_PDU_IN_PROGRESS;
+			}
+
+			assert(pdu->readv_offset == data_len);
+			/* All of this PDU has now been read from the socket. */
+			nvme_tcp_pdu_payload_handle(tqpair, reaped);
+			break;
+		case NVME_TCP_PDU_RECV_STATE_ERROR:
+			rc = NVME_TCP_PDU_FATAL;
+			break;
+		default:
+			assert(0);
+			break;
+		}
+	} while (prev_state != tqpair->recv_state);
+
+	return rc;
+}
+
+static void
+nvme_tcp_qpair_check_timeout(struct spdk_nvme_qpair *qpair)
+{
+	uint64_t t02;
+	struct nvme_tcp_req *tcp_req, *tmp;
+	struct nvme_tcp_qpair *tqpair = nvme_tcp_qpair(qpair);
+	struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
+	struct spdk_nvme_ctrlr_process *active_proc;
+
+	/* Don't check timeouts during controller initialization. */
+	if (ctrlr->state != NVME_CTRLR_STATE_READY) {
+		return;
+	}
+
+	if (nvme_qpair_is_admin_queue(qpair)) {
+		active_proc = nvme_ctrlr_get_current_process(ctrlr);
+	} else {
+		active_proc = qpair->active_proc;
+	}
+
+	/* Only check timeouts if the current process has a timeout callback. */
+	if (active_proc == NULL || active_proc->timeout_cb_fn == NULL) {
+		return;
+	}
+
+	t02 = spdk_get_ticks();
+	TAILQ_FOREACH_SAFE(tcp_req, &tqpair->outstanding_reqs, link, tmp) {
+		assert(tcp_req->req != NULL);
+
+		if (nvme_request_check_timeout(tcp_req->req, tcp_req->cid, active_proc, t02)) {
+			/*
+			 * The requests are in order, so as soon as one has not timed out,
+			 * stop iterating.
+			 */
+			break;
+		}
+	}
+}
+
+static int
+nvme_tcp_qpair_process_completions(struct spdk_nvme_qpair *qpair, uint32_t max_completions)
+{
+	struct nvme_tcp_qpair *tqpair = nvme_tcp_qpair(qpair);
+	uint32_t reaped;
+	int rc;
+
+	rc = spdk_sock_flush(tqpair->sock);
+	if (rc < 0) {
+		return rc;
+	}
+
+	if (max_completions == 0) {
+		max_completions = tqpair->num_entries;
+	} else {
+		max_completions = spdk_min(max_completions, tqpair->num_entries);
+	}
+
+	reaped = 0;
+	do {
+		rc = nvme_tcp_read_pdu(tqpair, &reaped);
+		if (rc < 0) {
+			SPDK_DEBUGLOG(SPDK_LOG_NVME, "Error polling CQ! (%d): %s\n",
+				      errno, spdk_strerror(errno));
+			goto fail;
+		} else if (rc == 0) {
+			/* Partial PDU is read */
+			break;
+		}
+
+	} while (reaped < max_completions);
+
+	if (spdk_unlikely(tqpair->qpair.ctrlr->timeout_enabled)) {
+		nvme_tcp_qpair_check_timeout(qpair);
+	}
+
+	return reaped;
+fail:
+
+	/*
+	 * Since admin queues take the ctrlr_lock before entering this function,
+	 * we can call nvme_transport_ctrlr_disconnect_qpair. For other qpairs we need
+	 * to call the generic function which will take the lock for us.
+	 */
+	qpair->transport_failure_reason = SPDK_NVME_QPAIR_FAILURE_UNKNOWN;
+
+	if (nvme_qpair_is_admin_queue(qpair)) {
+		nvme_transport_ctrlr_disconnect_qpair(qpair->ctrlr, qpair);
+	} else {
+		nvme_ctrlr_disconnect_qpair(qpair);
+	}
+	return -ENXIO;
+}
+
+static void
+nvme_tcp_qpair_sock_cb(void *ctx, struct spdk_sock_group *group, struct spdk_sock *sock)
+{
+	struct spdk_nvme_qpair *qpair = ctx;
+	struct nvme_tcp_poll_group *pgroup = nvme_tcp_poll_group(qpair->poll_group);
+	int32_t num_completions;
+
+	num_completions = spdk_nvme_qpair_process_completions(qpair, pgroup->completions_per_qpair);
+
+	if (pgroup->num_completions >= 0 && num_completions >= 0) {
+		pgroup->num_completions += num_completions;
+	} else {
+		pgroup->num_completions = -ENXIO;
+	}
+}
+
+static int
+nvme_tcp_qpair_icreq_send(struct nvme_tcp_qpair *tqpair)
+{
+	struct spdk_nvme_tcp_ic_req *ic_req;
+	struct nvme_tcp_pdu *pdu;
+	uint64_t icreq_timeout_tsc;
+	int rc;
+
+	pdu = &tqpair->send_pdu;
+	memset(&tqpair->send_pdu, 0, sizeof(tqpair->send_pdu));
+	ic_req = &pdu->hdr.ic_req;
+
+	ic_req->common.pdu_type = SPDK_NVME_TCP_PDU_TYPE_IC_REQ;
+	ic_req->common.hlen = ic_req->common.plen = sizeof(*ic_req);
+	ic_req->pfv = 0;
+	ic_req->maxr2t = NVME_TCP_MAX_R2T_DEFAULT - 1;
+	ic_req->hpda = NVME_TCP_HPDA_DEFAULT;
+
+	ic_req->dgst.bits.hdgst_enable = tqpair->qpair.ctrlr->opts.header_digest;
+	ic_req->dgst.bits.ddgst_enable = tqpair->qpair.ctrlr->opts.data_digest;
+
+	nvme_tcp_qpair_write_pdu(tqpair, pdu, nvme_tcp_send_icreq_complete, tqpair);
+
+	icreq_timeout_tsc = spdk_get_ticks() + (NVME_TCP_TIME_OUT_IN_SECONDS * spdk_get_ticks_hz());
+	do {
+		rc = nvme_tcp_qpair_process_completions(&tqpair->qpair, 0);
+	} while ((tqpair->state == NVME_TCP_QPAIR_STATE_INVALID) &&
+		 (rc == 0) && (spdk_get_ticks() <= icreq_timeout_tsc));
+
+	if (tqpair->state != NVME_TCP_QPAIR_STATE_RUNNING) {
+		SPDK_ERRLOG("Failed to construct the tqpair=%p via correct icresp\n", tqpair);
+		return -1;
+	}
+
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "Succesfully construct the tqpair=%p via correct icresp\n", tqpair);
+
+	return 0;
+}
+
+static int
+nvme_tcp_ctrlr_connect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)
+{
+	struct sockaddr_storage dst_addr;
+	struct sockaddr_storage src_addr;
+	int rc;
+	struct nvme_tcp_qpair *tqpair;
+	int family;
+	long int port;
+	struct spdk_sock_opts opts;
+
+	tqpair = nvme_tcp_qpair(qpair);
+
+	switch (ctrlr->trid.adrfam) {
+	case SPDK_NVMF_ADRFAM_IPV4:
+		family = AF_INET;
+		break;
+	case SPDK_NVMF_ADRFAM_IPV6:
+		family = AF_INET6;
+		break;
+	default:
+		SPDK_ERRLOG("Unhandled ADRFAM %d\n", ctrlr->trid.adrfam);
+		return -1;
+	}
+
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "adrfam %d ai_family %d\n", ctrlr->trid.adrfam, family);
+
+	memset(&dst_addr, 0, sizeof(dst_addr));
+
+	SPDK_DEBUGLOG(SPDK_LOG_NVME, "trsvcid is %s\n", ctrlr->trid.trsvcid);
+	rc = nvme_tcp_parse_addr(&dst_addr, family, ctrlr->trid.traddr, ctrlr->trid.trsvcid);
+	if (rc != 0) {
+		SPDK_ERRLOG("dst_addr nvme_tcp_parse_addr() failed\n");
+		return -1;
+	}
+
+	if (ctrlr->opts.src_addr[0] || ctrlr->opts.src_svcid[0]) {
+		memset(&src_addr, 0, sizeof(src_addr));
+		rc = nvme_tcp_parse_addr(&src_addr, family, ctrlr->opts.src_addr, ctrlr->opts.src_svcid);
+		if (rc != 0) {
+			SPDK_ERRLOG("src_addr nvme_tcp_parse_addr() failed\n");
+			return -1;
+		}
+	}
+
+	port = spdk_strtol(ctrlr->trid.trsvcid, 10);
+	if (port <= 0 || port >= INT_MAX) {
+		SPDK_ERRLOG("Invalid port: %s\n", ctrlr->trid.trsvcid);
+		return -1;
+	}
+
+	opts.opts_size = sizeof(opts);
+	spdk_sock_get_default_opts(&opts);
+	opts.priority = ctrlr->trid.priority;
+	tqpair->sock = spdk_sock_connect_ext(ctrlr->trid.traddr, port, NULL, &opts);
+	if (!tqpair->sock) {
+		SPDK_ERRLOG("sock connection error of tqpair=%p with addr=%s, port=%ld\n",
+			    tqpair, ctrlr->trid.traddr, port);
+		return -1;
+	}
+
+	tqpair->maxr2t = NVME_TCP_MAX_R2T_DEFAULT;
+	/* Explicitly set the state and recv_state of tqpair */
+	tqpair->state = NVME_TCP_QPAIR_STATE_INVALID;
+	if (tqpair->recv_state != NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY) {
+		nvme_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY);
+	}
+	rc = nvme_tcp_qpair_icreq_send(tqpair);
+	if (rc != 0) {
+		SPDK_ERRLOG("Unable to connect the tqpair\n");
+		return -1;
+	}
+
+	rc = nvme_fabric_qpair_connect(&tqpair->qpair, tqpair->num_entries);
+	if (rc < 0) {
+		SPDK_ERRLOG("Failed to send an NVMe-oF Fabric CONNECT command\n");
+		return -1;
+	}
+
+	return 0;
+}
+
+static struct spdk_nvme_qpair *
+nvme_tcp_ctrlr_create_qpair(struct spdk_nvme_ctrlr *ctrlr,
+			    uint16_t qid, uint32_t qsize,
+			    enum spdk_nvme_qprio qprio,
+			    uint32_t num_requests)
+{
+	struct nvme_tcp_qpair *tqpair;
+	struct spdk_nvme_qpair *qpair;
+	int rc;
+
+	tqpair = calloc(1, sizeof(struct nvme_tcp_qpair));
+	if (!tqpair) {
+		SPDK_ERRLOG("failed to get create tqpair\n");
+		return NULL;
+	}
+
+	tqpair->num_entries = qsize;
+	qpair = &tqpair->qpair;
+	rc = nvme_qpair_init(qpair, qid, ctrlr, qprio, num_requests);
+	if (rc != 0) {
+		free(tqpair);
+		return NULL;
+	}
+
+	rc = nvme_tcp_alloc_reqs(tqpair);
+	if (rc) {
+		nvme_tcp_ctrlr_delete_io_qpair(ctrlr, qpair);
+		return NULL;
+	}
+
+	return qpair;
+}
+
+static struct spdk_nvme_qpair *
+nvme_tcp_ctrlr_create_io_qpair(struct spdk_nvme_ctrlr *ctrlr, uint16_t qid,
+			       const struct spdk_nvme_io_qpair_opts *opts)
+{
+	return nvme_tcp_ctrlr_create_qpair(ctrlr, qid, opts->io_queue_size, opts->qprio,
+					   opts->io_queue_requests);
+}
+
+static struct spdk_nvme_ctrlr *nvme_tcp_ctrlr_construct(const struct spdk_nvme_transport_id *trid,
+		const struct spdk_nvme_ctrlr_opts *opts,
+		void *devhandle)
+{
+	struct nvme_tcp_ctrlr *tctrlr;
+	union spdk_nvme_cap_register cap;
+	union spdk_nvme_vs_register vs;
+	int rc;
+
+	tctrlr = calloc(1, sizeof(*tctrlr));
+	if (tctrlr == NULL) {
+		SPDK_ERRLOG("could not allocate ctrlr\n");
+		return NULL;
+	}
+
+	tctrlr->ctrlr.opts = *opts;
+	tctrlr->ctrlr.trid = *trid;
+
+	rc = nvme_ctrlr_construct(&tctrlr->ctrlr);
+	if (rc != 0) {
+		free(tctrlr);
+		return NULL;
+	}
+
+	tctrlr->ctrlr.adminq = nvme_tcp_ctrlr_create_qpair(&tctrlr->ctrlr, 0,
+			       tctrlr->ctrlr.opts.admin_queue_size, 0,
+			       tctrlr->ctrlr.opts.admin_queue_size);
+	if (!tctrlr->ctrlr.adminq) {
+		SPDK_ERRLOG("failed to create admin qpair\n");
+		nvme_tcp_ctrlr_destruct(&tctrlr->ctrlr);
+		return NULL;
+	}
+
+	rc = nvme_transport_ctrlr_connect_qpair(&tctrlr->ctrlr, tctrlr->ctrlr.adminq);
+	if (rc < 0) {
+		SPDK_ERRLOG("failed to connect admin qpair\n");
+		nvme_tcp_ctrlr_destruct(&tctrlr->ctrlr);
+		return NULL;
+	}
+
+	if (nvme_ctrlr_get_cap(&tctrlr->ctrlr, &cap)) {
+		SPDK_ERRLOG("get_cap() failed\n");
+		nvme_ctrlr_destruct(&tctrlr->ctrlr);
+		return NULL;
+	}
+
+	if (nvme_ctrlr_get_vs(&tctrlr->ctrlr, &vs)) {
+		SPDK_ERRLOG("get_vs() failed\n");
+		nvme_ctrlr_destruct(&tctrlr->ctrlr);
+		return NULL;
+	}
+
+	if (nvme_ctrlr_add_process(&tctrlr->ctrlr, 0) != 0) {
+		SPDK_ERRLOG("nvme_ctrlr_add_process() failed\n");
+		nvme_ctrlr_destruct(&tctrlr->ctrlr);
+		return NULL;
+	}
+
+	nvme_ctrlr_init_cap(&tctrlr->ctrlr, &cap, &vs);
+
+	return &tctrlr->ctrlr;
+}
+
+static uint32_t
+nvme_tcp_ctrlr_get_max_xfer_size(struct spdk_nvme_ctrlr *ctrlr)
+{
+	/* TCP transport doens't limit maximum IO transfer size. */
+	return UINT32_MAX;
+}
+
+static uint16_t
+nvme_tcp_ctrlr_get_max_sges(struct spdk_nvme_ctrlr *ctrlr)
+{
+	/*
+	 * We do not support >1 SGE in the initiator currently,
+	 *  so we can only return 1 here.  Once that support is
+	 *  added, this should return ctrlr->cdata.nvmf_specific.msdbd
+	 *  instead.
+	 */
+	return 1;
+}
+
+static int
+nvme_tcp_qpair_iterate_requests(struct spdk_nvme_qpair *qpair,
+				int (*iter_fn)(struct nvme_request *req, void *arg),
+				void *arg)
+{
+	struct nvme_tcp_qpair *tqpair = nvme_tcp_qpair(qpair);
+	struct nvme_tcp_req *tcp_req, *tmp;
+	int rc;
+
+	assert(iter_fn != NULL);
+
+	TAILQ_FOREACH_SAFE(tcp_req, &tqpair->outstanding_reqs, link, tmp) {
+		assert(tcp_req->req != NULL);
+
+		rc = iter_fn(tcp_req->req, arg);
+		if (rc != 0) {
+			return rc;
+		}
+	}
+
+	return 0;
+}
+
+static void
+nvme_tcp_admin_qpair_abort_aers(struct spdk_nvme_qpair *qpair)
+{
+	struct nvme_tcp_req *tcp_req, *tmp;
+	struct spdk_nvme_cpl cpl;
+	struct nvme_tcp_qpair *tqpair = nvme_tcp_qpair(qpair);
+
+	cpl.status.sc = SPDK_NVME_SC_ABORTED_SQ_DELETION;
+	cpl.status.sct = SPDK_NVME_SCT_GENERIC;
+
+	TAILQ_FOREACH_SAFE(tcp_req, &tqpair->outstanding_reqs, link, tmp) {
+		assert(tcp_req->req != NULL);
+		if (tcp_req->req->cmd.opc != SPDK_NVME_OPC_ASYNC_EVENT_REQUEST) {
+			continue;
+		}
+
+		nvme_tcp_req_complete(tcp_req, &cpl);
+		nvme_tcp_req_put(tqpair, tcp_req);
+	}
+}
+
+static struct spdk_nvme_transport_poll_group *
+nvme_tcp_poll_group_create(void)
+{
+	struct nvme_tcp_poll_group *group = calloc(1, sizeof(*group));
+
+	if (group == NULL) {
+		SPDK_ERRLOG("Unable to allocate poll group.\n");
+		return NULL;
+	}
+
+	group->sock_group = spdk_sock_group_create(group);
+	if (group->sock_group == NULL) {
+		free(group);
+		SPDK_ERRLOG("Unable to allocate sock group.\n");
+		return NULL;
+	}
+
+	return &group->group;
+}
+
+static int
+nvme_tcp_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair)
+{
+	struct nvme_tcp_poll_group *group = nvme_tcp_poll_group(qpair->poll_group);
+	struct nvme_tcp_qpair *tqpair = nvme_tcp_qpair(qpair);
+
+	if (spdk_sock_group_add_sock(group->sock_group, tqpair->sock, nvme_tcp_qpair_sock_cb, qpair)) {
+		return -EPROTO;
+	}
+	return 0;
+}
+
+static int
+nvme_tcp_poll_group_disconnect_qpair(struct spdk_nvme_qpair *qpair)
+{
+	struct nvme_tcp_poll_group *group = nvme_tcp_poll_group(qpair->poll_group);
+	struct nvme_tcp_qpair *tqpair = nvme_tcp_qpair(qpair);
+
+	if (tqpair->sock && group->sock_group) {
+		if (spdk_sock_group_remove_sock(group->sock_group, tqpair->sock)) {
+			return -EPROTO;
+		}
+	}
+	return 0;
+}
+
+static int
+nvme_tcp_poll_group_add(struct spdk_nvme_transport_poll_group *tgroup,
+			struct spdk_nvme_qpair *qpair)
+{
+	struct nvme_tcp_qpair *tqpair = nvme_tcp_qpair(qpair);
+	struct nvme_tcp_poll_group *group = nvme_tcp_poll_group(tgroup);
+
+	/* disconnected qpairs won't have a sock to add. */
+	if (nvme_qpair_get_state(qpair) >= NVME_QPAIR_CONNECTED) {
+		if (spdk_sock_group_add_sock(group->sock_group, tqpair->sock, nvme_tcp_qpair_sock_cb, qpair)) {
+			return -EPROTO;
+		}
+	}
+
+	return 0;
+}
+
+static int
+nvme_tcp_poll_group_remove(struct spdk_nvme_transport_poll_group *tgroup,
+			   struct spdk_nvme_qpair *qpair)
+{
+	if (qpair->poll_group_tailq_head == &tgroup->connected_qpairs) {
+		return nvme_poll_group_disconnect_qpair(qpair);
+	}
+
+	return 0;
+}
+
+static int64_t
+nvme_tcp_poll_group_process_completions(struct spdk_nvme_transport_poll_group *tgroup,
+					uint32_t completions_per_qpair, spdk_nvme_disconnected_qpair_cb disconnected_qpair_cb)
+{
+	struct nvme_tcp_poll_group *group = nvme_tcp_poll_group(tgroup);
+	struct spdk_nvme_qpair *qpair, *tmp_qpair;
+
+	group->completions_per_qpair = completions_per_qpair;
+	group->num_completions = 0;
+
+	spdk_sock_group_poll(group->sock_group);
+
+	STAILQ_FOREACH_SAFE(qpair, &tgroup->disconnected_qpairs, poll_group_stailq, tmp_qpair) {
+		disconnected_qpair_cb(qpair, tgroup->group->ctx);
+	}
+
+	return group->num_completions;
+}
+
+static int
+nvme_tcp_poll_group_destroy(struct spdk_nvme_transport_poll_group *tgroup)
+{
+	int rc;
+	struct nvme_tcp_poll_group *group = nvme_tcp_poll_group(tgroup);
+
+	if (!STAILQ_EMPTY(&tgroup->connected_qpairs) || !STAILQ_EMPTY(&tgroup->disconnected_qpairs)) {
+		return -EBUSY;
+	}
+
+	rc = spdk_sock_group_close(&group->sock_group);
+	if (rc != 0) {
+		SPDK_ERRLOG("Failed to close the sock group for a tcp poll group.\n");
+		assert(false);
+	}
+
+	free(tgroup);
+
+	return 0;
+}
+
+const struct spdk_nvme_transport_ops tcp_ops = {
+	.name = "TCP",
+	.type = SPDK_NVME_TRANSPORT_TCP,
+	.ctrlr_construct = nvme_tcp_ctrlr_construct,
+	.ctrlr_scan = nvme_fabric_ctrlr_scan,
+	.ctrlr_destruct = nvme_tcp_ctrlr_destruct,
+	.ctrlr_enable = nvme_tcp_ctrlr_enable,
+
+	.ctrlr_set_reg_4 = nvme_fabric_ctrlr_set_reg_4,
+	.ctrlr_set_reg_8 = nvme_fabric_ctrlr_set_reg_8,
+	.ctrlr_get_reg_4 = nvme_fabric_ctrlr_get_reg_4,
+	.ctrlr_get_reg_8 = nvme_fabric_ctrlr_get_reg_8,
+
+	.ctrlr_get_max_xfer_size = nvme_tcp_ctrlr_get_max_xfer_size,
+	.ctrlr_get_max_sges = nvme_tcp_ctrlr_get_max_sges,
+
+	.ctrlr_create_io_qpair = nvme_tcp_ctrlr_create_io_qpair,
+	.ctrlr_delete_io_qpair = nvme_tcp_ctrlr_delete_io_qpair,
+	.ctrlr_connect_qpair = nvme_tcp_ctrlr_connect_qpair,
+	.ctrlr_disconnect_qpair = nvme_tcp_ctrlr_disconnect_qpair,
+
+	.qpair_abort_reqs = nvme_tcp_qpair_abort_reqs,
+	.qpair_reset = nvme_tcp_qpair_reset,
+	.qpair_submit_request = nvme_tcp_qpair_submit_request,
+	.qpair_process_completions = nvme_tcp_qpair_process_completions,
+	.qpair_iterate_requests = nvme_tcp_qpair_iterate_requests,
+	.admin_qpair_abort_aers = nvme_tcp_admin_qpair_abort_aers,
+
+	.poll_group_create = nvme_tcp_poll_group_create,
+	.poll_group_connect_qpair = nvme_tcp_poll_group_connect_qpair,
+	.poll_group_disconnect_qpair = nvme_tcp_poll_group_disconnect_qpair,
+	.poll_group_add = nvme_tcp_poll_group_add,
+	.poll_group_remove = nvme_tcp_poll_group_remove,
+	.poll_group_process_completions = nvme_tcp_poll_group_process_completions,
+	.poll_group_destroy = nvme_tcp_poll_group_destroy,
+};
+
+SPDK_NVME_TRANSPORT_REGISTER(tcp, &tcp_ops);
diff --git a/src/spdk/lib/nvme/nvme_transport.c b/src/spdk/lib/nvme/nvme_transport.c
new file mode 100644
index 000000000..76efd5966
--- /dev/null
+++ b/src/spdk/lib/nvme/nvme_transport.c
@@ -0,0 +1,591 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * NVMe transport abstraction
+ */
+
+#include "nvme_internal.h"
+#include "spdk/queue.h"
+
+#define SPDK_MAX_NUM_OF_TRANSPORTS 16
+
+struct spdk_nvme_transport {
+	struct spdk_nvme_transport_ops	ops;
+	TAILQ_ENTRY(spdk_nvme_transport)	link;
+};
+
+TAILQ_HEAD(nvme_transport_list, spdk_nvme_transport) g_spdk_nvme_transports =
+	TAILQ_HEAD_INITIALIZER(g_spdk_nvme_transports);
+
+struct spdk_nvme_transport g_spdk_transports[SPDK_MAX_NUM_OF_TRANSPORTS] = {};
+int g_current_transport_index = 0;
+
+const struct spdk_nvme_transport *
+nvme_get_first_transport(void)
+{
+	return TAILQ_FIRST(&g_spdk_nvme_transports);
+}
+
+const struct spdk_nvme_transport *
+nvme_get_next_transport(const struct spdk_nvme_transport *transport)
+{
+	return TAILQ_NEXT(transport, link);
+}
+
+/*
+ * Unfortunately, due to NVMe PCIe multiprocess support, we cannot store the
+ * transport object in either the controller struct or the admin qpair. THis means
+ * that a lot of admin related transport calls will have to call nvme_get_transport
+ * in order to knwo which functions to call.
+ * In the I/O path, we have the ability to store the transport struct in the I/O
+ * qpairs to avoid taking a performance hit.
+ */
+const struct spdk_nvme_transport *
+nvme_get_transport(const char *transport_name)
+{
+	struct spdk_nvme_transport *registered_transport;
+
+	TAILQ_FOREACH(registered_transport, &g_spdk_nvme_transports, link) {
+		if (strcasecmp(transport_name, registered_transport->ops.name) == 0) {
+			return registered_transport;
+		}
+	}
+
+	return NULL;
+}
+
+bool
+spdk_nvme_transport_available(enum spdk_nvme_transport_type trtype)
+{
+	return nvme_get_transport(spdk_nvme_transport_id_trtype_str(trtype)) == NULL ? false : true;
+}
+
+bool
+spdk_nvme_transport_available_by_name(const char *transport_name)
+{
+	return nvme_get_transport(transport_name) == NULL ? false : true;
+}
+
+void spdk_nvme_transport_register(const struct spdk_nvme_transport_ops *ops)
+{
+	struct spdk_nvme_transport *new_transport;
+
+	if (nvme_get_transport(ops->name)) {
+		SPDK_ERRLOG("Double registering NVMe transport %s is prohibited.\n", ops->name);
+		assert(false);
+	}
+
+	if (g_current_transport_index == SPDK_MAX_NUM_OF_TRANSPORTS) {
+		SPDK_ERRLOG("Unable to register new NVMe transport.\n");
+		assert(false);
+		return;
+	}
+	new_transport = &g_spdk_transports[g_current_transport_index++];
+
+	new_transport->ops = *ops;
+	TAILQ_INSERT_TAIL(&g_spdk_nvme_transports, new_transport, link);
+}
+
+struct spdk_nvme_ctrlr *nvme_transport_ctrlr_construct(const struct spdk_nvme_transport_id *trid,
+		const struct spdk_nvme_ctrlr_opts *opts,
+		void *devhandle)
+{
+	const struct spdk_nvme_transport *transport = nvme_get_transport(trid->trstring);
+	struct spdk_nvme_ctrlr *ctrlr;
+
+	if (transport == NULL) {
+		SPDK_ERRLOG("Transport %s doesn't exist.", trid->trstring);
+		return NULL;
+	}
+
+	ctrlr = transport->ops.ctrlr_construct(trid, opts, devhandle);
+
+	return ctrlr;
+}
+
+int
+nvme_transport_ctrlr_scan(struct spdk_nvme_probe_ctx *probe_ctx,
+			  bool direct_connect)
+{
+	const struct spdk_nvme_transport *transport = nvme_get_transport(probe_ctx->trid.trstring);
+
+	if (transport == NULL) {
+		SPDK_ERRLOG("Transport %s doesn't exist.", probe_ctx->trid.trstring);
+		return -ENOENT;
+	}
+
+	return transport->ops.ctrlr_scan(probe_ctx, direct_connect);
+}
+
+int
+nvme_transport_ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr)
+{
+	const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
+
+	assert(transport != NULL);
+	return transport->ops.ctrlr_destruct(ctrlr);
+}
+
+int
+nvme_transport_ctrlr_enable(struct spdk_nvme_ctrlr *ctrlr)
+{
+	const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
+
+	assert(transport != NULL);
+	return transport->ops.ctrlr_enable(ctrlr);
+}
+
+int
+nvme_transport_ctrlr_set_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t value)
+{
+	const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
+
+	assert(transport != NULL);
+	return transport->ops.ctrlr_set_reg_4(ctrlr, offset, value);
+}
+
+int
+nvme_transport_ctrlr_set_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t value)
+{
+	const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
+
+	assert(transport != NULL);
+	return transport->ops.ctrlr_set_reg_8(ctrlr, offset, value);
+}
+
+int
+nvme_transport_ctrlr_get_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t *value)
+{
+	const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
+
+	assert(transport != NULL);
+	return transport->ops.ctrlr_get_reg_4(ctrlr, offset, value);
+}
+
+int
+nvme_transport_ctrlr_get_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t *value)
+{
+	const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
+
+	assert(transport != NULL);
+	return transport->ops.ctrlr_get_reg_8(ctrlr, offset, value);
+}
+
+uint32_t
+nvme_transport_ctrlr_get_max_xfer_size(struct spdk_nvme_ctrlr *ctrlr)
+{
+	const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
+
+	assert(transport != NULL);
+	return transport->ops.ctrlr_get_max_xfer_size(ctrlr);
+}
+
+uint16_t
+nvme_transport_ctrlr_get_max_sges(struct spdk_nvme_ctrlr *ctrlr)
+{
+	const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
+
+	assert(transport != NULL);
+	return transport->ops.ctrlr_get_max_sges(ctrlr);
+}
+
+int
+nvme_transport_ctrlr_reserve_cmb(struct spdk_nvme_ctrlr *ctrlr)
+{
+	const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
+
+	assert(transport != NULL);
+	if (transport->ops.ctrlr_reserve_cmb != NULL) {
+		return transport->ops.ctrlr_reserve_cmb(ctrlr);
+	}
+
+	return -ENOTSUP;
+}
+
+void *
+nvme_transport_ctrlr_map_cmb(struct spdk_nvme_ctrlr *ctrlr, size_t *size)
+{
+	const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
+
+	assert(transport != NULL);
+	if (transport->ops.ctrlr_map_cmb != NULL) {
+		return transport->ops.ctrlr_map_cmb(ctrlr, size);
+	}
+
+	return NULL;
+}
+
+int
+nvme_transport_ctrlr_unmap_cmb(struct spdk_nvme_ctrlr *ctrlr)
+{
+	const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
+
+	assert(transport != NULL);
+	if (transport->ops.ctrlr_unmap_cmb != NULL) {
+		return transport->ops.ctrlr_unmap_cmb(ctrlr);
+	}
+
+	return 0;
+}
+
+struct spdk_nvme_qpair *
+nvme_transport_ctrlr_create_io_qpair(struct spdk_nvme_ctrlr *ctrlr, uint16_t qid,
+				     const struct spdk_nvme_io_qpair_opts *opts)
+{
+	struct spdk_nvme_qpair *qpair;
+	const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
+
+	assert(transport != NULL);
+	qpair = transport->ops.ctrlr_create_io_qpair(ctrlr, qid, opts);
+	if (qpair != NULL && !nvme_qpair_is_admin_queue(qpair)) {
+		qpair->transport = transport;
+	}
+
+	return qpair;
+}
+
+int
+nvme_transport_ctrlr_delete_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)
+{
+	const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
+
+	assert(transport != NULL);
+
+	/* Do not rely on qpair->transport.  For multi-process cases, a foreign process may delete
+	 * the IO qpair, in which case the transport object would be invalid (each process has their
+	 * own unique transport objects since they contain function pointers).  So we look up the
+	 * transport object in the delete_io_qpair case.
+	 */
+	return transport->ops.ctrlr_delete_io_qpair(ctrlr, qpair);
+}
+
+int
+nvme_transport_ctrlr_connect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)
+{
+	const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
+	uint8_t transport_failure_reason;
+	int rc;
+
+	assert(transport != NULL);
+	if (!nvme_qpair_is_admin_queue(qpair)) {
+		qpair->transport = transport;
+	}
+
+	transport_failure_reason = qpair->transport_failure_reason;
+	qpair->transport_failure_reason = SPDK_NVME_QPAIR_FAILURE_NONE;
+
+	nvme_qpair_set_state(qpair, NVME_QPAIR_CONNECTING);
+	rc = transport->ops.ctrlr_connect_qpair(ctrlr, qpair);
+	if (rc != 0) {
+		goto err;
+	}
+
+	nvme_qpair_set_state(qpair, NVME_QPAIR_CONNECTED);
+	if (qpair->poll_group) {
+		rc = nvme_poll_group_connect_qpair(qpair);
+		if (rc) {
+			goto err;
+		}
+	}
+
+	return rc;
+
+err:
+	/* If the qpair was unable to reconnect, restore the original failure reason. */
+	qpair->transport_failure_reason = transport_failure_reason;
+	nvme_transport_ctrlr_disconnect_qpair(ctrlr, qpair);
+	nvme_qpair_set_state(qpair, NVME_QPAIR_DISCONNECTED);
+	return rc;
+}
+
+void
+nvme_transport_ctrlr_disconnect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)
+{
+	const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
+
+	if (nvme_qpair_get_state(qpair) == NVME_QPAIR_DISCONNECTING ||
+	    nvme_qpair_get_state(qpair) == NVME_QPAIR_DISCONNECTED) {
+		return;
+	}
+
+	nvme_qpair_set_state(qpair, NVME_QPAIR_DISCONNECTING);
+	assert(transport != NULL);
+	if (qpair->poll_group) {
+		nvme_poll_group_disconnect_qpair(qpair);
+	}
+
+	transport->ops.ctrlr_disconnect_qpair(ctrlr, qpair);
+
+	nvme_qpair_abort_reqs(qpair, 0);
+	nvme_qpair_set_state(qpair, NVME_QPAIR_DISCONNECTED);
+}
+
+void
+nvme_transport_qpair_abort_reqs(struct spdk_nvme_qpair *qpair, uint32_t dnr)
+{
+	const struct spdk_nvme_transport *transport;
+
+	assert(dnr <= 1);
+	if (spdk_likely(!nvme_qpair_is_admin_queue(qpair))) {
+		qpair->transport->ops.qpair_abort_reqs(qpair, dnr);
+	} else {
+		transport = nvme_get_transport(qpair->ctrlr->trid.trstring);
+		assert(transport != NULL);
+		transport->ops.qpair_abort_reqs(qpair, dnr);
+	}
+}
+
+int
+nvme_transport_qpair_reset(struct spdk_nvme_qpair *qpair)
+{
+	const struct spdk_nvme_transport *transport;
+
+	if (spdk_likely(!nvme_qpair_is_admin_queue(qpair))) {
+		return qpair->transport->ops.qpair_reset(qpair);
+	}
+
+	transport = nvme_get_transport(qpair->ctrlr->trid.trstring);
+	assert(transport != NULL);
+	return transport->ops.qpair_reset(qpair);
+}
+
+int
+nvme_transport_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req)
+{
+	const struct spdk_nvme_transport *transport;
+
+	if (spdk_likely(!nvme_qpair_is_admin_queue(qpair))) {
+		return qpair->transport->ops.qpair_submit_request(qpair, req);
+	}
+
+	transport = nvme_get_transport(qpair->ctrlr->trid.trstring);
+	assert(transport != NULL);
+	return transport->ops.qpair_submit_request(qpair, req);
+}
+
+int32_t
+nvme_transport_qpair_process_completions(struct spdk_nvme_qpair *qpair, uint32_t max_completions)
+{
+	const struct spdk_nvme_transport *transport;
+
+	if (spdk_likely(!nvme_qpair_is_admin_queue(qpair))) {
+		return qpair->transport->ops.qpair_process_completions(qpair, max_completions);
+	}
+
+	transport = nvme_get_transport(qpair->ctrlr->trid.trstring);
+	assert(transport != NULL);
+	return transport->ops.qpair_process_completions(qpair, max_completions);
+}
+
+int
+nvme_transport_qpair_iterate_requests(struct spdk_nvme_qpair *qpair,
+				      int (*iter_fn)(struct nvme_request *req, void *arg),
+				      void *arg)
+{
+	const struct spdk_nvme_transport *transport;
+
+	if (spdk_likely(!nvme_qpair_is_admin_queue(qpair))) {
+		return qpair->transport->ops.qpair_iterate_requests(qpair, iter_fn, arg);
+	}
+
+	transport = nvme_get_transport(qpair->ctrlr->trid.trstring);
+	assert(transport != NULL);
+	return transport->ops.qpair_iterate_requests(qpair, iter_fn, arg);
+}
+
+void
+nvme_transport_admin_qpair_abort_aers(struct spdk_nvme_qpair *qpair)
+{
+	const struct spdk_nvme_transport *transport = nvme_get_transport(qpair->ctrlr->trid.trstring);
+
+	assert(transport != NULL);
+	transport->ops.admin_qpair_abort_aers(qpair);
+}
+
+struct spdk_nvme_transport_poll_group *
+nvme_transport_poll_group_create(const struct spdk_nvme_transport *transport)
+{
+	struct spdk_nvme_transport_poll_group *group = NULL;
+
+	group = transport->ops.poll_group_create();
+	if (group) {
+		group->transport = transport;
+		STAILQ_INIT(&group->connected_qpairs);
+		STAILQ_INIT(&group->disconnected_qpairs);
+	}
+
+	return group;
+}
+
+int
+nvme_transport_poll_group_add(struct spdk_nvme_transport_poll_group *tgroup,
+			      struct spdk_nvme_qpair *qpair)
+{
+	int rc;
+
+	rc = tgroup->transport->ops.poll_group_add(tgroup, qpair);
+	if (rc == 0) {
+		qpair->poll_group = tgroup;
+		assert(nvme_qpair_get_state(qpair) < NVME_QPAIR_CONNECTED);
+		qpair->poll_group_tailq_head = &tgroup->disconnected_qpairs;
+		STAILQ_INSERT_TAIL(&tgroup->disconnected_qpairs, qpair, poll_group_stailq);
+	}
+
+	return rc;
+}
+
+int
+nvme_transport_poll_group_remove(struct spdk_nvme_transport_poll_group *tgroup,
+				 struct spdk_nvme_qpair *qpair)
+{
+	int rc;
+
+	rc = tgroup->transport->ops.poll_group_remove(tgroup, qpair);
+	if (rc == 0) {
+		if (qpair->poll_group_tailq_head == &tgroup->connected_qpairs) {
+			STAILQ_REMOVE(&tgroup->connected_qpairs, qpair, spdk_nvme_qpair, poll_group_stailq);
+		} else if (qpair->poll_group_tailq_head == &tgroup->disconnected_qpairs) {
+			STAILQ_REMOVE(&tgroup->disconnected_qpairs, qpair, spdk_nvme_qpair, poll_group_stailq);
+		} else {
+			return -ENOENT;
+		}
+
+		qpair->poll_group = NULL;
+		qpair->poll_group_tailq_head = NULL;
+	}
+
+	return rc;
+}
+
+int64_t
+nvme_transport_poll_group_process_completions(struct spdk_nvme_transport_poll_group *tgroup,
+		uint32_t completions_per_qpair, spdk_nvme_disconnected_qpair_cb disconnected_qpair_cb)
+{
+	struct spdk_nvme_qpair *qpair;
+	int64_t rc;
+
+	tgroup->in_completion_context = true;
+	rc = tgroup->transport->ops.poll_group_process_completions(tgroup, completions_per_qpair,
+			disconnected_qpair_cb);
+	tgroup->in_completion_context = false;
+
+	if (spdk_unlikely(tgroup->num_qpairs_to_delete > 0)) {
+		/* deleted qpairs are more likely to be in the disconnected qpairs list. */
+		STAILQ_FOREACH(qpair, &tgroup->disconnected_qpairs, poll_group_stailq) {
+			if (spdk_unlikely(qpair->delete_after_completion_context)) {
+				spdk_nvme_ctrlr_free_io_qpair(qpair);
+				if (--tgroup->num_qpairs_to_delete == 0) {
+					return rc;
+				}
+			}
+		}
+
+		STAILQ_FOREACH(qpair, &tgroup->connected_qpairs, poll_group_stailq) {
+			if (spdk_unlikely(qpair->delete_after_completion_context)) {
+				spdk_nvme_ctrlr_free_io_qpair(qpair);
+				if (--tgroup->num_qpairs_to_delete == 0) {
+					return rc;
+				}
+			}
+		}
+		/* Just in case. */
+		SPDK_DEBUGLOG(SPDK_LOG_NVME, "Mismatch between qpairs to delete and poll group number.\n");
+		tgroup->num_qpairs_to_delete = 0;
+	}
+
+	return rc;
+}
+
+int
+nvme_transport_poll_group_destroy(struct spdk_nvme_transport_poll_group *tgroup)
+{
+	return tgroup->transport->ops.poll_group_destroy(tgroup);
+}
+
+int
+nvme_transport_poll_group_disconnect_qpair(struct spdk_nvme_qpair *qpair)
+{
+	struct spdk_nvme_transport_poll_group *tgroup;
+	int rc;
+
+	tgroup = qpair->poll_group;
+
+	if (qpair->poll_group_tailq_head == &tgroup->disconnected_qpairs) {
+		return 0;
+	}
+
+	if (qpair->poll_group_tailq_head == &tgroup->connected_qpairs) {
+		rc = tgroup->transport->ops.poll_group_disconnect_qpair(qpair);
+		if (rc == 0) {
+			qpair->poll_group_tailq_head = &tgroup->disconnected_qpairs;
+			STAILQ_REMOVE(&tgroup->connected_qpairs, qpair, spdk_nvme_qpair, poll_group_stailq);
+			STAILQ_INSERT_TAIL(&tgroup->disconnected_qpairs, qpair, poll_group_stailq);
+			/* EINPROGRESS indicates that a call has already been made to this function.
+			 * It just keeps us from segfaulting on a double removal/insert.
+			 */
+		} else if (rc == -EINPROGRESS) {
+			rc = 0;
+		}
+		return rc;
+	}
+
+	return -EINVAL;
+}
+
+int
+nvme_transport_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair)
+{
+	struct spdk_nvme_transport_poll_group *tgroup;
+	int rc;
+
+	tgroup = qpair->poll_group;
+
+	if (qpair->poll_group_tailq_head == &tgroup->connected_qpairs) {
+		return 0;
+	}
+
+	if (qpair->poll_group_tailq_head == &tgroup->disconnected_qpairs) {
+		rc = tgroup->transport->ops.poll_group_connect_qpair(qpair);
+		if (rc == 0) {
+			qpair->poll_group_tailq_head = &tgroup->connected_qpairs;
+			STAILQ_REMOVE(&tgroup->disconnected_qpairs, qpair, spdk_nvme_qpair, poll_group_stailq);
+			STAILQ_INSERT_TAIL(&tgroup->connected_qpairs, qpair, poll_group_stailq);
+		}
+
+		return rc == -EINPROGRESS ? 0 : rc;
+	}
+
+
+	return -EINVAL;
+}
diff --git a/src/spdk/lib/nvme/nvme_uevent.c b/src/spdk/lib/nvme/nvme_uevent.c
new file mode 100644
index 000000000..1bcfff1cb
--- /dev/null
+++ b/src/spdk/lib/nvme/nvme_uevent.c
@@ -0,0 +1,213 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+#include "spdk/string.h"
+
+#include "spdk/log.h"
+
+#include "nvme_uevent.h"
+
+#ifdef __linux__
+
+#include <linux/netlink.h>
+
+#define SPDK_UEVENT_MSG_LEN 4096
+
+int
+nvme_uevent_connect(void)
+{
+	struct sockaddr_nl addr;
+	int netlink_fd;
+	int size = 64 * 1024;
+	int flag;
+
+	memset(&addr, 0, sizeof(addr));
+	addr.nl_family = AF_NETLINK;
+	addr.nl_pid = getpid();
+	addr.nl_groups = 0xffffffff;
+
+	netlink_fd = socket(PF_NETLINK, SOCK_DGRAM, NETLINK_KOBJECT_UEVENT);
+	if (netlink_fd < 0) {
+		return -1;
+	}
+
+	setsockopt(netlink_fd, SOL_SOCKET, SO_RCVBUFFORCE, &size, sizeof(size));
+
+	flag = fcntl(netlink_fd, F_GETFL);
+	if (fcntl(netlink_fd, F_SETFL, flag | O_NONBLOCK) < 0) {
+		SPDK_ERRLOG("fcntl can't set nonblocking mode for socket, fd: %d (%s)\n", netlink_fd,
+			    spdk_strerror(errno));
+		close(netlink_fd);
+		return -1;
+	}
+
+	if (bind(netlink_fd, (struct sockaddr *) &addr, sizeof(addr)) < 0) {
+		close(netlink_fd);
+		return -1;
+	}
+	return netlink_fd;
+}
+
+/* Note: We only parse the event from uio subsystem and will ignore
+ *       all the event from other subsystem. the event from uio subsystem
+ *       as below:
+ *       action: "add" or "remove"
+ *       subsystem: "uio"
+ *       dev_path: "/devices/pci0000:80/0000:80:01.0/0000:81:00.0/uio/uio0"
+ */
+static int
+parse_event(const char *buf, struct spdk_uevent *event)
+{
+	char action[SPDK_UEVENT_MSG_LEN];
+	char subsystem[SPDK_UEVENT_MSG_LEN];
+	char dev_path[SPDK_UEVENT_MSG_LEN];
+	char driver[SPDK_UEVENT_MSG_LEN];
+	char vfio_pci_addr[SPDK_UEVENT_MSG_LEN];
+
+	memset(action, 0, SPDK_UEVENT_MSG_LEN);
+	memset(subsystem, 0, SPDK_UEVENT_MSG_LEN);
+	memset(dev_path, 0, SPDK_UEVENT_MSG_LEN);
+	memset(driver, 0, SPDK_UEVENT_MSG_LEN);
+	memset(vfio_pci_addr, 0, SPDK_UEVENT_MSG_LEN);
+
+	while (*buf) {
+		if (!strncmp(buf, "ACTION=", 7)) {
+			buf += 7;
+			snprintf(action, sizeof(action), "%s", buf);
+		} else if (!strncmp(buf, "DEVPATH=", 8)) {
+			buf += 8;
+			snprintf(dev_path, sizeof(dev_path), "%s", buf);
+		} else if (!strncmp(buf, "SUBSYSTEM=", 10)) {
+			buf += 10;
+			snprintf(subsystem, sizeof(subsystem), "%s", buf);
+		} else if (!strncmp(buf, "DRIVER=", 7)) {
+			buf += 7;
+			snprintf(driver, sizeof(driver), "%s", buf);
+		} else if (!strncmp(buf, "PCI_SLOT_NAME=", 14)) {
+			buf += 14;
+			snprintf(vfio_pci_addr, sizeof(vfio_pci_addr), "%s", buf);
+		}
+		while (*buf++)
+			;
+	}
+
+	if (!strncmp(subsystem, "uio", 3)) {
+		char *pci_address, *tmp;
+		struct spdk_pci_addr pci_addr;
+
+		event->subsystem = SPDK_NVME_UEVENT_SUBSYSTEM_UIO;
+		if (!strncmp(action, "add", 3)) {
+			event->action = SPDK_NVME_UEVENT_ADD;
+		}
+		if (!strncmp(action, "remove", 6)) {
+			event->action = SPDK_NVME_UEVENT_REMOVE;
+		}
+		tmp = strstr(dev_path, "/uio/");
+
+		memset(tmp, 0, SPDK_UEVENT_MSG_LEN - (tmp - dev_path));
+
+		pci_address = strrchr(dev_path, '/');
+		pci_address++;
+		if (spdk_pci_addr_parse(&pci_addr, pci_address) != 0) {
+			SPDK_ERRLOG("Invalid format for NVMe BDF: %s\n", pci_address);
+			return -1;
+		}
+		spdk_pci_addr_fmt(event->traddr, sizeof(event->traddr), &pci_addr);
+		return 1;
+	}
+	if (!strncmp(driver, "vfio-pci", 8)) {
+		struct spdk_pci_addr pci_addr;
+
+		event->subsystem = SPDK_NVME_UEVENT_SUBSYSTEM_VFIO;
+		if (!strncmp(action, "bind", 4)) {
+			event->action = SPDK_NVME_UEVENT_ADD;
+		}
+		if (!strncmp(action, "remove", 6)) {
+			event->action = SPDK_NVME_UEVENT_REMOVE;
+		}
+		if (spdk_pci_addr_parse(&pci_addr, vfio_pci_addr) != 0) {
+			SPDK_ERRLOG("Invalid format for NVMe BDF: %s\n", vfio_pci_addr);
+			return -1;
+		}
+		spdk_pci_addr_fmt(event->traddr, sizeof(event->traddr), &pci_addr);
+		return 1;
+
+	}
+	return -1;
+}
+
+int
+nvme_get_uevent(int fd, struct spdk_uevent *uevent)
+{
+	int ret;
+	char buf[SPDK_UEVENT_MSG_LEN];
+
+	memset(uevent, 0, sizeof(struct spdk_uevent));
+	memset(buf, 0, SPDK_UEVENT_MSG_LEN);
+
+	ret = recv(fd, buf, SPDK_UEVENT_MSG_LEN - 1, MSG_DONTWAIT);
+	if (ret > 0) {
+		return parse_event(buf, uevent);
+	}
+
+	if (ret < 0) {
+		if (errno == EAGAIN || errno == EWOULDBLOCK) {
+			return 0;
+		} else {
+			SPDK_ERRLOG("Socket read error(%d): %s\n", errno, spdk_strerror(errno));
+			return -1;
+		}
+	}
+
+	/* connection closed */
+	if (ret == 0) {
+		return -1;
+	}
+	return 0;
+}
+
+#else /* Not Linux */
+
+int
+nvme_uevent_connect(void)
+{
+	return -1;
+}
+
+int
+nvme_get_uevent(int fd, struct spdk_uevent *uevent)
+{
+	return -1;
+}
+#endif
diff --git a/src/spdk/lib/nvme/nvme_uevent.h b/src/spdk/lib/nvme/nvme_uevent.h
new file mode 100644
index 000000000..778d73c2a
--- /dev/null
+++ b/src/spdk/lib/nvme/nvme_uevent.h
@@ -0,0 +1,61 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * SPDK uevent
+ */
+
+#include "spdk/env.h"
+#include "spdk/nvmf_spec.h"
+
+#ifndef SPDK_UEVENT_H_
+#define SPDK_UEVENT_H_
+
+#define SPDK_NVME_UEVENT_SUBSYSTEM_UIO 1
+#define SPDK_NVME_UEVENT_SUBSYSTEM_VFIO 2
+
+enum spdk_nvme_uevent_action {
+	SPDK_NVME_UEVENT_ADD = 0,
+	SPDK_NVME_UEVENT_REMOVE = 1,
+};
+
+struct spdk_uevent {
+	enum spdk_nvme_uevent_action action;
+	int subsystem;
+	char traddr[SPDK_NVMF_TRADDR_MAX_LEN + 1];
+};
+
+int nvme_uevent_connect(void);
+int nvme_get_uevent(int fd, struct spdk_uevent *uevent);
+
+#endif /* SPDK_UEVENT_H_ */
diff --git a/src/spdk/lib/nvme/spdk_nvme.map b/src/spdk/lib/nvme/spdk_nvme.map
new file mode 100644
index 000000000..63a04eeca
--- /dev/null
+++ b/src/spdk/lib/nvme/spdk_nvme.map
@@ -0,0 +1,185 @@
+{
+	global:
+
+	# public functions from nvme.h
+	spdk_nvme_transport_register;
+	spdk_nvme_transport_available;
+	spdk_nvme_transport_available_by_name;
+	spdk_nvme_transport_id_parse;
+	spdk_nvme_transport_id_populate_trstring;
+	spdk_nvme_transport_id_parse_trtype;
+	spdk_nvme_transport_id_trtype_str;
+	spdk_nvme_transport_id_adrfam_str;
+	spdk_nvme_transport_id_parse_adrfam;
+	spdk_nvme_transport_id_compare;
+	spdk_nvme_trid_populate_transport;
+	spdk_nvme_host_id_parse;
+
+	spdk_nvme_prchk_flags_parse;
+	spdk_nvme_prchk_flags_str;
+
+	spdk_nvme_probe;
+	spdk_nvme_connect;
+	spdk_nvme_connect_async;
+	spdk_nvme_probe_async;
+	spdk_nvme_probe_poll_async;
+	spdk_nvme_detach;
+
+	spdk_nvme_ctrlr_is_discovery;
+	spdk_nvme_ctrlr_get_default_ctrlr_opts;
+	spdk_nvme_ctrlr_set_trid;
+	spdk_nvme_ctrlr_reset;
+	spdk_nvme_ctrlr_fail;
+	spdk_nvme_ctrlr_is_failed;
+	spdk_nvme_ctrlr_get_data;
+	spdk_nvme_ctrlr_get_regs_csts;
+	spdk_nvme_ctrlr_get_regs_cap;
+	spdk_nvme_ctrlr_get_regs_vs;
+	spdk_nvme_ctrlr_get_regs_cmbsz;
+	spdk_nvme_ctrlr_get_num_ns;
+	spdk_nvme_ctrlr_get_pci_device;
+	spdk_nvme_ctrlr_get_max_xfer_size;
+	spdk_nvme_ctrlr_is_active_ns;
+	spdk_nvme_ctrlr_get_first_active_ns;
+	spdk_nvme_ctrlr_get_next_active_ns;
+	spdk_nvme_ctrlr_is_log_page_supported;
+	spdk_nvme_ctrlr_is_feature_supported;
+	spdk_nvme_ctrlr_register_aer_callback;
+	spdk_nvme_ctrlr_register_timeout_callback;
+	spdk_nvme_ctrlr_get_default_io_qpair_opts;
+	spdk_nvme_ctrlr_alloc_io_qpair;
+	spdk_nvme_ctrlr_connect_io_qpair;
+	spdk_nvme_ctrlr_disconnect_io_qpair;
+	spdk_nvme_ctrlr_reconnect_io_qpair;
+	spdk_nvme_ctrlr_get_admin_qp_failure_reason;
+	spdk_nvme_ctrlr_free_io_qpair;
+	spdk_nvme_ctrlr_io_cmd_raw_no_payload_build;
+	spdk_nvme_ctrlr_cmd_io_raw;
+	spdk_nvme_ctrlr_cmd_io_raw_with_md;
+	spdk_nvme_ctrlr_cmd_admin_raw;
+	spdk_nvme_ctrlr_process_admin_completions;
+	spdk_nvme_ctrlr_get_ns;
+	spdk_nvme_ctrlr_cmd_get_log_page;
+	spdk_nvme_ctrlr_cmd_get_log_page_ext;
+	spdk_nvme_ctrlr_cmd_abort;
+	spdk_nvme_ctrlr_cmd_abort_ext;
+	spdk_nvme_ctrlr_cmd_set_feature;
+	spdk_nvme_ctrlr_cmd_get_feature;
+	spdk_nvme_ctrlr_cmd_get_feature_ns;
+	spdk_nvme_ctrlr_cmd_set_feature_ns;
+	spdk_nvme_ctrlr_cmd_security_receive;
+	spdk_nvme_ctrlr_cmd_security_send;
+	spdk_nvme_ctrlr_security_receive;
+	spdk_nvme_ctrlr_security_send;
+	spdk_nvme_ctrlr_get_flags;
+	spdk_nvme_ctrlr_attach_ns;
+	spdk_nvme_ctrlr_detach_ns;
+	spdk_nvme_ctrlr_create_ns;
+	spdk_nvme_ctrlr_delete_ns;
+	spdk_nvme_ctrlr_format;
+	spdk_nvme_ctrlr_update_firmware;
+	spdk_nvme_ctrlr_get_registers;
+	spdk_nvme_ctrlr_reserve_cmb;
+	spdk_nvme_ctrlr_map_cmb;
+	spdk_nvme_ctrlr_unmap_cmb;
+	spdk_nvme_ctrlr_get_transport_id;
+
+	spdk_nvme_poll_group_create;
+	spdk_nvme_poll_group_add;
+	spdk_nvme_poll_group_remove;
+	spdk_nvme_poll_group_destroy;
+	spdk_nvme_poll_group_process_completions;
+	spdk_nvme_poll_group_get_ctx;
+
+	spdk_nvme_ns_get_data;
+	spdk_nvme_ns_get_id;
+	spdk_nvme_ns_get_ctrlr;
+	spdk_nvme_ns_is_active;
+	spdk_nvme_ns_get_max_io_xfer_size;
+	spdk_nvme_ns_get_sector_size;
+	spdk_nvme_ns_get_extended_sector_size;
+	spdk_nvme_ns_get_num_sectors;
+	spdk_nvme_ns_get_size;
+	spdk_nvme_ns_get_pi_type;
+	spdk_nvme_ns_get_md_size;
+	spdk_nvme_ns_supports_extended_lba;
+	spdk_nvme_ns_supports_compare;
+	spdk_nvme_ns_get_dealloc_logical_block_read_value;
+	spdk_nvme_ns_get_optimal_io_boundary;
+	spdk_nvme_ns_get_uuid;
+	spdk_nvme_ns_get_flags;
+
+	spdk_nvme_ns_cmd_write;
+	spdk_nvme_ns_cmd_writev;
+	spdk_nvme_ns_cmd_writev_with_md;
+	spdk_nvme_ns_cmd_write_with_md;
+	spdk_nvme_ns_cmd_write_zeroes;
+	spdk_nvme_ns_cmd_write_uncorrectable;
+	spdk_nvme_ns_cmd_read;
+	spdk_nvme_ns_cmd_readv;
+	spdk_nvme_ns_cmd_readv_with_md;
+	spdk_nvme_ns_cmd_read_with_md;
+	spdk_nvme_ns_cmd_dataset_management;
+	spdk_nvme_ns_cmd_flush;
+	spdk_nvme_ns_cmd_reservation_register;
+	spdk_nvme_ns_cmd_reservation_release;
+	spdk_nvme_ns_cmd_reservation_acquire;
+	spdk_nvme_ns_cmd_reservation_report;
+	spdk_nvme_ns_cmd_compare;
+	spdk_nvme_ns_cmd_comparev;
+	spdk_nvme_ns_cmd_comparev_with_md;
+	spdk_nvme_ns_cmd_compare_with_md;
+
+	spdk_nvme_qpair_process_completions;
+	spdk_nvme_qpair_get_failure_reason;
+	spdk_nvme_qpair_add_cmd_error_injection;
+	spdk_nvme_qpair_remove_cmd_error_injection;
+	spdk_nvme_qpair_print_command;
+	spdk_nvme_qpair_print_completion;
+	spdk_nvme_print_command;
+	spdk_nvme_print_completion;
+
+	spdk_nvme_cpl_get_status_string;
+
+	spdk_nvme_rdma_init_hooks;
+
+	spdk_nvme_cuse_get_ctrlr_name;
+	spdk_nvme_cuse_get_ns_name;
+	spdk_nvme_cuse_register;
+	spdk_nvme_cuse_unregister;
+	spdk_nvme_cuse_update_namespaces;
+
+	spdk_nvme_map_prps;
+
+	# public functions from nvme_ocssd.h
+	spdk_nvme_ctrlr_is_ocssd_supported;
+	spdk_nvme_ocssd_ctrlr_cmd_geometry;
+	spdk_nvme_ocssd_ns_cmd_vector_reset;
+	spdk_nvme_ocssd_ns_cmd_vector_write;
+	spdk_nvme_ocssd_ns_cmd_vector_write_with_md;
+	spdk_nvme_ocssd_ns_cmd_vector_read;
+	spdk_nvme_ocssd_ns_cmd_vector_read_with_md;
+	spdk_nvme_ocssd_ns_cmd_vector_copy;
+
+	# public functions from opal.h
+	spdk_opal_dev_construct;
+	spdk_opal_dev_destruct;
+	spdk_opal_get_d0_features_info;
+	spdk_opal_supported;
+	spdk_opal_cmd_take_ownership;
+	spdk_opal_cmd_revert_tper;
+	spdk_opal_cmd_activate_locking_sp;
+	spdk_opal_cmd_lock_unlock;
+	spdk_opal_cmd_setup_locking_range;
+	spdk_opal_cmd_get_max_ranges;
+	spdk_opal_cmd_get_locking_range_info;
+	spdk_opal_cmd_enable_user;
+	spdk_opal_cmd_add_user_to_locking_range;
+	spdk_opal_cmd_set_new_passwd;
+	spdk_opal_cmd_erase_locking_range;
+	spdk_opal_cmd_secure_erase_locking_range;
+	spdk_opal_get_locking_range_info;
+	spdk_opal_free_locking_range_info;
+
+	local: *;
+};