diff options
Diffstat (limited to 'src/spdk/lib/vhost')
| -rw-r--r-- | src/spdk/lib/vhost/Makefile | 54 | ||||
| -rw-r--r-- | src/spdk/lib/vhost/rte_vhost_compat.c | 402 | ||||
| -rw-r--r-- | src/spdk/lib/vhost/spdk_vhost.map | 27 | ||||
| -rw-r--r-- | src/spdk/lib/vhost/vhost.c | 1634 | ||||
| -rw-r--r-- | src/spdk/lib/vhost/vhost_blk.c | 1354 | ||||
| -rw-r--r-- | src/spdk/lib/vhost/vhost_internal.h | 496 | ||||
| -rw-r--r-- | src/spdk/lib/vhost/vhost_nvme.c | 1500 | ||||
| -rw-r--r-- | src/spdk/lib/vhost/vhost_rpc.c | 652 | ||||
| -rw-r--r-- | src/spdk/lib/vhost/vhost_scsi.c | 1536 |
9 files changed, 7655 insertions, 0 deletions
diff --git a/src/spdk/lib/vhost/Makefile b/src/spdk/lib/vhost/Makefile new file mode 100644 index 000000000..1fe9b6e40 --- /dev/null +++ b/src/spdk/lib/vhost/Makefile @@ -0,0 +1,54 @@ +# +# BSD LICENSE +# +# Copyright (c) Intel Corporation. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..) +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk + +SO_VER := 4 +SO_MINOR := 0 + +CFLAGS += -I. +CFLAGS += $(ENV_CFLAGS) + +C_SRCS = vhost.c vhost_rpc.c vhost_scsi.c vhost_blk.c rte_vhost_compat.c + +ifeq ($(CONFIG_VHOST_INTERNAL_LIB),y) +C_SRCS += vhost_nvme.c +CFLAGS := -I../rte_vhost $(CFLAGS) +endif + +LIBNAME = vhost + +SPDK_MAP_FILE = $(abspath $(CURDIR)/spdk_vhost.map) + +include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk diff --git a/src/spdk/lib/vhost/rte_vhost_compat.c b/src/spdk/lib/vhost/rte_vhost_compat.c new file mode 100644 index 000000000..53f31bfd7 --- /dev/null +++ b/src/spdk/lib/vhost/rte_vhost_compat.c @@ -0,0 +1,402 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** \file + * Set of workarounds for rte_vhost to make it work with device types + * other than vhost-net. + */ + +#include "spdk/stdinc.h" + +#include "spdk/env.h" +#include "spdk/likely.h" +#include "spdk/string.h" +#include "spdk/util.h" +#include "spdk/memory.h" +#include "spdk/barrier.h" +#include "spdk/vhost.h" +#include "vhost_internal.h" + +#include "spdk_internal/vhost_user.h" + +static inline void +vhost_session_mem_region_calc(uint64_t *previous_start, uint64_t *start, uint64_t *end, + uint64_t *len, struct rte_vhost_mem_region *region) +{ + *start = FLOOR_2MB(region->mmap_addr); + *end = CEIL_2MB(region->mmap_addr + region->mmap_size); + if (*start == *previous_start) { + *start += (size_t) VALUE_2MB; + } + *previous_start = *start; + *len = *end - *start; +} + +void +vhost_session_mem_register(struct rte_vhost_memory *mem) +{ + uint64_t start, end, len; + uint32_t i; + uint64_t previous_start = UINT64_MAX; + + + for (i = 0; i < mem->nregions; i++) { + vhost_session_mem_region_calc(&previous_start, &start, &end, &len, &mem->regions[i]); + SPDK_INFOLOG(SPDK_LOG_VHOST, "Registering VM memory for vtophys translation - 0x%jx len:0x%jx\n", + start, len); + + if (spdk_mem_register((void *)start, len) != 0) { + SPDK_WARNLOG("Failed to register memory region %"PRIu32". Future vtophys translation might fail.\n", + i); + continue; + } + } +} + +void +vhost_session_mem_unregister(struct rte_vhost_memory *mem) +{ + uint64_t start, end, len; + uint32_t i; + uint64_t previous_start = UINT64_MAX; + + for (i = 0; i < mem->nregions; i++) { + vhost_session_mem_region_calc(&previous_start, &start, &end, &len, &mem->regions[i]); + if (spdk_vtophys((void *) start, NULL) == SPDK_VTOPHYS_ERROR) { + continue; /* region has not been registered */ + } + + if (spdk_mem_unregister((void *)start, len) != 0) { + assert(false); + } + } +} + +static int +new_connection(int vid) +{ + char ifname[PATH_MAX]; + + if (rte_vhost_get_ifname(vid, ifname, PATH_MAX) < 0) { + SPDK_ERRLOG("Couldn't get a valid ifname for device with vid %d\n", vid); + return -1; + } + + return vhost_new_connection_cb(vid, ifname); +} + +static int +start_device(int vid) +{ + return vhost_start_device_cb(vid); +} + +static void +stop_device(int vid) +{ + vhost_stop_device_cb(vid); +} + +static void +destroy_connection(int vid) +{ + vhost_destroy_connection_cb(vid); +} + +static const struct vhost_device_ops g_spdk_vhost_ops = { + .new_device = start_device, + .destroy_device = stop_device, + .new_connection = new_connection, + .destroy_connection = destroy_connection, +#ifdef SPDK_CONFIG_VHOST_INTERNAL_LIB + .get_config = vhost_get_config_cb, + .set_config = vhost_set_config_cb, + .vhost_nvme_admin_passthrough = vhost_nvme_admin_passthrough, + .vhost_nvme_set_cq_call = vhost_nvme_set_cq_call, + .vhost_nvme_get_cap = vhost_nvme_get_cap, + .vhost_nvme_set_bar_mr = vhost_nvme_set_bar_mr, +#endif +}; + +#ifndef SPDK_CONFIG_VHOST_INTERNAL_LIB + +static enum rte_vhost_msg_result +extern_vhost_pre_msg_handler(int vid, void *_msg) +{ + struct vhost_user_msg *msg = _msg; + struct spdk_vhost_session *vsession; + + vsession = vhost_session_find_by_vid(vid); + if (vsession == NULL) { + SPDK_ERRLOG("Received a message to unitialized session (vid %d).\n", vid); + assert(false); + return RTE_VHOST_MSG_RESULT_ERR; + } + + switch (msg->request) { + case VHOST_USER_GET_VRING_BASE: + if (vsession->forced_polling && vsession->started) { + /* Our queue is stopped for whatever reason, but we may still + * need to poll it after it's initialized again. + */ + g_spdk_vhost_ops.destroy_device(vid); + } + break; + case VHOST_USER_SET_VRING_BASE: + case VHOST_USER_SET_VRING_ADDR: + case VHOST_USER_SET_VRING_NUM: + case VHOST_USER_SET_VRING_KICK: + if (vsession->forced_polling && vsession->started) { + /* Additional queues are being initialized, so we either processed + * enough I/Os and are switching from SeaBIOS to the OS now, or + * we were never in SeaBIOS in the first place. Either way, we + * don't need our workaround anymore. + */ + g_spdk_vhost_ops.destroy_device(vid); + vsession->forced_polling = false; + } + break; + case VHOST_USER_SET_VRING_CALL: + /* rte_vhost will close the previous callfd and won't notify + * us about any change. This will effectively make SPDK fail + * to deliver any subsequent interrupts until a session is + * restarted. We stop the session here before closing the previous + * fd (so that all interrupts must have been delivered by the + * time the descriptor is closed) and start right after (which + * will make SPDK retrieve the latest, up-to-date callfd from + * rte_vhost. + */ + case VHOST_USER_SET_MEM_TABLE: + /* rte_vhost will unmap previous memory that SPDK may still + * have pending DMA operations on. We can't let that happen, + * so stop the device before letting rte_vhost unmap anything. + * This will block until all pending I/Os are finished. + * We will start the device again from the post-processing + * message handler. + */ + if (vsession->started) { + g_spdk_vhost_ops.destroy_device(vid); + vsession->needs_restart = true; + } + break; + case VHOST_USER_GET_CONFIG: { + int rc = 0; + + spdk_vhost_lock(); + if (vsession->vdev->backend->vhost_get_config) { + rc = vsession->vdev->backend->vhost_get_config(vsession->vdev, + msg->payload.cfg.region, msg->payload.cfg.size); + if (rc != 0) { + msg->size = 0; + } + } + spdk_vhost_unlock(); + + return RTE_VHOST_MSG_RESULT_REPLY; + } + case VHOST_USER_SET_CONFIG: { + int rc = 0; + + spdk_vhost_lock(); + if (vsession->vdev->backend->vhost_set_config) { + rc = vsession->vdev->backend->vhost_set_config(vsession->vdev, + msg->payload.cfg.region, msg->payload.cfg.offset, + msg->payload.cfg.size, msg->payload.cfg.flags); + } + spdk_vhost_unlock(); + + return rc == 0 ? RTE_VHOST_MSG_RESULT_OK : RTE_VHOST_MSG_RESULT_ERR; + } + default: + break; + } + + return RTE_VHOST_MSG_RESULT_NOT_HANDLED; +} + +static enum rte_vhost_msg_result +extern_vhost_post_msg_handler(int vid, void *_msg) +{ + struct vhost_user_msg *msg = _msg; + struct spdk_vhost_session *vsession; + + vsession = vhost_session_find_by_vid(vid); + if (vsession == NULL) { + SPDK_ERRLOG("Received a message to unitialized session (vid %d).\n", vid); + assert(false); + return RTE_VHOST_MSG_RESULT_ERR; + } + + if (vsession->needs_restart) { + g_spdk_vhost_ops.new_device(vid); + vsession->needs_restart = false; + return RTE_VHOST_MSG_RESULT_NOT_HANDLED; + } + + switch (msg->request) { + case VHOST_USER_SET_FEATURES: + /* rte_vhost requires all queues to be fully initialized in order + * to start I/O processing. This behavior is not compliant with the + * vhost-user specification and doesn't work with QEMU 2.12+, which + * will only initialize 1 I/O queue for the SeaBIOS boot. + * Theoretically, we should start polling each virtqueue individually + * after receiving its SET_VRING_KICK message, but rte_vhost is not + * designed to poll individual queues. So here we use a workaround + * to detect when the vhost session could be potentially at that SeaBIOS + * stage and we mark it to start polling as soon as its first virtqueue + * gets initialized. This doesn't hurt any non-QEMU vhost slaves + * and allows QEMU 2.12+ to boot correctly. SET_FEATURES could be sent + * at any time, but QEMU will send it at least once on SeaBIOS + * initialization - whenever powered-up or rebooted. + */ + vsession->forced_polling = true; + break; + case VHOST_USER_SET_VRING_KICK: + /* vhost-user spec tells us to start polling a queue after receiving + * its SET_VRING_KICK message. Let's do it! + */ + if (vsession->forced_polling && !vsession->started) { + g_spdk_vhost_ops.new_device(vid); + } + break; + default: + break; + } + + return RTE_VHOST_MSG_RESULT_NOT_HANDLED; +} + +struct rte_vhost_user_extern_ops g_spdk_extern_vhost_ops = { + .pre_msg_handle = extern_vhost_pre_msg_handler, + .post_msg_handle = extern_vhost_post_msg_handler, +}; + +void +vhost_session_install_rte_compat_hooks(struct spdk_vhost_session *vsession) +{ + int rc; + + rc = rte_vhost_extern_callback_register(vsession->vid, &g_spdk_extern_vhost_ops, NULL); + if (rc != 0) { + SPDK_ERRLOG("rte_vhost_extern_callback_register() failed for vid = %d\n", + vsession->vid); + return; + } +} + +#else /* SPDK_CONFIG_VHOST_INTERNAL_LIB */ + +void +vhost_session_install_rte_compat_hooks(struct spdk_vhost_session *vsession) +{ + /* nothing to do. all the changes are already incorporated into rte_vhost */ +} + +#endif + +int +vhost_register_unix_socket(const char *path, const char *ctrl_name, + uint64_t virtio_features, uint64_t disabled_features, uint64_t protocol_features) +{ + struct stat file_stat; +#ifndef SPDK_CONFIG_VHOST_INTERNAL_LIB + uint64_t features = 0; +#endif + + /* Register vhost driver to handle vhost messages. */ + if (stat(path, &file_stat) != -1) { + if (!S_ISSOCK(file_stat.st_mode)) { + SPDK_ERRLOG("Cannot create a domain socket at path \"%s\": " + "The file already exists and is not a socket.\n", + path); + return -EIO; + } else if (unlink(path) != 0) { + SPDK_ERRLOG("Cannot create a domain socket at path \"%s\": " + "The socket already exists and failed to unlink.\n", + path); + return -EIO; + } + } + + if (rte_vhost_driver_register(path, 0) != 0) { + SPDK_ERRLOG("Could not register controller %s with vhost library\n", ctrl_name); + SPDK_ERRLOG("Check if domain socket %s already exists\n", path); + return -EIO; + } + if (rte_vhost_driver_set_features(path, virtio_features) || + rte_vhost_driver_disable_features(path, disabled_features)) { + SPDK_ERRLOG("Couldn't set vhost features for controller %s\n", ctrl_name); + + rte_vhost_driver_unregister(path); + return -EIO; + } + + if (rte_vhost_driver_callback_register(path, &g_spdk_vhost_ops) != 0) { + rte_vhost_driver_unregister(path); + SPDK_ERRLOG("Couldn't register callbacks for controller %s\n", ctrl_name); + return -EIO; + } + +#ifndef SPDK_CONFIG_VHOST_INTERNAL_LIB + rte_vhost_driver_get_protocol_features(path, &features); + features |= protocol_features; + rte_vhost_driver_set_protocol_features(path, features); +#endif + + if (rte_vhost_driver_start(path) != 0) { + SPDK_ERRLOG("Failed to start vhost driver for controller %s (%d): %s\n", + ctrl_name, errno, spdk_strerror(errno)); + rte_vhost_driver_unregister(path); + return -EIO; + } + + return 0; +} + +int +vhost_get_mem_table(int vid, struct rte_vhost_memory **mem) +{ + return rte_vhost_get_mem_table(vid, mem); +} + +int +vhost_driver_unregister(const char *path) +{ + return rte_vhost_driver_unregister(path); +} + +int +vhost_get_negotiated_features(int vid, uint64_t *negotiated_features) +{ + return rte_vhost_get_negotiated_features(vid, negotiated_features); +} diff --git a/src/spdk/lib/vhost/spdk_vhost.map b/src/spdk/lib/vhost/spdk_vhost.map new file mode 100644 index 000000000..de38e5a5e --- /dev/null +++ b/src/spdk/lib/vhost/spdk_vhost.map @@ -0,0 +1,27 @@ +{ + global: + + # public functions + spdk_vhost_set_socket_path; + spdk_vhost_init; + spdk_vhost_fini; + spdk_vhost_config_json; + spdk_vhost_shutdown_cb; + spdk_vhost_lock; + spdk_vhost_trylock; + spdk_vhost_unlock; + spdk_vhost_dev_find; + spdk_vhost_dev_next; + spdk_vhost_dev_get_name; + spdk_vhost_dev_get_cpumask; + spdk_vhost_set_coalescing; + spdk_vhost_get_coalescing; + spdk_vhost_scsi_dev_construct; + spdk_vhost_scsi_dev_add_tgt; + spdk_vhost_scsi_dev_get_tgt; + spdk_vhost_scsi_dev_remove_tgt; + spdk_vhost_blk_construct; + spdk_vhost_dev_remove; + + local: *; +}; diff --git a/src/spdk/lib/vhost/vhost.c b/src/spdk/lib/vhost/vhost.c new file mode 100644 index 000000000..b904d8bf9 --- /dev/null +++ b/src/spdk/lib/vhost/vhost.c @@ -0,0 +1,1634 @@ +/*- + * BSD LICENSE + * + * Copyright(c) Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "spdk/stdinc.h" + +#include "spdk/env.h" +#include "spdk/likely.h" +#include "spdk/string.h" +#include "spdk/util.h" +#include "spdk/memory.h" +#include "spdk/barrier.h" +#include "spdk/vhost.h" +#include "vhost_internal.h" + +static struct spdk_cpuset g_vhost_core_mask; + +/* Path to folder where character device will be created. Can be set by user. */ +static char dev_dirname[PATH_MAX] = ""; + +/* Thread performing all vhost management operations */ +static struct spdk_thread *g_vhost_init_thread; + +static spdk_vhost_fini_cb g_fini_cpl_cb; + +/** + * DPDK calls our callbacks synchronously but the work those callbacks + * perform needs to be async. Luckily, all DPDK callbacks are called on + * a DPDK-internal pthread, so we'll just wait on a semaphore in there. + */ +static sem_t g_dpdk_sem; + +/** Return code for the current DPDK callback */ +static int g_dpdk_response; + +struct vhost_session_fn_ctx { + /** Device pointer obtained before enqueuing the event */ + struct spdk_vhost_dev *vdev; + + /** ID of the session to send event to. */ + uint32_t vsession_id; + + /** User provided function to be executed on session's thread. */ + spdk_vhost_session_fn cb_fn; + + /** + * User provided function to be called on the init thread + * after iterating through all sessions. + */ + spdk_vhost_dev_fn cpl_fn; + + /** Custom user context */ + void *user_ctx; +}; + +static TAILQ_HEAD(, spdk_vhost_dev) g_vhost_devices = TAILQ_HEAD_INITIALIZER( + g_vhost_devices); +static pthread_mutex_t g_vhost_mutex = PTHREAD_MUTEX_INITIALIZER; + +void *vhost_gpa_to_vva(struct spdk_vhost_session *vsession, uint64_t addr, uint64_t len) +{ + void *vva; + uint64_t newlen; + + newlen = len; + vva = (void *)rte_vhost_va_from_guest_pa(vsession->mem, addr, &newlen); + if (newlen != len) { + return NULL; + } + + return vva; + +} + +static void +vhost_log_req_desc(struct spdk_vhost_session *vsession, struct spdk_vhost_virtqueue *virtqueue, + uint16_t req_id) +{ + struct vring_desc *desc, *desc_table; + uint32_t desc_table_size; + int rc; + + if (spdk_likely(!vhost_dev_has_feature(vsession, VHOST_F_LOG_ALL))) { + return; + } + + rc = vhost_vq_get_desc(vsession, virtqueue, req_id, &desc, &desc_table, &desc_table_size); + if (spdk_unlikely(rc != 0)) { + SPDK_ERRLOG("Can't log used ring descriptors!\n"); + return; + } + + do { + if (vhost_vring_desc_is_wr(desc)) { + /* To be honest, only pages realy touched should be logged, but + * doing so would require tracking those changes in each backed. + * Also backend most likely will touch all/most of those pages so + * for lets assume we touched all pages passed to as writeable buffers. */ + rte_vhost_log_write(vsession->vid, desc->addr, desc->len); + } + vhost_vring_desc_get_next(&desc, desc_table, desc_table_size); + } while (desc); +} + +static void +vhost_log_used_vring_elem(struct spdk_vhost_session *vsession, + struct spdk_vhost_virtqueue *virtqueue, + uint16_t idx) +{ + uint64_t offset, len; + + if (spdk_likely(!vhost_dev_has_feature(vsession, VHOST_F_LOG_ALL))) { + return; + } + + if (spdk_unlikely(virtqueue->packed.packed_ring)) { + offset = idx * sizeof(struct vring_packed_desc); + len = sizeof(struct vring_packed_desc); + } else { + offset = offsetof(struct vring_used, ring[idx]); + len = sizeof(virtqueue->vring.used->ring[idx]); + } + + rte_vhost_log_used_vring(vsession->vid, virtqueue->vring_idx, offset, len); +} + +static void +vhost_log_used_vring_idx(struct spdk_vhost_session *vsession, + struct spdk_vhost_virtqueue *virtqueue) +{ + uint64_t offset, len; + uint16_t vq_idx; + + if (spdk_likely(!vhost_dev_has_feature(vsession, VHOST_F_LOG_ALL))) { + return; + } + + offset = offsetof(struct vring_used, idx); + len = sizeof(virtqueue->vring.used->idx); + vq_idx = virtqueue - vsession->virtqueue; + + rte_vhost_log_used_vring(vsession->vid, vq_idx, offset, len); +} + +/* + * Get available requests from avail ring. + */ +uint16_t +vhost_vq_avail_ring_get(struct spdk_vhost_virtqueue *virtqueue, uint16_t *reqs, + uint16_t reqs_len) +{ + struct rte_vhost_vring *vring = &virtqueue->vring; + struct vring_avail *avail = vring->avail; + uint16_t size_mask = vring->size - 1; + uint16_t last_idx = virtqueue->last_avail_idx, avail_idx = avail->idx; + uint16_t count, i; + + count = avail_idx - last_idx; + if (spdk_likely(count == 0)) { + return 0; + } + + if (spdk_unlikely(count > vring->size)) { + /* TODO: the queue is unrecoverably broken and should be marked so. + * For now we will fail silently and report there are no new avail entries. + */ + return 0; + } + + count = spdk_min(count, reqs_len); + virtqueue->last_avail_idx += count; + for (i = 0; i < count; i++) { + reqs[i] = vring->avail->ring[(last_idx + i) & size_mask]; + } + + SPDK_DEBUGLOG(SPDK_LOG_VHOST_RING, + "AVAIL: last_idx=%"PRIu16" avail_idx=%"PRIu16" count=%"PRIu16"\n", + last_idx, avail_idx, count); + + return count; +} + +static bool +vhost_vring_desc_is_indirect(struct vring_desc *cur_desc) +{ + return !!(cur_desc->flags & VRING_DESC_F_INDIRECT); +} + +static bool +vhost_vring_packed_desc_is_indirect(struct vring_packed_desc *cur_desc) +{ + return (cur_desc->flags & VRING_DESC_F_INDIRECT) != 0; +} + +int +vhost_vq_get_desc(struct spdk_vhost_session *vsession, struct spdk_vhost_virtqueue *virtqueue, + uint16_t req_idx, struct vring_desc **desc, struct vring_desc **desc_table, + uint32_t *desc_table_size) +{ + if (spdk_unlikely(req_idx >= virtqueue->vring.size)) { + return -1; + } + + *desc = &virtqueue->vring.desc[req_idx]; + + if (vhost_vring_desc_is_indirect(*desc)) { + *desc_table_size = (*desc)->len / sizeof(**desc); + *desc_table = vhost_gpa_to_vva(vsession, (*desc)->addr, + sizeof(**desc) * *desc_table_size); + *desc = *desc_table; + if (*desc == NULL) { + return -1; + } + + return 0; + } + + *desc_table = virtqueue->vring.desc; + *desc_table_size = virtqueue->vring.size; + + return 0; +} + +int +vhost_vq_get_desc_packed(struct spdk_vhost_session *vsession, + struct spdk_vhost_virtqueue *virtqueue, + uint16_t req_idx, struct vring_packed_desc **desc, + struct vring_packed_desc **desc_table, uint32_t *desc_table_size) +{ + *desc = &virtqueue->vring.desc_packed[req_idx]; + + /* In packed ring when the desc is non-indirect we get next desc + * by judging (desc->flag & VRING_DESC_F_NEXT) != 0. When the desc + * is indirect we get next desc by idx and desc_table_size. It's + * different from split ring. + */ + if (vhost_vring_packed_desc_is_indirect(*desc)) { + *desc_table_size = (*desc)->len / sizeof(struct vring_packed_desc); + *desc_table = vhost_gpa_to_vva(vsession, (*desc)->addr, + (*desc)->len); + *desc = *desc_table; + if (spdk_unlikely(*desc == NULL)) { + return -1; + } + } else { + *desc_table = NULL; + *desc_table_size = 0; + } + + return 0; +} + +int +vhost_vq_used_signal(struct spdk_vhost_session *vsession, + struct spdk_vhost_virtqueue *virtqueue) +{ + if (virtqueue->used_req_cnt == 0) { + return 0; + } + + virtqueue->req_cnt += virtqueue->used_req_cnt; + virtqueue->used_req_cnt = 0; + + SPDK_DEBUGLOG(SPDK_LOG_VHOST_RING, + "Queue %td - USED RING: sending IRQ: last used %"PRIu16"\n", + virtqueue - vsession->virtqueue, virtqueue->last_used_idx); + + if (rte_vhost_vring_call(vsession->vid, virtqueue->vring_idx) == 0) { + /* interrupt signalled */ + return 1; + } else { + /* interrupt not signalled */ + return 0; + } +} + + +static void +check_session_io_stats(struct spdk_vhost_session *vsession, uint64_t now) +{ + struct spdk_vhost_virtqueue *virtqueue; + uint32_t irq_delay_base = vsession->coalescing_delay_time_base; + uint32_t io_threshold = vsession->coalescing_io_rate_threshold; + int32_t irq_delay; + uint32_t req_cnt; + uint16_t q_idx; + + if (now < vsession->next_stats_check_time) { + return; + } + + vsession->next_stats_check_time = now + vsession->stats_check_interval; + for (q_idx = 0; q_idx < vsession->max_queues; q_idx++) { + virtqueue = &vsession->virtqueue[q_idx]; + + req_cnt = virtqueue->req_cnt + virtqueue->used_req_cnt; + if (req_cnt <= io_threshold) { + continue; + } + + irq_delay = (irq_delay_base * (req_cnt - io_threshold)) / io_threshold; + virtqueue->irq_delay_time = (uint32_t) spdk_max(0, irq_delay); + + virtqueue->req_cnt = 0; + virtqueue->next_event_time = now; + } +} + +static inline bool +vhost_vq_event_is_suppressed(struct spdk_vhost_virtqueue *vq) +{ + if (spdk_unlikely(vq->packed.packed_ring)) { + if (vq->vring.driver_event->flags & VRING_PACKED_EVENT_FLAG_DISABLE) { + return true; + } + } else { + if (vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT) { + return true; + } + } + + return false; +} + +void +vhost_session_used_signal(struct spdk_vhost_session *vsession) +{ + struct spdk_vhost_virtqueue *virtqueue; + uint64_t now; + uint16_t q_idx; + + if (vsession->coalescing_delay_time_base == 0) { + for (q_idx = 0; q_idx < vsession->max_queues; q_idx++) { + virtqueue = &vsession->virtqueue[q_idx]; + + if (virtqueue->vring.desc == NULL) { + continue; + } + + if (vhost_vq_event_is_suppressed(virtqueue)) { + continue; + } + + vhost_vq_used_signal(vsession, virtqueue); + } + } else { + now = spdk_get_ticks(); + check_session_io_stats(vsession, now); + + for (q_idx = 0; q_idx < vsession->max_queues; q_idx++) { + virtqueue = &vsession->virtqueue[q_idx]; + + /* No need for event right now */ + if (now < virtqueue->next_event_time) { + continue; + } + + if (vhost_vq_event_is_suppressed(virtqueue)) { + continue; + } + + if (!vhost_vq_used_signal(vsession, virtqueue)) { + continue; + } + + /* Syscall is quite long so update time */ + now = spdk_get_ticks(); + virtqueue->next_event_time = now + virtqueue->irq_delay_time; + } + } +} + +static int +vhost_session_set_coalescing(struct spdk_vhost_dev *vdev, + struct spdk_vhost_session *vsession, void *ctx) +{ + vsession->coalescing_delay_time_base = + vdev->coalescing_delay_us * spdk_get_ticks_hz() / 1000000ULL; + vsession->coalescing_io_rate_threshold = + vdev->coalescing_iops_threshold * SPDK_VHOST_STATS_CHECK_INTERVAL_MS / 1000U; + return 0; +} + +static int +vhost_dev_set_coalescing(struct spdk_vhost_dev *vdev, uint32_t delay_base_us, + uint32_t iops_threshold) +{ + uint64_t delay_time_base = delay_base_us * spdk_get_ticks_hz() / 1000000ULL; + uint32_t io_rate = iops_threshold * SPDK_VHOST_STATS_CHECK_INTERVAL_MS / 1000U; + + if (delay_time_base >= UINT32_MAX) { + SPDK_ERRLOG("Delay time of %"PRIu32" is to big\n", delay_base_us); + return -EINVAL; + } else if (io_rate == 0) { + SPDK_ERRLOG("IOPS rate of %"PRIu32" is too low. Min is %u\n", io_rate, + 1000U / SPDK_VHOST_STATS_CHECK_INTERVAL_MS); + return -EINVAL; + } + + vdev->coalescing_delay_us = delay_base_us; + vdev->coalescing_iops_threshold = iops_threshold; + return 0; +} + +int +spdk_vhost_set_coalescing(struct spdk_vhost_dev *vdev, uint32_t delay_base_us, + uint32_t iops_threshold) +{ + int rc; + + rc = vhost_dev_set_coalescing(vdev, delay_base_us, iops_threshold); + if (rc != 0) { + return rc; + } + + vhost_dev_foreach_session(vdev, vhost_session_set_coalescing, NULL, NULL); + return 0; +} + +void +spdk_vhost_get_coalescing(struct spdk_vhost_dev *vdev, uint32_t *delay_base_us, + uint32_t *iops_threshold) +{ + if (delay_base_us) { + *delay_base_us = vdev->coalescing_delay_us; + } + + if (iops_threshold) { + *iops_threshold = vdev->coalescing_iops_threshold; + } +} + +/* + * Enqueue id and len to used ring. + */ +void +vhost_vq_used_ring_enqueue(struct spdk_vhost_session *vsession, + struct spdk_vhost_virtqueue *virtqueue, + uint16_t id, uint32_t len) +{ + struct rte_vhost_vring *vring = &virtqueue->vring; + struct vring_used *used = vring->used; + uint16_t last_idx = virtqueue->last_used_idx & (vring->size - 1); + uint16_t vq_idx = virtqueue->vring_idx; + + SPDK_DEBUGLOG(SPDK_LOG_VHOST_RING, + "Queue %td - USED RING: last_idx=%"PRIu16" req id=%"PRIu16" len=%"PRIu32"\n", + virtqueue - vsession->virtqueue, virtqueue->last_used_idx, id, len); + + vhost_log_req_desc(vsession, virtqueue, id); + + virtqueue->last_used_idx++; + used->ring[last_idx].id = id; + used->ring[last_idx].len = len; + + /* Ensure the used ring is updated before we log it or increment used->idx. */ + spdk_smp_wmb(); + + rte_vhost_set_last_inflight_io_split(vsession->vid, vq_idx, id); + + vhost_log_used_vring_elem(vsession, virtqueue, last_idx); + * (volatile uint16_t *) &used->idx = virtqueue->last_used_idx; + vhost_log_used_vring_idx(vsession, virtqueue); + + rte_vhost_clr_inflight_desc_split(vsession->vid, vq_idx, virtqueue->last_used_idx, id); + + virtqueue->used_req_cnt++; +} + +void +vhost_vq_packed_ring_enqueue(struct spdk_vhost_session *vsession, + struct spdk_vhost_virtqueue *virtqueue, + uint16_t num_descs, uint16_t buffer_id, + uint32_t length) +{ + struct vring_packed_desc *desc = &virtqueue->vring.desc_packed[virtqueue->last_used_idx]; + bool used, avail; + + SPDK_DEBUGLOG(SPDK_LOG_VHOST_RING, + "Queue %td - RING: buffer_id=%"PRIu16"\n", + virtqueue - vsession->virtqueue, buffer_id); + + /* When the descriptor is used, two flags in descriptor + * avail flag and used flag are set to equal + * and used flag value == used_wrap_counter. + */ + used = !!(desc->flags & VRING_DESC_F_USED); + avail = !!(desc->flags & VRING_DESC_F_AVAIL); + if (spdk_unlikely(used == virtqueue->packed.used_phase && used == avail)) { + SPDK_ERRLOG("descriptor has been used before\n"); + return; + } + + /* In used desc addr is unused and len specifies the buffer length + * that has been written to by the device. + */ + desc->addr = 0; + desc->len = length; + + /* This bit specifies whether any data has been written by the device */ + if (length != 0) { + desc->flags |= VRING_DESC_F_WRITE; + } + + /* Buffer ID is included in the last descriptor in the list. + * The driver needs to keep track of the size of the list corresponding + * to each buffer ID. + */ + desc->id = buffer_id; + + /* A device MUST NOT make the descriptor used before buffer_id is + * written to the descriptor. + */ + spdk_smp_wmb(); + /* To mark a desc as used, the device sets the F_USED bit in flags to match + * the internal Device ring wrap counter. It also sets the F_AVAIL bit to + * match the same value. + */ + if (virtqueue->packed.used_phase) { + desc->flags |= VRING_DESC_F_AVAIL_USED; + } else { + desc->flags &= ~VRING_DESC_F_AVAIL_USED; + } + + vhost_log_used_vring_elem(vsession, virtqueue, virtqueue->last_used_idx); + virtqueue->last_used_idx += num_descs; + if (virtqueue->last_used_idx >= virtqueue->vring.size) { + virtqueue->last_used_idx -= virtqueue->vring.size; + virtqueue->packed.used_phase = !virtqueue->packed.used_phase; + } + + virtqueue->used_req_cnt++; +} + +bool +vhost_vq_packed_ring_is_avail(struct spdk_vhost_virtqueue *virtqueue) +{ + uint16_t flags = virtqueue->vring.desc_packed[virtqueue->last_avail_idx].flags; + + /* To mark a desc as available, the driver sets the F_AVAIL bit in flags + * to match the internal avail wrap counter. It also sets the F_USED bit to + * match the inverse value but it's not mandatory. + */ + return (!!(flags & VRING_DESC_F_AVAIL) == virtqueue->packed.avail_phase); +} + +bool +vhost_vring_packed_desc_is_wr(struct vring_packed_desc *cur_desc) +{ + return (cur_desc->flags & VRING_DESC_F_WRITE) != 0; +} + +int +vhost_vring_packed_desc_get_next(struct vring_packed_desc **desc, uint16_t *req_idx, + struct spdk_vhost_virtqueue *vq, + struct vring_packed_desc *desc_table, + uint32_t desc_table_size) +{ + if (desc_table != NULL) { + /* When the desc_table isn't NULL means it's indirect and we get the next + * desc by req_idx and desc_table_size. The return value is NULL means + * we reach the last desc of this request. + */ + (*req_idx)++; + if (*req_idx < desc_table_size) { + *desc = &desc_table[*req_idx]; + } else { + *desc = NULL; + } + } else { + /* When the desc_table is NULL means it's non-indirect and we get the next + * desc by req_idx and F_NEXT in flags. The return value is NULL means + * we reach the last desc of this request. When return new desc + * we update the req_idx too. + */ + if (((*desc)->flags & VRING_DESC_F_NEXT) == 0) { + *desc = NULL; + return 0; + } + + *req_idx = (*req_idx + 1) % vq->vring.size; + *desc = &vq->vring.desc_packed[*req_idx]; + } + + return 0; +} + +static int +vhost_vring_desc_payload_to_iov(struct spdk_vhost_session *vsession, struct iovec *iov, + uint16_t *iov_index, uintptr_t payload, uint64_t remaining) +{ + uintptr_t vva; + uint64_t len; + + do { + if (*iov_index >= SPDK_VHOST_IOVS_MAX) { + SPDK_ERRLOG("SPDK_VHOST_IOVS_MAX(%d) reached\n", SPDK_VHOST_IOVS_MAX); + return -1; + } + len = remaining; + vva = (uintptr_t)rte_vhost_va_from_guest_pa(vsession->mem, payload, &len); + if (vva == 0 || len == 0) { + SPDK_ERRLOG("gpa_to_vva(%p) == NULL\n", (void *)payload); + return -1; + } + iov[*iov_index].iov_base = (void *)vva; + iov[*iov_index].iov_len = len; + remaining -= len; + payload += len; + (*iov_index)++; + } while (remaining); + + return 0; +} + +int +vhost_vring_packed_desc_to_iov(struct spdk_vhost_session *vsession, struct iovec *iov, + uint16_t *iov_index, const struct vring_packed_desc *desc) +{ + return vhost_vring_desc_payload_to_iov(vsession, iov, iov_index, + desc->addr, desc->len); +} + +/* 1, Traverse the desc chain to get the buffer_id and return buffer_id as task_idx. + * 2, Update the vq->last_avail_idx to point next available desc chain. + * 3, Update the avail_wrap_counter if last_avail_idx overturn. + */ +uint16_t +vhost_vring_packed_desc_get_buffer_id(struct spdk_vhost_virtqueue *vq, uint16_t req_idx, + uint16_t *num_descs) +{ + struct vring_packed_desc *desc; + uint16_t desc_head = req_idx; + + *num_descs = 1; + + desc = &vq->vring.desc_packed[req_idx]; + if (!vhost_vring_packed_desc_is_indirect(desc)) { + while ((desc->flags & VRING_DESC_F_NEXT) != 0) { + req_idx = (req_idx + 1) % vq->vring.size; + desc = &vq->vring.desc_packed[req_idx]; + (*num_descs)++; + } + } + + /* Queue Size doesn't have to be a power of 2 + * Device maintains last_avail_idx so we can make sure + * the value is valid(0 ~ vring.size - 1) + */ + vq->last_avail_idx = (req_idx + 1) % vq->vring.size; + if (vq->last_avail_idx < desc_head) { + vq->packed.avail_phase = !vq->packed.avail_phase; + } + + return desc->id; +} + +int +vhost_vring_desc_get_next(struct vring_desc **desc, + struct vring_desc *desc_table, uint32_t desc_table_size) +{ + struct vring_desc *old_desc = *desc; + uint16_t next_idx; + + if ((old_desc->flags & VRING_DESC_F_NEXT) == 0) { + *desc = NULL; + return 0; + } + + next_idx = old_desc->next; + if (spdk_unlikely(next_idx >= desc_table_size)) { + *desc = NULL; + return -1; + } + + *desc = &desc_table[next_idx]; + return 0; +} + +int +vhost_vring_desc_to_iov(struct spdk_vhost_session *vsession, struct iovec *iov, + uint16_t *iov_index, const struct vring_desc *desc) +{ + return vhost_vring_desc_payload_to_iov(vsession, iov, iov_index, + desc->addr, desc->len); +} + +static struct spdk_vhost_session * +vhost_session_find_by_id(struct spdk_vhost_dev *vdev, unsigned id) +{ + struct spdk_vhost_session *vsession; + + TAILQ_FOREACH(vsession, &vdev->vsessions, tailq) { + if (vsession->id == id) { + return vsession; + } + } + + return NULL; +} + +struct spdk_vhost_session * +vhost_session_find_by_vid(int vid) +{ + struct spdk_vhost_dev *vdev; + struct spdk_vhost_session *vsession; + + TAILQ_FOREACH(vdev, &g_vhost_devices, tailq) { + TAILQ_FOREACH(vsession, &vdev->vsessions, tailq) { + if (vsession->vid == vid) { + return vsession; + } + } + } + + return NULL; +} + +struct spdk_vhost_dev * +spdk_vhost_dev_next(struct spdk_vhost_dev *vdev) +{ + if (vdev == NULL) { + return TAILQ_FIRST(&g_vhost_devices); + } + + return TAILQ_NEXT(vdev, tailq); +} + +struct spdk_vhost_dev * +spdk_vhost_dev_find(const char *ctrlr_name) +{ + struct spdk_vhost_dev *vdev; + size_t dev_dirname_len = strlen(dev_dirname); + + if (strncmp(ctrlr_name, dev_dirname, dev_dirname_len) == 0) { + ctrlr_name += dev_dirname_len; + } + + TAILQ_FOREACH(vdev, &g_vhost_devices, tailq) { + if (strcmp(vdev->name, ctrlr_name) == 0) { + return vdev; + } + } + + return NULL; +} + +static int +vhost_parse_core_mask(const char *mask, struct spdk_cpuset *cpumask) +{ + int rc; + + if (cpumask == NULL) { + return -1; + } + + if (mask == NULL) { + spdk_cpuset_copy(cpumask, &g_vhost_core_mask); + return 0; + } + + rc = spdk_cpuset_parse(cpumask, mask); + if (rc < 0) { + SPDK_ERRLOG("invalid cpumask %s\n", mask); + return -1; + } + + spdk_cpuset_and(cpumask, &g_vhost_core_mask); + + if (spdk_cpuset_count(cpumask) == 0) { + SPDK_ERRLOG("no cpu is selected among core mask(=%s)\n", + spdk_cpuset_fmt(&g_vhost_core_mask)); + return -1; + } + + return 0; +} + +static void +vhost_setup_core_mask(void *ctx) +{ + struct spdk_thread *thread = spdk_get_thread(); + spdk_cpuset_or(&g_vhost_core_mask, spdk_thread_get_cpumask(thread)); +} + +static void +vhost_setup_core_mask_done(void *ctx) +{ + spdk_vhost_init_cb init_cb = ctx; + + if (spdk_cpuset_count(&g_vhost_core_mask) == 0) { + init_cb(-ECHILD); + return; + } + + init_cb(0); +} + +static void +vhost_dev_thread_exit(void *arg1) +{ + spdk_thread_exit(spdk_get_thread()); +} + +int +vhost_dev_register(struct spdk_vhost_dev *vdev, const char *name, const char *mask_str, + const struct spdk_vhost_dev_backend *backend) +{ + char path[PATH_MAX]; + struct spdk_cpuset cpumask = {}; + int rc; + + assert(vdev); + if (name == NULL) { + SPDK_ERRLOG("Can't register controller with no name\n"); + return -EINVAL; + } + + if (vhost_parse_core_mask(mask_str, &cpumask) != 0) { + SPDK_ERRLOG("cpumask %s is invalid (core mask is 0x%s)\n", + mask_str, spdk_cpuset_fmt(&g_vhost_core_mask)); + return -EINVAL; + } + + if (spdk_vhost_dev_find(name)) { + SPDK_ERRLOG("vhost controller %s already exists.\n", name); + return -EEXIST; + } + + if (snprintf(path, sizeof(path), "%s%s", dev_dirname, name) >= (int)sizeof(path)) { + SPDK_ERRLOG("Resulting socket path for controller %s is too long: %s%s\n", name, dev_dirname, + name); + return -EINVAL; + } + + vdev->name = strdup(name); + vdev->path = strdup(path); + if (vdev->name == NULL || vdev->path == NULL) { + rc = -EIO; + goto out; + } + + vdev->thread = spdk_thread_create(vdev->name, &cpumask); + if (vdev->thread == NULL) { + SPDK_ERRLOG("Failed to create thread for vhost controller %s.\n", name); + rc = -EIO; + goto out; + } + + vdev->registered = true; + vdev->backend = backend; + TAILQ_INIT(&vdev->vsessions); + + vhost_dev_set_coalescing(vdev, SPDK_VHOST_COALESCING_DELAY_BASE_US, + SPDK_VHOST_VQ_IOPS_COALESCING_THRESHOLD); + + if (vhost_register_unix_socket(path, name, vdev->virtio_features, vdev->disabled_features, + vdev->protocol_features)) { + spdk_thread_send_msg(vdev->thread, vhost_dev_thread_exit, NULL); + rc = -EIO; + goto out; + } + + TAILQ_INSERT_TAIL(&g_vhost_devices, vdev, tailq); + + SPDK_INFOLOG(SPDK_LOG_VHOST, "Controller %s: new controller added\n", vdev->name); + return 0; + +out: + free(vdev->name); + free(vdev->path); + return rc; +} + +int +vhost_dev_unregister(struct spdk_vhost_dev *vdev) +{ + if (!TAILQ_EMPTY(&vdev->vsessions)) { + SPDK_ERRLOG("Controller %s has still valid connection.\n", vdev->name); + return -EBUSY; + } + + if (vdev->registered && vhost_driver_unregister(vdev->path) != 0) { + SPDK_ERRLOG("Could not unregister controller %s with vhost library\n" + "Check if domain socket %s still exists\n", + vdev->name, vdev->path); + return -EIO; + } + + SPDK_INFOLOG(SPDK_LOG_VHOST, "Controller %s: removed\n", vdev->name); + + spdk_thread_send_msg(vdev->thread, vhost_dev_thread_exit, NULL); + + free(vdev->name); + free(vdev->path); + TAILQ_REMOVE(&g_vhost_devices, vdev, tailq); + return 0; +} + +const char * +spdk_vhost_dev_get_name(struct spdk_vhost_dev *vdev) +{ + assert(vdev != NULL); + return vdev->name; +} + +const struct spdk_cpuset * +spdk_vhost_dev_get_cpumask(struct spdk_vhost_dev *vdev) +{ + assert(vdev != NULL); + return spdk_thread_get_cpumask(vdev->thread); +} + +static void +wait_for_semaphore(int timeout_sec, const char *errmsg) +{ + struct timespec timeout; + int rc; + + clock_gettime(CLOCK_REALTIME, &timeout); + timeout.tv_sec += timeout_sec; + rc = sem_timedwait(&g_dpdk_sem, &timeout); + if (rc != 0) { + SPDK_ERRLOG("Timeout waiting for event: %s.\n", errmsg); + sem_wait(&g_dpdk_sem); + } +} + +static void +vhost_session_cb_done(int rc) +{ + g_dpdk_response = rc; + sem_post(&g_dpdk_sem); +} + +void +vhost_session_start_done(struct spdk_vhost_session *vsession, int response) +{ + if (response == 0) { + vsession->started = true; + + assert(vsession->vdev->active_session_num < UINT32_MAX); + vsession->vdev->active_session_num++; + } + + vhost_session_cb_done(response); +} + +void +vhost_session_stop_done(struct spdk_vhost_session *vsession, int response) +{ + if (response == 0) { + vsession->started = false; + + assert(vsession->vdev->active_session_num > 0); + vsession->vdev->active_session_num--; + } + + vhost_session_cb_done(response); +} + +static void +vhost_event_cb(void *arg1) +{ + struct vhost_session_fn_ctx *ctx = arg1; + struct spdk_vhost_session *vsession; + + if (pthread_mutex_trylock(&g_vhost_mutex) != 0) { + spdk_thread_send_msg(spdk_get_thread(), vhost_event_cb, arg1); + return; + } + + vsession = vhost_session_find_by_id(ctx->vdev, ctx->vsession_id); + ctx->cb_fn(ctx->vdev, vsession, NULL); + pthread_mutex_unlock(&g_vhost_mutex); +} + +int +vhost_session_send_event(struct spdk_vhost_session *vsession, + spdk_vhost_session_fn cb_fn, unsigned timeout_sec, + const char *errmsg) +{ + struct vhost_session_fn_ctx ev_ctx = {0}; + struct spdk_vhost_dev *vdev = vsession->vdev; + + ev_ctx.vdev = vdev; + ev_ctx.vsession_id = vsession->id; + ev_ctx.cb_fn = cb_fn; + + spdk_thread_send_msg(vdev->thread, vhost_event_cb, &ev_ctx); + + pthread_mutex_unlock(&g_vhost_mutex); + wait_for_semaphore(timeout_sec, errmsg); + pthread_mutex_lock(&g_vhost_mutex); + + return g_dpdk_response; +} + +static void +foreach_session_finish_cb(void *arg1) +{ + struct vhost_session_fn_ctx *ev_ctx = arg1; + struct spdk_vhost_dev *vdev = ev_ctx->vdev; + + if (pthread_mutex_trylock(&g_vhost_mutex) != 0) { + spdk_thread_send_msg(spdk_get_thread(), + foreach_session_finish_cb, arg1); + return; + } + + assert(vdev->pending_async_op_num > 0); + vdev->pending_async_op_num--; + if (ev_ctx->cpl_fn != NULL) { + ev_ctx->cpl_fn(vdev, ev_ctx->user_ctx); + } + + pthread_mutex_unlock(&g_vhost_mutex); + free(ev_ctx); +} + +static void +foreach_session(void *arg1) +{ + struct vhost_session_fn_ctx *ev_ctx = arg1; + struct spdk_vhost_session *vsession; + struct spdk_vhost_dev *vdev = ev_ctx->vdev; + int rc; + + if (pthread_mutex_trylock(&g_vhost_mutex) != 0) { + spdk_thread_send_msg(spdk_get_thread(), foreach_session, arg1); + return; + } + + TAILQ_FOREACH(vsession, &vdev->vsessions, tailq) { + if (vsession->initialized) { + rc = ev_ctx->cb_fn(vdev, vsession, ev_ctx->user_ctx); + if (rc < 0) { + goto out; + } + } + } + +out: + pthread_mutex_unlock(&g_vhost_mutex); + + spdk_thread_send_msg(g_vhost_init_thread, foreach_session_finish_cb, arg1); +} + +void +vhost_dev_foreach_session(struct spdk_vhost_dev *vdev, + spdk_vhost_session_fn fn, + spdk_vhost_dev_fn cpl_fn, + void *arg) +{ + struct vhost_session_fn_ctx *ev_ctx; + + ev_ctx = calloc(1, sizeof(*ev_ctx)); + if (ev_ctx == NULL) { + SPDK_ERRLOG("Failed to alloc vhost event.\n"); + assert(false); + return; + } + + ev_ctx->vdev = vdev; + ev_ctx->cb_fn = fn; + ev_ctx->cpl_fn = cpl_fn; + ev_ctx->user_ctx = arg; + + assert(vdev->pending_async_op_num < UINT32_MAX); + vdev->pending_async_op_num++; + + spdk_thread_send_msg(vdev->thread, foreach_session, ev_ctx); +} + +static int +_stop_session(struct spdk_vhost_session *vsession) +{ + struct spdk_vhost_dev *vdev = vsession->vdev; + struct spdk_vhost_virtqueue *q; + int rc; + uint16_t i; + + rc = vdev->backend->stop_session(vsession); + if (rc != 0) { + SPDK_ERRLOG("Couldn't stop device with vid %d.\n", vsession->vid); + pthread_mutex_unlock(&g_vhost_mutex); + return rc; + } + + for (i = 0; i < vsession->max_queues; i++) { + q = &vsession->virtqueue[i]; + + /* vring.desc and vring.desc_packed are in a union struct + * so q->vring.desc can replace q->vring.desc_packed. + */ + if (q->vring.desc == NULL) { + continue; + } + + /* Packed virtqueues support up to 2^15 entries each + * so left one bit can be used as wrap counter. + */ + if (q->packed.packed_ring) { + q->last_avail_idx = q->last_avail_idx | + ((uint16_t)q->packed.avail_phase << 15); + q->last_used_idx = q->last_used_idx | + ((uint16_t)q->packed.used_phase << 15); + } + + rte_vhost_set_vring_base(vsession->vid, i, q->last_avail_idx, q->last_used_idx); + } + + vhost_session_mem_unregister(vsession->mem); + free(vsession->mem); + + return 0; +} + +int +vhost_stop_device_cb(int vid) +{ + struct spdk_vhost_session *vsession; + int rc; + + pthread_mutex_lock(&g_vhost_mutex); + vsession = vhost_session_find_by_vid(vid); + if (vsession == NULL) { + SPDK_ERRLOG("Couldn't find session with vid %d.\n", vid); + pthread_mutex_unlock(&g_vhost_mutex); + return -EINVAL; + } + + if (!vsession->started) { + /* already stopped, nothing to do */ + pthread_mutex_unlock(&g_vhost_mutex); + return -EALREADY; + } + + rc = _stop_session(vsession); + pthread_mutex_unlock(&g_vhost_mutex); + + return rc; +} + +int +vhost_start_device_cb(int vid) +{ + struct spdk_vhost_dev *vdev; + struct spdk_vhost_session *vsession; + int rc = -1; + uint16_t i; + bool packed_ring; + + pthread_mutex_lock(&g_vhost_mutex); + + vsession = vhost_session_find_by_vid(vid); + if (vsession == NULL) { + SPDK_ERRLOG("Couldn't find session with vid %d.\n", vid); + goto out; + } + + vdev = vsession->vdev; + if (vsession->started) { + /* already started, nothing to do */ + rc = 0; + goto out; + } + + if (vhost_get_negotiated_features(vid, &vsession->negotiated_features) != 0) { + SPDK_ERRLOG("vhost device %d: Failed to get negotiated driver features\n", vid); + goto out; + } + + packed_ring = ((vsession->negotiated_features & (1ULL << VIRTIO_F_RING_PACKED)) != 0); + + vsession->max_queues = 0; + memset(vsession->virtqueue, 0, sizeof(vsession->virtqueue)); + for (i = 0; i < SPDK_VHOST_MAX_VQUEUES; i++) { + struct spdk_vhost_virtqueue *q = &vsession->virtqueue[i]; + + q->vring_idx = -1; + if (rte_vhost_get_vhost_vring(vid, i, &q->vring)) { + continue; + } + q->vring_idx = i; + rte_vhost_get_vhost_ring_inflight(vid, i, &q->vring_inflight); + + /* vring.desc and vring.desc_packed are in a union struct + * so q->vring.desc can replace q->vring.desc_packed. + */ + if (q->vring.desc == NULL || q->vring.size == 0) { + continue; + } + + if (rte_vhost_get_vring_base(vsession->vid, i, &q->last_avail_idx, &q->last_used_idx)) { + q->vring.desc = NULL; + continue; + } + + if (packed_ring) { + /* Packed virtqueues support up to 2^15 entries each + * so left one bit can be used as wrap counter. + */ + q->packed.avail_phase = q->last_avail_idx >> 15; + q->last_avail_idx = q->last_avail_idx & 0x7FFF; + q->packed.used_phase = q->last_used_idx >> 15; + q->last_used_idx = q->last_used_idx & 0x7FFF; + + /* Disable I/O submission notifications, we'll be polling. */ + q->vring.device_event->flags = VRING_PACKED_EVENT_FLAG_DISABLE; + } else { + /* Disable I/O submission notifications, we'll be polling. */ + q->vring.used->flags = VRING_USED_F_NO_NOTIFY; + } + + q->packed.packed_ring = packed_ring; + vsession->max_queues = i + 1; + } + + if (vhost_get_mem_table(vid, &vsession->mem) != 0) { + SPDK_ERRLOG("vhost device %d: Failed to get guest memory table\n", vid); + goto out; + } + + /* + * Not sure right now but this look like some kind of QEMU bug and guest IO + * might be frozed without kicking all queues after live-migration. This look like + * the previous vhost instance failed to effectively deliver all interrupts before + * the GET_VRING_BASE message. This shouldn't harm guest since spurious interrupts + * should be ignored by guest virtio driver. + * + * Tested on QEMU 2.10.91 and 2.11.50. + */ + for (i = 0; i < vsession->max_queues; i++) { + struct spdk_vhost_virtqueue *q = &vsession->virtqueue[i]; + + /* vring.desc and vring.desc_packed are in a union struct + * so q->vring.desc can replace q->vring.desc_packed. + */ + if (q->vring.desc != NULL && q->vring.size > 0) { + rte_vhost_vring_call(vsession->vid, q->vring_idx); + } + } + + vhost_session_set_coalescing(vdev, vsession, NULL); + vhost_session_mem_register(vsession->mem); + vsession->initialized = true; + rc = vdev->backend->start_session(vsession); + if (rc != 0) { + vhost_session_mem_unregister(vsession->mem); + free(vsession->mem); + goto out; + } + +out: + pthread_mutex_unlock(&g_vhost_mutex); + return rc; +} + +#ifdef SPDK_CONFIG_VHOST_INTERNAL_LIB +int +vhost_get_config_cb(int vid, uint8_t *config, uint32_t len) +{ + struct spdk_vhost_session *vsession; + struct spdk_vhost_dev *vdev; + int rc = -1; + + pthread_mutex_lock(&g_vhost_mutex); + vsession = vhost_session_find_by_vid(vid); + if (vsession == NULL) { + SPDK_ERRLOG("Couldn't find session with vid %d.\n", vid); + goto out; + } + + vdev = vsession->vdev; + if (vdev->backend->vhost_get_config) { + rc = vdev->backend->vhost_get_config(vdev, config, len); + } + +out: + pthread_mutex_unlock(&g_vhost_mutex); + return rc; +} + +int +vhost_set_config_cb(int vid, uint8_t *config, uint32_t offset, uint32_t size, uint32_t flags) +{ + struct spdk_vhost_session *vsession; + struct spdk_vhost_dev *vdev; + int rc = -1; + + pthread_mutex_lock(&g_vhost_mutex); + vsession = vhost_session_find_by_vid(vid); + if (vsession == NULL) { + SPDK_ERRLOG("Couldn't find session with vid %d.\n", vid); + goto out; + } + + vdev = vsession->vdev; + if (vdev->backend->vhost_set_config) { + rc = vdev->backend->vhost_set_config(vdev, config, offset, size, flags); + } + +out: + pthread_mutex_unlock(&g_vhost_mutex); + return rc; +} +#endif + +int +spdk_vhost_set_socket_path(const char *basename) +{ + int ret; + + if (basename && strlen(basename) > 0) { + ret = snprintf(dev_dirname, sizeof(dev_dirname) - 2, "%s", basename); + if (ret <= 0) { + return -EINVAL; + } + if ((size_t)ret >= sizeof(dev_dirname) - 2) { + SPDK_ERRLOG("Char dev dir path length %d is too long\n", ret); + return -EINVAL; + } + + if (dev_dirname[ret - 1] != '/') { + dev_dirname[ret] = '/'; + dev_dirname[ret + 1] = '\0'; + } + } + + return 0; +} + +void +vhost_dump_info_json(struct spdk_vhost_dev *vdev, struct spdk_json_write_ctx *w) +{ + assert(vdev->backend->dump_info_json != NULL); + vdev->backend->dump_info_json(vdev, w); +} + +int +spdk_vhost_dev_remove(struct spdk_vhost_dev *vdev) +{ + if (vdev->pending_async_op_num) { + return -EBUSY; + } + + return vdev->backend->remove_device(vdev); +} + +int +vhost_new_connection_cb(int vid, const char *ifname) +{ + struct spdk_vhost_dev *vdev; + struct spdk_vhost_session *vsession; + + pthread_mutex_lock(&g_vhost_mutex); + + vdev = spdk_vhost_dev_find(ifname); + if (vdev == NULL) { + SPDK_ERRLOG("Couldn't find device with vid %d to create connection for.\n", vid); + pthread_mutex_unlock(&g_vhost_mutex); + return -1; + } + + /* We expect sessions inside vdev->vsessions to be sorted in ascending + * order in regard of vsession->id. For now we always set id = vsessions_cnt++ + * and append each session to the very end of the vsessions list. + * This is required for spdk_vhost_dev_foreach_session() to work. + */ + if (vdev->vsessions_num == UINT_MAX) { + assert(false); + return -EINVAL; + } + + if (posix_memalign((void **)&vsession, SPDK_CACHE_LINE_SIZE, sizeof(*vsession) + + vdev->backend->session_ctx_size)) { + SPDK_ERRLOG("vsession alloc failed\n"); + pthread_mutex_unlock(&g_vhost_mutex); + return -1; + } + memset(vsession, 0, sizeof(*vsession) + vdev->backend->session_ctx_size); + + vsession->vdev = vdev; + vsession->vid = vid; + vsession->id = vdev->vsessions_num++; + vsession->name = spdk_sprintf_alloc("%ss%u", vdev->name, vsession->vid); + if (vsession->name == NULL) { + SPDK_ERRLOG("vsession alloc failed\n"); + pthread_mutex_unlock(&g_vhost_mutex); + free(vsession); + return -1; + } + vsession->started = false; + vsession->initialized = false; + vsession->next_stats_check_time = 0; + vsession->stats_check_interval = SPDK_VHOST_STATS_CHECK_INTERVAL_MS * + spdk_get_ticks_hz() / 1000UL; + TAILQ_INSERT_TAIL(&vdev->vsessions, vsession, tailq); + + vhost_session_install_rte_compat_hooks(vsession); + pthread_mutex_unlock(&g_vhost_mutex); + return 0; +} + +int +vhost_destroy_connection_cb(int vid) +{ + struct spdk_vhost_session *vsession; + int rc = 0; + + pthread_mutex_lock(&g_vhost_mutex); + vsession = vhost_session_find_by_vid(vid); + if (vsession == NULL) { + SPDK_ERRLOG("Couldn't find session with vid %d.\n", vid); + pthread_mutex_unlock(&g_vhost_mutex); + return -EINVAL; + } + + if (vsession->started) { + rc = _stop_session(vsession); + } + + TAILQ_REMOVE(&vsession->vdev->vsessions, vsession, tailq); + free(vsession->name); + free(vsession); + pthread_mutex_unlock(&g_vhost_mutex); + + return rc; +} + +void +spdk_vhost_lock(void) +{ + pthread_mutex_lock(&g_vhost_mutex); +} + +int +spdk_vhost_trylock(void) +{ + return -pthread_mutex_trylock(&g_vhost_mutex); +} + +void +spdk_vhost_unlock(void) +{ + pthread_mutex_unlock(&g_vhost_mutex); +} + +void +spdk_vhost_init(spdk_vhost_init_cb init_cb) +{ + size_t len; + int ret; + + g_vhost_init_thread = spdk_get_thread(); + assert(g_vhost_init_thread != NULL); + + if (dev_dirname[0] == '\0') { + if (getcwd(dev_dirname, sizeof(dev_dirname) - 1) == NULL) { + SPDK_ERRLOG("getcwd failed (%d): %s\n", errno, spdk_strerror(errno)); + ret = -1; + goto out; + } + + len = strlen(dev_dirname); + if (dev_dirname[len - 1] != '/') { + dev_dirname[len] = '/'; + dev_dirname[len + 1] = '\0'; + } + } + + ret = sem_init(&g_dpdk_sem, 0, 0); + if (ret != 0) { + SPDK_ERRLOG("Failed to initialize semaphore for rte_vhost pthread.\n"); + ret = -1; + goto out; + } + + ret = vhost_scsi_controller_construct(); + if (ret != 0) { + SPDK_ERRLOG("Cannot construct vhost controllers\n"); + goto out; + } + + ret = vhost_blk_controller_construct(); + if (ret != 0) { + SPDK_ERRLOG("Cannot construct vhost block controllers\n"); + goto out; + } + +#ifdef SPDK_CONFIG_VHOST_INTERNAL_LIB + ret = vhost_nvme_controller_construct(); + if (ret != 0) { + SPDK_ERRLOG("Cannot construct vhost NVMe controllers\n"); + goto out; + } +#endif + + spdk_cpuset_zero(&g_vhost_core_mask); + + /* iterate threads instead of using SPDK_ENV_FOREACH_CORE to ensure that threads are really + * created. + */ + spdk_for_each_thread(vhost_setup_core_mask, init_cb, vhost_setup_core_mask_done); + return; +out: + init_cb(ret); +} + +static void +vhost_fini(void *arg1) +{ + struct spdk_vhost_dev *vdev, *tmp; + + spdk_vhost_lock(); + vdev = spdk_vhost_dev_next(NULL); + while (vdev != NULL) { + tmp = spdk_vhost_dev_next(vdev); + spdk_vhost_dev_remove(vdev); + /* don't care if it fails, there's nothing we can do for now */ + vdev = tmp; + } + spdk_vhost_unlock(); + + spdk_cpuset_zero(&g_vhost_core_mask); + + /* All devices are removed now. */ + sem_destroy(&g_dpdk_sem); + + g_fini_cpl_cb(); +} + +static void * +session_shutdown(void *arg) +{ + struct spdk_vhost_dev *vdev = NULL; + + TAILQ_FOREACH(vdev, &g_vhost_devices, tailq) { + vhost_driver_unregister(vdev->path); + vdev->registered = false; + } + + SPDK_INFOLOG(SPDK_LOG_VHOST, "Exiting\n"); + spdk_thread_send_msg(g_vhost_init_thread, vhost_fini, NULL); + return NULL; +} + +void +spdk_vhost_fini(spdk_vhost_fini_cb fini_cb) +{ + pthread_t tid; + int rc; + + assert(spdk_get_thread() == g_vhost_init_thread); + g_fini_cpl_cb = fini_cb; + + /* rte_vhost API for removing sockets is not asynchronous. Since it may call SPDK + * ops for stopping a device or removing a connection, we need to call it from + * a separate thread to avoid deadlock. + */ + rc = pthread_create(&tid, NULL, &session_shutdown, NULL); + if (rc < 0) { + SPDK_ERRLOG("Failed to start session shutdown thread (%d): %s\n", rc, spdk_strerror(rc)); + abort(); + } + pthread_detach(tid); +} + +void +spdk_vhost_config_json(struct spdk_json_write_ctx *w) +{ + struct spdk_vhost_dev *vdev; + uint32_t delay_base_us; + uint32_t iops_threshold; + + spdk_json_write_array_begin(w); + + spdk_vhost_lock(); + vdev = spdk_vhost_dev_next(NULL); + while (vdev != NULL) { + vdev->backend->write_config_json(vdev, w); + + spdk_vhost_get_coalescing(vdev, &delay_base_us, &iops_threshold); + if (delay_base_us) { + spdk_json_write_object_begin(w); + spdk_json_write_named_string(w, "method", "vhost_controller_set_coalescing"); + + spdk_json_write_named_object_begin(w, "params"); + spdk_json_write_named_string(w, "ctrlr", vdev->name); + spdk_json_write_named_uint32(w, "delay_base_us", delay_base_us); + spdk_json_write_named_uint32(w, "iops_threshold", iops_threshold); + spdk_json_write_object_end(w); + + spdk_json_write_object_end(w); + } + vdev = spdk_vhost_dev_next(vdev); + } + spdk_vhost_unlock(); + + spdk_json_write_array_end(w); +} + +SPDK_LOG_REGISTER_COMPONENT("vhost", SPDK_LOG_VHOST) +SPDK_LOG_REGISTER_COMPONENT("vhost_ring", SPDK_LOG_VHOST_RING) diff --git a/src/spdk/lib/vhost/vhost_blk.c b/src/spdk/lib/vhost/vhost_blk.c new file mode 100644 index 000000000..d387cb27d --- /dev/null +++ b/src/spdk/lib/vhost/vhost_blk.c @@ -0,0 +1,1354 @@ +/*- + * BSD LICENSE + * + * Copyright(c) Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <linux/virtio_blk.h> + +#include "spdk/env.h" +#include "spdk/bdev.h" +#include "spdk/bdev_module.h" +#include "spdk/conf.h" +#include "spdk/thread.h" +#include "spdk/likely.h" +#include "spdk/string.h" +#include "spdk/util.h" +#include "spdk/vhost.h" + +#include "vhost_internal.h" +#include <rte_version.h> + +/* Minimal set of features supported by every SPDK VHOST-BLK device */ +#define SPDK_VHOST_BLK_FEATURES_BASE (SPDK_VHOST_FEATURES | \ + (1ULL << VIRTIO_BLK_F_SIZE_MAX) | (1ULL << VIRTIO_BLK_F_SEG_MAX) | \ + (1ULL << VIRTIO_BLK_F_GEOMETRY) | (1ULL << VIRTIO_BLK_F_BLK_SIZE) | \ + (1ULL << VIRTIO_BLK_F_TOPOLOGY) | (1ULL << VIRTIO_BLK_F_BARRIER) | \ + (1ULL << VIRTIO_BLK_F_SCSI) | (1ULL << VIRTIO_BLK_F_CONFIG_WCE) | \ + (1ULL << VIRTIO_BLK_F_MQ)) + +/* Not supported features */ +#define SPDK_VHOST_BLK_DISABLED_FEATURES (SPDK_VHOST_DISABLED_FEATURES | \ + (1ULL << VIRTIO_BLK_F_GEOMETRY) | (1ULL << VIRTIO_BLK_F_CONFIG_WCE) | \ + (1ULL << VIRTIO_BLK_F_BARRIER) | (1ULL << VIRTIO_BLK_F_SCSI)) + +/* Vhost-blk support protocol features */ +#ifndef SPDK_CONFIG_VHOST_INTERNAL_LIB +#define SPDK_VHOST_BLK_PROTOCOL_FEATURES ((1ULL << VHOST_USER_PROTOCOL_F_CONFIG) | \ + (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) +#else +#define SPDK_VHOST_BLK_PROTOCOL_FEATURES (1ULL << VHOST_USER_PROTOCOL_F_CONFIG) +#endif + +struct spdk_vhost_blk_task { + struct spdk_bdev_io *bdev_io; + struct spdk_vhost_blk_session *bvsession; + struct spdk_vhost_virtqueue *vq; + + volatile uint8_t *status; + + uint16_t req_idx; + uint16_t num_descs; + uint16_t buffer_id; + + /* for io wait */ + struct spdk_bdev_io_wait_entry bdev_io_wait; + + /* If set, the task is currently used for I/O processing. */ + bool used; + + /** Number of bytes that were written. */ + uint32_t used_len; + uint16_t iovcnt; + struct iovec iovs[SPDK_VHOST_IOVS_MAX]; +}; + +struct spdk_vhost_blk_dev { + struct spdk_vhost_dev vdev; + struct spdk_bdev *bdev; + struct spdk_bdev_desc *bdev_desc; + /* dummy_io_channel is used to hold a bdev reference */ + struct spdk_io_channel *dummy_io_channel; + bool readonly; +}; + +struct spdk_vhost_blk_session { + /* The parent session must be the very first field in this struct */ + struct spdk_vhost_session vsession; + struct spdk_vhost_blk_dev *bvdev; + struct spdk_poller *requestq_poller; + struct spdk_io_channel *io_channel; + struct spdk_poller *stop_poller; +}; + +/* forward declaration */ +static const struct spdk_vhost_dev_backend vhost_blk_device_backend; + +static int +process_blk_request(struct spdk_vhost_blk_task *task, + struct spdk_vhost_blk_session *bvsession, + struct spdk_vhost_virtqueue *vq); + +static void +blk_task_finish(struct spdk_vhost_blk_task *task) +{ + assert(task->bvsession->vsession.task_cnt > 0); + task->bvsession->vsession.task_cnt--; + task->used = false; +} + +static void +blk_task_init(struct spdk_vhost_blk_task *task) +{ + task->used = true; + task->iovcnt = SPDK_COUNTOF(task->iovs); + task->status = NULL; + task->used_len = 0; +} + +static void +blk_task_enqueue(struct spdk_vhost_blk_task *task) +{ + if (task->vq->packed.packed_ring) { + vhost_vq_packed_ring_enqueue(&task->bvsession->vsession, task->vq, + task->num_descs, + task->buffer_id, task->used_len); + } else { + vhost_vq_used_ring_enqueue(&task->bvsession->vsession, task->vq, + task->req_idx, task->used_len); + } +} + +static void +invalid_blk_request(struct spdk_vhost_blk_task *task, uint8_t status) +{ + if (task->status) { + *task->status = status; + } + + blk_task_enqueue(task); + blk_task_finish(task); + SPDK_DEBUGLOG(SPDK_LOG_VHOST_BLK_DATA, "Invalid request (status=%" PRIu8")\n", status); +} + +/* + * Process task's descriptor chain and setup data related fields. + * Return + * total size of suplied buffers + * + * FIXME: Make this function return to rd_cnt and wr_cnt + */ +static int +blk_iovs_split_queue_setup(struct spdk_vhost_blk_session *bvsession, + struct spdk_vhost_virtqueue *vq, + uint16_t req_idx, struct iovec *iovs, uint16_t *iovs_cnt, uint32_t *length) +{ + struct spdk_vhost_session *vsession = &bvsession->vsession; + struct spdk_vhost_dev *vdev = vsession->vdev; + struct vring_desc *desc, *desc_table; + uint16_t out_cnt = 0, cnt = 0; + uint32_t desc_table_size, len = 0; + uint32_t desc_handled_cnt; + int rc; + + rc = vhost_vq_get_desc(vsession, vq, req_idx, &desc, &desc_table, &desc_table_size); + if (rc != 0) { + SPDK_ERRLOG("%s: invalid descriptor at index %"PRIu16".\n", vdev->name, req_idx); + return -1; + } + + desc_handled_cnt = 0; + while (1) { + /* + * Maximum cnt reached? + * Should not happen if request is well formatted, otherwise this is a BUG. + */ + if (spdk_unlikely(cnt == *iovs_cnt)) { + SPDK_DEBUGLOG(SPDK_LOG_VHOST_BLK, "%s: max IOVs in request reached (req_idx = %"PRIu16").\n", + vsession->name, req_idx); + return -1; + } + + if (spdk_unlikely(vhost_vring_desc_to_iov(vsession, iovs, &cnt, desc))) { + SPDK_DEBUGLOG(SPDK_LOG_VHOST_BLK, "%s: invalid descriptor %" PRIu16" (req_idx = %"PRIu16").\n", + vsession->name, req_idx, cnt); + return -1; + } + + len += desc->len; + + out_cnt += vhost_vring_desc_is_wr(desc); + + rc = vhost_vring_desc_get_next(&desc, desc_table, desc_table_size); + if (rc != 0) { + SPDK_ERRLOG("%s: descriptor chain at index %"PRIu16" terminated unexpectedly.\n", + vsession->name, req_idx); + return -1; + } else if (desc == NULL) { + break; + } + + desc_handled_cnt++; + if (spdk_unlikely(desc_handled_cnt > desc_table_size)) { + /* Break a cycle and report an error, if any. */ + SPDK_ERRLOG("%s: found a cycle in the descriptor chain: desc_table_size = %d, desc_handled_cnt = %d.\n", + vsession->name, desc_table_size, desc_handled_cnt); + return -1; + } + } + + /* + * There must be least two descriptors. + * First contain request so it must be readable. + * Last descriptor contain buffer for response so it must be writable. + */ + if (spdk_unlikely(out_cnt == 0 || cnt < 2)) { + return -1; + } + + *length = len; + *iovs_cnt = cnt; + return 0; +} + +static int +blk_iovs_packed_queue_setup(struct spdk_vhost_blk_session *bvsession, + struct spdk_vhost_virtqueue *vq, + uint16_t req_idx, struct iovec *iovs, uint16_t *iovs_cnt, uint32_t *length) +{ + struct spdk_vhost_session *vsession = &bvsession->vsession; + struct spdk_vhost_dev *vdev = vsession->vdev; + struct vring_packed_desc *desc = NULL, *desc_table; + uint16_t out_cnt = 0, cnt = 0; + uint32_t desc_table_size, len = 0; + int rc = 0; + + rc = vhost_vq_get_desc_packed(vsession, vq, req_idx, &desc, + &desc_table, &desc_table_size); + if (spdk_unlikely(rc != 0)) { + SPDK_ERRLOG("%s: Invalid descriptor at index %"PRIu16".\n", vdev->name, req_idx); + return rc; + } + + if (desc_table != NULL) { + req_idx = 0; + } + + while (1) { + /* + * Maximum cnt reached? + * Should not happen if request is well formatted, otherwise this is a BUG. + */ + if (spdk_unlikely(cnt == *iovs_cnt)) { + SPDK_ERRLOG("%s: max IOVs in request reached (req_idx = %"PRIu16").\n", + vsession->name, req_idx); + return -EINVAL; + } + + if (spdk_unlikely(vhost_vring_packed_desc_to_iov(vsession, iovs, &cnt, desc))) { + SPDK_ERRLOG("%s: invalid descriptor %" PRIu16" (req_idx = %"PRIu16").\n", + vsession->name, req_idx, cnt); + return -EINVAL; + } + + len += desc->len; + out_cnt += vhost_vring_packed_desc_is_wr(desc); + + /* desc is NULL means we reach the last desc of this request */ + vhost_vring_packed_desc_get_next(&desc, &req_idx, vq, desc_table, desc_table_size); + if (desc == NULL) { + break; + } + } + + /* + * There must be least two descriptors. + * First contain request so it must be readable. + * Last descriptor contain buffer for response so it must be writable. + */ + if (spdk_unlikely(out_cnt == 0 || cnt < 2)) { + return -EINVAL; + } + + *length = len; + *iovs_cnt = cnt; + + return 0; +} + +static void +blk_request_finish(bool success, struct spdk_vhost_blk_task *task) +{ + *task->status = success ? VIRTIO_BLK_S_OK : VIRTIO_BLK_S_IOERR; + + blk_task_enqueue(task); + + SPDK_DEBUGLOG(SPDK_LOG_VHOST_BLK, "Finished task (%p) req_idx=%d\n status: %s\n", task, + task->req_idx, success ? "OK" : "FAIL"); + blk_task_finish(task); +} + +static void +blk_request_complete_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) +{ + struct spdk_vhost_blk_task *task = cb_arg; + + spdk_bdev_free_io(bdev_io); + blk_request_finish(success, task); +} + +static void +blk_request_resubmit(void *arg) +{ + struct spdk_vhost_blk_task *task = (struct spdk_vhost_blk_task *)arg; + int rc = 0; + + blk_task_init(task); + + rc = process_blk_request(task, task->bvsession, task->vq); + if (rc == 0) { + SPDK_DEBUGLOG(SPDK_LOG_VHOST_BLK, "====== Task %p resubmitted ======\n", task); + } else { + SPDK_DEBUGLOG(SPDK_LOG_VHOST_BLK, "====== Task %p failed ======\n", task); + } +} + +static inline void +blk_request_queue_io(struct spdk_vhost_blk_task *task) +{ + int rc; + struct spdk_vhost_blk_session *bvsession = task->bvsession; + struct spdk_bdev *bdev = bvsession->bvdev->bdev; + + task->bdev_io_wait.bdev = bdev; + task->bdev_io_wait.cb_fn = blk_request_resubmit; + task->bdev_io_wait.cb_arg = task; + + rc = spdk_bdev_queue_io_wait(bdev, bvsession->io_channel, &task->bdev_io_wait); + if (rc != 0) { + SPDK_ERRLOG("%s: failed to queue I/O, rc=%d\n", bvsession->vsession.name, rc); + invalid_blk_request(task, VIRTIO_BLK_S_IOERR); + } +} + +static int +process_blk_request(struct spdk_vhost_blk_task *task, + struct spdk_vhost_blk_session *bvsession, + struct spdk_vhost_virtqueue *vq) +{ + struct spdk_vhost_blk_dev *bvdev = bvsession->bvdev; + const struct virtio_blk_outhdr *req; + struct virtio_blk_discard_write_zeroes *desc; + struct iovec *iov; + uint32_t type; + uint32_t payload_len; + uint64_t flush_bytes; + int rc; + + if (vq->packed.packed_ring) { + rc = blk_iovs_packed_queue_setup(bvsession, vq, task->req_idx, task->iovs, &task->iovcnt, + &payload_len); + } else { + rc = blk_iovs_split_queue_setup(bvsession, vq, task->req_idx, task->iovs, &task->iovcnt, + &payload_len); + } + + if (rc) { + SPDK_DEBUGLOG(SPDK_LOG_VHOST_BLK, "Invalid request (req_idx = %"PRIu16").\n", task->req_idx); + /* Only READ and WRITE are supported for now. */ + invalid_blk_request(task, VIRTIO_BLK_S_UNSUPP); + return -1; + } + + iov = &task->iovs[0]; + if (spdk_unlikely(iov->iov_len != sizeof(*req))) { + SPDK_DEBUGLOG(SPDK_LOG_VHOST_BLK, + "First descriptor size is %zu but expected %zu (req_idx = %"PRIu16").\n", + iov->iov_len, sizeof(*req), task->req_idx); + invalid_blk_request(task, VIRTIO_BLK_S_UNSUPP); + return -1; + } + + req = iov->iov_base; + + iov = &task->iovs[task->iovcnt - 1]; + if (spdk_unlikely(iov->iov_len != 1)) { + SPDK_DEBUGLOG(SPDK_LOG_VHOST_BLK, + "Last descriptor size is %zu but expected %d (req_idx = %"PRIu16").\n", + iov->iov_len, 1, task->req_idx); + invalid_blk_request(task, VIRTIO_BLK_S_UNSUPP); + return -1; + } + + task->status = iov->iov_base; + payload_len -= sizeof(*req) + sizeof(*task->status); + task->iovcnt -= 2; + + type = req->type; +#ifdef VIRTIO_BLK_T_BARRIER + /* Don't care about barier for now (as QEMU's virtio-blk do). */ + type &= ~VIRTIO_BLK_T_BARRIER; +#endif + + switch (type) { + case VIRTIO_BLK_T_IN: + case VIRTIO_BLK_T_OUT: + if (spdk_unlikely(payload_len == 0 || (payload_len & (512 - 1)) != 0)) { + SPDK_ERRLOG("%s - passed IO buffer is not multiple of 512b (req_idx = %"PRIu16").\n", + type ? "WRITE" : "READ", task->req_idx); + invalid_blk_request(task, VIRTIO_BLK_S_UNSUPP); + return -1; + } + + if (type == VIRTIO_BLK_T_IN) { + task->used_len = payload_len + sizeof(*task->status); + rc = spdk_bdev_readv(bvdev->bdev_desc, bvsession->io_channel, + &task->iovs[1], task->iovcnt, req->sector * 512, + payload_len, blk_request_complete_cb, task); + } else if (!bvdev->readonly) { + task->used_len = sizeof(*task->status); + rc = spdk_bdev_writev(bvdev->bdev_desc, bvsession->io_channel, + &task->iovs[1], task->iovcnt, req->sector * 512, + payload_len, blk_request_complete_cb, task); + } else { + SPDK_DEBUGLOG(SPDK_LOG_VHOST_BLK, "Device is in read-only mode!\n"); + rc = -1; + } + + if (rc) { + if (rc == -ENOMEM) { + SPDK_DEBUGLOG(SPDK_LOG_VHOST_BLK, "No memory, start to queue io.\n"); + blk_request_queue_io(task); + } else { + invalid_blk_request(task, VIRTIO_BLK_S_IOERR); + return -1; + } + } + break; + case VIRTIO_BLK_T_DISCARD: + desc = task->iovs[1].iov_base; + if (payload_len != sizeof(*desc)) { + SPDK_NOTICELOG("Invalid discard payload size: %u\n", payload_len); + invalid_blk_request(task, VIRTIO_BLK_S_IOERR); + return -1; + } + + rc = spdk_bdev_unmap(bvdev->bdev_desc, bvsession->io_channel, + desc->sector * 512, desc->num_sectors * 512, + blk_request_complete_cb, task); + if (rc) { + if (rc == -ENOMEM) { + SPDK_DEBUGLOG(SPDK_LOG_VHOST_BLK, "No memory, start to queue io.\n"); + blk_request_queue_io(task); + } else { + invalid_blk_request(task, VIRTIO_BLK_S_IOERR); + return -1; + } + } + break; + case VIRTIO_BLK_T_WRITE_ZEROES: + desc = task->iovs[1].iov_base; + if (payload_len != sizeof(*desc)) { + SPDK_NOTICELOG("Invalid write zeroes payload size: %u\n", payload_len); + invalid_blk_request(task, VIRTIO_BLK_S_IOERR); + return -1; + } + + /* Zeroed and Unmap the range, SPDK doen't support it. */ + if (desc->flags & VIRTIO_BLK_WRITE_ZEROES_FLAG_UNMAP) { + SPDK_NOTICELOG("Can't support Write Zeroes with Unmap flag\n"); + invalid_blk_request(task, VIRTIO_BLK_S_UNSUPP); + return -1; + } + + rc = spdk_bdev_write_zeroes(bvdev->bdev_desc, bvsession->io_channel, + desc->sector * 512, desc->num_sectors * 512, + blk_request_complete_cb, task); + if (rc) { + if (rc == -ENOMEM) { + SPDK_DEBUGLOG(SPDK_LOG_VHOST_BLK, "No memory, start to queue io.\n"); + blk_request_queue_io(task); + } else { + invalid_blk_request(task, VIRTIO_BLK_S_IOERR); + return -1; + } + } + break; + case VIRTIO_BLK_T_FLUSH: + flush_bytes = spdk_bdev_get_num_blocks(bvdev->bdev) * spdk_bdev_get_block_size(bvdev->bdev); + if (req->sector != 0) { + SPDK_NOTICELOG("sector must be zero for flush command\n"); + invalid_blk_request(task, VIRTIO_BLK_S_IOERR); + return -1; + } + rc = spdk_bdev_flush(bvdev->bdev_desc, bvsession->io_channel, + 0, flush_bytes, + blk_request_complete_cb, task); + if (rc) { + if (rc == -ENOMEM) { + SPDK_DEBUGLOG(SPDK_LOG_VHOST_BLK, "No memory, start to queue io.\n"); + blk_request_queue_io(task); + } else { + invalid_blk_request(task, VIRTIO_BLK_S_IOERR); + return -1; + } + } + break; + case VIRTIO_BLK_T_GET_ID: + if (!task->iovcnt || !payload_len) { + invalid_blk_request(task, VIRTIO_BLK_S_UNSUPP); + return -1; + } + task->used_len = spdk_min((size_t)VIRTIO_BLK_ID_BYTES, task->iovs[1].iov_len); + spdk_strcpy_pad(task->iovs[1].iov_base, spdk_bdev_get_product_name(bvdev->bdev), + task->used_len, ' '); + blk_request_finish(true, task); + break; + default: + SPDK_DEBUGLOG(SPDK_LOG_VHOST_BLK, "Not supported request type '%"PRIu32"'.\n", type); + invalid_blk_request(task, VIRTIO_BLK_S_UNSUPP); + return -1; + } + + return 0; +} + +static void +process_blk_task(struct spdk_vhost_virtqueue *vq, uint16_t req_idx) +{ + struct spdk_vhost_blk_task *task; + uint16_t task_idx = req_idx, num_descs; + + if (vq->packed.packed_ring) { + /* Packed ring used the buffer_id as the task_idx to get task struct. + * In kernel driver, it uses the vq->free_head to set the buffer_id so the value + * must be in the range of 0 ~ vring.size. The free_head value must be unique + * in the outstanding requests. + * We can't use the req_idx as the task_idx because the desc can be reused in + * the next phase even when it's not completed in the previous phase. For example, + * At phase 0, last_used_idx was 2 and desc0 was not completed.Then after moving + * phase 1, last_avail_idx is updated to 1. In this case, req_idx can not be used + * as task_idx because we will know task[0]->used is true at phase 1. + * The split queue is quite different, the desc would insert into the free list when + * device completes the request, the driver gets the desc from the free list which + * ensures the req_idx is unique in the outstanding requests. + */ + task_idx = vhost_vring_packed_desc_get_buffer_id(vq, req_idx, &num_descs); + } + + task = &((struct spdk_vhost_blk_task *)vq->tasks)[task_idx]; + if (spdk_unlikely(task->used)) { + SPDK_ERRLOG("%s: request with idx '%"PRIu16"' is already pending.\n", + task->bvsession->vsession.name, task_idx); + task->used_len = 0; + blk_task_enqueue(task); + return; + } + + if (vq->packed.packed_ring) { + task->req_idx = req_idx; + task->num_descs = num_descs; + task->buffer_id = task_idx; + } + + task->bvsession->vsession.task_cnt++; + + blk_task_init(task); + + if (process_blk_request(task, task->bvsession, vq) == 0) { + SPDK_DEBUGLOG(SPDK_LOG_VHOST_BLK, "====== Task %p req_idx %d submitted ======\n", task, + task_idx); + } else { + SPDK_ERRLOG("====== Task %p req_idx %d failed ======\n", task, task_idx); + } +} + +static void +submit_inflight_desc(struct spdk_vhost_blk_session *bvsession, + struct spdk_vhost_virtqueue *vq) +{ + struct spdk_vhost_session *vsession = &bvsession->vsession; + spdk_vhost_resubmit_info *resubmit = vq->vring_inflight.resubmit_inflight; + spdk_vhost_resubmit_desc *resubmit_list; + uint16_t req_idx; + + if (spdk_likely(resubmit == NULL || resubmit->resubmit_list == NULL)) { + return; + } + + resubmit_list = resubmit->resubmit_list; + while (resubmit->resubmit_num-- > 0) { + req_idx = resubmit_list[resubmit->resubmit_num].index; + SPDK_DEBUGLOG(SPDK_LOG_VHOST_BLK, "====== Start processing request idx %"PRIu16"======\n", + req_idx); + + if (spdk_unlikely(req_idx >= vq->vring.size)) { + SPDK_ERRLOG("%s: request idx '%"PRIu16"' exceeds virtqueue size (%"PRIu16").\n", + vsession->name, req_idx, vq->vring.size); + vhost_vq_used_ring_enqueue(vsession, vq, req_idx, 0); + continue; + } + + process_blk_task(vq, req_idx); + } + + free(resubmit_list); + resubmit->resubmit_list = NULL; +} + +static void +process_vq(struct spdk_vhost_blk_session *bvsession, struct spdk_vhost_virtqueue *vq) +{ + struct spdk_vhost_session *vsession = &bvsession->vsession; + uint16_t reqs[SPDK_VHOST_VQ_MAX_SUBMISSIONS]; + uint16_t reqs_cnt, i; + + submit_inflight_desc(bvsession, vq); + + reqs_cnt = vhost_vq_avail_ring_get(vq, reqs, SPDK_COUNTOF(reqs)); + if (!reqs_cnt) { + return; + } + + for (i = 0; i < reqs_cnt; i++) { + SPDK_DEBUGLOG(SPDK_LOG_VHOST_BLK, "====== Starting processing request idx %"PRIu16"======\n", + reqs[i]); + + if (spdk_unlikely(reqs[i] >= vq->vring.size)) { + SPDK_ERRLOG("%s: request idx '%"PRIu16"' exceeds virtqueue size (%"PRIu16").\n", + vsession->name, reqs[i], vq->vring.size); + vhost_vq_used_ring_enqueue(vsession, vq, reqs[i], 0); + continue; + } + + rte_vhost_set_inflight_desc_split(vsession->vid, vq->vring_idx, reqs[i]); + + process_blk_task(vq, reqs[i]); + } +} + +static void +process_packed_vq(struct spdk_vhost_blk_session *bvsession, struct spdk_vhost_virtqueue *vq) +{ + uint16_t i = 0; + + while (i++ < SPDK_VHOST_VQ_MAX_SUBMISSIONS && + vhost_vq_packed_ring_is_avail(vq)) { + SPDK_DEBUGLOG(SPDK_LOG_VHOST_BLK, "====== Starting processing request idx %"PRIu16"======\n", + vq->last_avail_idx); + + process_blk_task(vq, vq->last_avail_idx); + } +} + +static int +vdev_worker(void *arg) +{ + struct spdk_vhost_blk_session *bvsession = arg; + struct spdk_vhost_session *vsession = &bvsession->vsession; + + uint16_t q_idx; + bool packed_ring; + + /* In a session, every vq supports the same format */ + packed_ring = vsession->virtqueue[0].packed.packed_ring; + for (q_idx = 0; q_idx < vsession->max_queues; q_idx++) { + if (packed_ring) { + process_packed_vq(bvsession, &vsession->virtqueue[q_idx]); + } else { + process_vq(bvsession, &vsession->virtqueue[q_idx]); + } + } + + vhost_session_used_signal(vsession); + + return SPDK_POLLER_BUSY; +} + +static void +no_bdev_process_vq(struct spdk_vhost_blk_session *bvsession, struct spdk_vhost_virtqueue *vq) +{ + struct spdk_vhost_session *vsession = &bvsession->vsession; + struct iovec iovs[SPDK_VHOST_IOVS_MAX]; + uint32_t length; + uint16_t iovcnt, req_idx; + + if (vhost_vq_avail_ring_get(vq, &req_idx, 1) != 1) { + return; + } + + iovcnt = SPDK_COUNTOF(iovs); + if (blk_iovs_split_queue_setup(bvsession, vq, req_idx, iovs, &iovcnt, &length) == 0) { + *(volatile uint8_t *)iovs[iovcnt - 1].iov_base = VIRTIO_BLK_S_IOERR; + SPDK_DEBUGLOG(SPDK_LOG_VHOST_BLK_DATA, "Aborting request %" PRIu16"\n", req_idx); + } + + vhost_vq_used_ring_enqueue(vsession, vq, req_idx, 0); +} + +static void +no_bdev_process_packed_vq(struct spdk_vhost_blk_session *bvsession, struct spdk_vhost_virtqueue *vq) +{ + struct spdk_vhost_session *vsession = &bvsession->vsession; + struct spdk_vhost_blk_task *task; + uint32_t length; + uint16_t req_idx = vq->last_avail_idx; + uint16_t task_idx, num_descs; + + if (!vhost_vq_packed_ring_is_avail(vq)) { + return; + } + + task_idx = vhost_vring_packed_desc_get_buffer_id(vq, req_idx, &num_descs); + task = &((struct spdk_vhost_blk_task *)vq->tasks)[task_idx]; + if (spdk_unlikely(task->used)) { + SPDK_ERRLOG("%s: request with idx '%"PRIu16"' is already pending.\n", + vsession->name, req_idx); + vhost_vq_packed_ring_enqueue(vsession, vq, num_descs, + task->buffer_id, task->used_len); + return; + } + + task->req_idx = req_idx; + task->num_descs = num_descs; + task->buffer_id = task_idx; + blk_task_init(task); + + if (blk_iovs_packed_queue_setup(bvsession, vq, task->req_idx, task->iovs, &task->iovcnt, + &length)) { + *(volatile uint8_t *)(task->iovs[task->iovcnt - 1].iov_base) = VIRTIO_BLK_S_IOERR; + SPDK_DEBUGLOG(SPDK_LOG_VHOST_BLK_DATA, "Aborting request %" PRIu16"\n", req_idx); + } + + task->used = false; + vhost_vq_packed_ring_enqueue(vsession, vq, num_descs, + task->buffer_id, task->used_len); +} + +static int +no_bdev_vdev_worker(void *arg) +{ + struct spdk_vhost_blk_session *bvsession = arg; + struct spdk_vhost_session *vsession = &bvsession->vsession; + uint16_t q_idx; + bool packed_ring; + + /* In a session, every vq supports the same format */ + packed_ring = vsession->virtqueue[0].packed.packed_ring; + for (q_idx = 0; q_idx < vsession->max_queues; q_idx++) { + if (packed_ring) { + no_bdev_process_packed_vq(bvsession, &vsession->virtqueue[q_idx]); + } else { + no_bdev_process_vq(bvsession, &vsession->virtqueue[q_idx]); + } + } + + vhost_session_used_signal(vsession); + + if (vsession->task_cnt == 0 && bvsession->io_channel) { + spdk_put_io_channel(bvsession->io_channel); + bvsession->io_channel = NULL; + } + + return SPDK_POLLER_BUSY; +} + +static struct spdk_vhost_blk_session * +to_blk_session(struct spdk_vhost_session *vsession) +{ + assert(vsession->vdev->backend == &vhost_blk_device_backend); + return (struct spdk_vhost_blk_session *)vsession; +} + +static struct spdk_vhost_blk_dev * +to_blk_dev(struct spdk_vhost_dev *vdev) +{ + if (vdev == NULL) { + return NULL; + } + + if (vdev->backend != &vhost_blk_device_backend) { + SPDK_ERRLOG("%s: not a vhost-blk device\n", vdev->name); + return NULL; + } + + return SPDK_CONTAINEROF(vdev, struct spdk_vhost_blk_dev, vdev); +} + +static int +vhost_session_bdev_resize_cb(struct spdk_vhost_dev *vdev, + struct spdk_vhost_session *vsession, + void *ctx) +{ +#if RTE_VERSION >= RTE_VERSION_NUM(20, 02, 0, 0) + SPDK_NOTICELOG("bdev send slave msg to vid(%d)\n", vsession->vid); + rte_vhost_slave_config_change(vsession->vid, false); +#else + SPDK_NOTICELOG("bdev does not support resize until DPDK submodule version >= 20.02\n"); +#endif + + return 0; +} + +static void +blk_resize_cb(void *resize_ctx) +{ + struct spdk_vhost_blk_dev *bvdev = resize_ctx; + + spdk_vhost_lock(); + vhost_dev_foreach_session(&bvdev->vdev, vhost_session_bdev_resize_cb, + NULL, NULL); + spdk_vhost_unlock(); +} + +static void +vhost_dev_bdev_remove_cpl_cb(struct spdk_vhost_dev *vdev, void *ctx) +{ + + /* All sessions have been notified, time to close the bdev */ + struct spdk_vhost_blk_dev *bvdev = to_blk_dev(vdev); + + assert(bvdev != NULL); + spdk_put_io_channel(bvdev->dummy_io_channel); + spdk_bdev_close(bvdev->bdev_desc); + bvdev->bdev_desc = NULL; + bvdev->bdev = NULL; +} + +static int +vhost_session_bdev_remove_cb(struct spdk_vhost_dev *vdev, + struct spdk_vhost_session *vsession, + void *ctx) +{ + struct spdk_vhost_blk_session *bvsession; + + bvsession = (struct spdk_vhost_blk_session *)vsession; + if (bvsession->requestq_poller) { + spdk_poller_unregister(&bvsession->requestq_poller); + bvsession->requestq_poller = SPDK_POLLER_REGISTER(no_bdev_vdev_worker, bvsession, 0); + } + + return 0; +} + +static void +bdev_remove_cb(void *remove_ctx) +{ + struct spdk_vhost_blk_dev *bvdev = remove_ctx; + + SPDK_WARNLOG("%s: hot-removing bdev - all further requests will fail.\n", + bvdev->vdev.name); + + spdk_vhost_lock(); + vhost_dev_foreach_session(&bvdev->vdev, vhost_session_bdev_remove_cb, + vhost_dev_bdev_remove_cpl_cb, NULL); + spdk_vhost_unlock(); +} + +static void +bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, + void *event_ctx) +{ + SPDK_DEBUGLOG(SPDK_LOG_VHOST_BLK, "Bdev event: type %d, name %s\n", + type, + bdev->name); + + switch (type) { + case SPDK_BDEV_EVENT_REMOVE: + SPDK_NOTICELOG("bdev name (%s) received event(SPDK_BDEV_EVENT_REMOVE)\n", bdev->name); + bdev_remove_cb(event_ctx); + break; + case SPDK_BDEV_EVENT_RESIZE: + SPDK_NOTICELOG("bdev name (%s) received event(SPDK_BDEV_EVENT_RESIZE)\n", bdev->name); + blk_resize_cb(event_ctx); + break; + default: + SPDK_NOTICELOG("Unsupported bdev event: type %d\n", type); + break; + } +} + +static void +free_task_pool(struct spdk_vhost_blk_session *bvsession) +{ + struct spdk_vhost_session *vsession = &bvsession->vsession; + struct spdk_vhost_virtqueue *vq; + uint16_t i; + + for (i = 0; i < vsession->max_queues; i++) { + vq = &vsession->virtqueue[i]; + if (vq->tasks == NULL) { + continue; + } + + spdk_free(vq->tasks); + vq->tasks = NULL; + } +} + +static int +alloc_task_pool(struct spdk_vhost_blk_session *bvsession) +{ + struct spdk_vhost_session *vsession = &bvsession->vsession; + struct spdk_vhost_virtqueue *vq; + struct spdk_vhost_blk_task *task; + uint32_t task_cnt; + uint16_t i; + uint32_t j; + + for (i = 0; i < vsession->max_queues; i++) { + vq = &vsession->virtqueue[i]; + if (vq->vring.desc == NULL) { + continue; + } + + task_cnt = vq->vring.size; + if (task_cnt > SPDK_VHOST_MAX_VQ_SIZE) { + /* sanity check */ + SPDK_ERRLOG("%s: virtuque %"PRIu16" is too big. (size = %"PRIu32", max = %"PRIu32")\n", + vsession->name, i, task_cnt, SPDK_VHOST_MAX_VQ_SIZE); + free_task_pool(bvsession); + return -1; + } + vq->tasks = spdk_zmalloc(sizeof(struct spdk_vhost_blk_task) * task_cnt, + SPDK_CACHE_LINE_SIZE, NULL, + SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA); + if (vq->tasks == NULL) { + SPDK_ERRLOG("%s: failed to allocate %"PRIu32" tasks for virtqueue %"PRIu16"\n", + vsession->name, task_cnt, i); + free_task_pool(bvsession); + return -1; + } + + for (j = 0; j < task_cnt; j++) { + task = &((struct spdk_vhost_blk_task *)vq->tasks)[j]; + task->bvsession = bvsession; + task->req_idx = j; + task->vq = vq; + } + } + + return 0; +} + +static int +vhost_blk_start_cb(struct spdk_vhost_dev *vdev, + struct spdk_vhost_session *vsession, void *unused) +{ + struct spdk_vhost_blk_session *bvsession = to_blk_session(vsession); + struct spdk_vhost_blk_dev *bvdev; + int i, rc = 0; + + bvdev = to_blk_dev(vdev); + assert(bvdev != NULL); + bvsession->bvdev = bvdev; + + /* validate all I/O queues are in a contiguous index range */ + for (i = 0; i < vsession->max_queues; i++) { + /* vring.desc and vring.desc_packed are in a union struct + * so q->vring.desc can replace q->vring.desc_packed. + */ + if (vsession->virtqueue[i].vring.desc == NULL) { + SPDK_ERRLOG("%s: queue %"PRIu32" is empty\n", vsession->name, i); + rc = -1; + goto out; + } + } + + rc = alloc_task_pool(bvsession); + if (rc != 0) { + SPDK_ERRLOG("%s: failed to alloc task pool.\n", vsession->name); + goto out; + } + + if (bvdev->bdev) { + bvsession->io_channel = spdk_bdev_get_io_channel(bvdev->bdev_desc); + if (!bvsession->io_channel) { + free_task_pool(bvsession); + SPDK_ERRLOG("%s: I/O channel allocation failed\n", vsession->name); + rc = -1; + goto out; + } + } + + bvsession->requestq_poller = SPDK_POLLER_REGISTER(bvdev->bdev ? vdev_worker : no_bdev_vdev_worker, + bvsession, 0); + SPDK_INFOLOG(SPDK_LOG_VHOST, "%s: started poller on lcore %d\n", + vsession->name, spdk_env_get_current_core()); +out: + vhost_session_start_done(vsession, rc); + return rc; +} + +static int +vhost_blk_start(struct spdk_vhost_session *vsession) +{ + return vhost_session_send_event(vsession, vhost_blk_start_cb, + 3, "start session"); +} + +static int +destroy_session_poller_cb(void *arg) +{ + struct spdk_vhost_blk_session *bvsession = arg; + struct spdk_vhost_session *vsession = &bvsession->vsession; + int i; + + if (vsession->task_cnt > 0) { + return SPDK_POLLER_BUSY; + } + + if (spdk_vhost_trylock() != 0) { + return SPDK_POLLER_BUSY; + } + + for (i = 0; i < vsession->max_queues; i++) { + vsession->virtqueue[i].next_event_time = 0; + vhost_vq_used_signal(vsession, &vsession->virtqueue[i]); + } + + SPDK_INFOLOG(SPDK_LOG_VHOST, "%s: stopping poller on lcore %d\n", + vsession->name, spdk_env_get_current_core()); + + if (bvsession->io_channel) { + spdk_put_io_channel(bvsession->io_channel); + bvsession->io_channel = NULL; + } + + free_task_pool(bvsession); + spdk_poller_unregister(&bvsession->stop_poller); + vhost_session_stop_done(vsession, 0); + + spdk_vhost_unlock(); + return SPDK_POLLER_BUSY; +} + +static int +vhost_blk_stop_cb(struct spdk_vhost_dev *vdev, + struct spdk_vhost_session *vsession, void *unused) +{ + struct spdk_vhost_blk_session *bvsession = to_blk_session(vsession); + + spdk_poller_unregister(&bvsession->requestq_poller); + bvsession->stop_poller = SPDK_POLLER_REGISTER(destroy_session_poller_cb, + bvsession, 1000); + return 0; +} + +static int +vhost_blk_stop(struct spdk_vhost_session *vsession) +{ + return vhost_session_send_event(vsession, vhost_blk_stop_cb, + 3, "stop session"); +} + +static void +vhost_blk_dump_info_json(struct spdk_vhost_dev *vdev, struct spdk_json_write_ctx *w) +{ + struct spdk_vhost_blk_dev *bvdev; + + bvdev = to_blk_dev(vdev); + assert(bvdev != NULL); + + spdk_json_write_named_object_begin(w, "block"); + + spdk_json_write_named_bool(w, "readonly", bvdev->readonly); + + spdk_json_write_name(w, "bdev"); + if (bvdev->bdev) { + spdk_json_write_string(w, spdk_bdev_get_name(bvdev->bdev)); + } else { + spdk_json_write_null(w); + } + + spdk_json_write_object_end(w); +} + +static void +vhost_blk_write_config_json(struct spdk_vhost_dev *vdev, struct spdk_json_write_ctx *w) +{ + struct spdk_vhost_blk_dev *bvdev; + + bvdev = to_blk_dev(vdev); + assert(bvdev != NULL); + + if (!bvdev->bdev) { + return; + } + + spdk_json_write_object_begin(w); + spdk_json_write_named_string(w, "method", "vhost_create_blk_controller"); + + spdk_json_write_named_object_begin(w, "params"); + spdk_json_write_named_string(w, "ctrlr", vdev->name); + spdk_json_write_named_string(w, "dev_name", spdk_bdev_get_name(bvdev->bdev)); + spdk_json_write_named_string(w, "cpumask", + spdk_cpuset_fmt(spdk_thread_get_cpumask(vdev->thread))); + spdk_json_write_named_bool(w, "readonly", bvdev->readonly); + spdk_json_write_object_end(w); + + spdk_json_write_object_end(w); +} + +static int vhost_blk_destroy(struct spdk_vhost_dev *dev); + +static int +vhost_blk_get_config(struct spdk_vhost_dev *vdev, uint8_t *config, + uint32_t len) +{ + struct virtio_blk_config blkcfg; + struct spdk_vhost_blk_dev *bvdev; + struct spdk_bdev *bdev; + uint32_t blk_size; + uint64_t blkcnt; + + memset(&blkcfg, 0, sizeof(blkcfg)); + bvdev = to_blk_dev(vdev); + assert(bvdev != NULL); + bdev = bvdev->bdev; + if (bdev == NULL) { + /* We can't just return -1 here as this GET_CONFIG message might + * be caused by a QEMU VM reboot. Returning -1 will indicate an + * error to QEMU, who might then decide to terminate itself. + * We don't want that. A simple reboot shouldn't break the system. + * + * Presenting a block device with block size 0 and block count 0 + * doesn't cause any problems on QEMU side and the virtio-pci + * device is even still available inside the VM, but there will + * be no block device created for it - the kernel drivers will + * silently reject it. + */ + blk_size = 0; + blkcnt = 0; + } else { + blk_size = spdk_bdev_get_block_size(bdev); + blkcnt = spdk_bdev_get_num_blocks(bdev); + if (spdk_bdev_get_buf_align(bdev) > 1) { + blkcfg.size_max = SPDK_BDEV_LARGE_BUF_MAX_SIZE; + blkcfg.seg_max = spdk_min(SPDK_VHOST_IOVS_MAX - 2 - 1, BDEV_IO_NUM_CHILD_IOV - 2 - 1); + } else { + blkcfg.size_max = 131072; + /* -2 for REQ and RESP and -1 for region boundary splitting */ + blkcfg.seg_max = SPDK_VHOST_IOVS_MAX - 2 - 1; + } + } + + blkcfg.blk_size = blk_size; + /* minimum I/O size in blocks */ + blkcfg.min_io_size = 1; + /* expressed in 512 Bytes sectors */ + blkcfg.capacity = (blkcnt * blk_size) / 512; + /* QEMU can overwrite this value when started */ + blkcfg.num_queues = SPDK_VHOST_MAX_VQUEUES; + + if (bdev && spdk_bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_UNMAP)) { + /* 16MiB, expressed in 512 Bytes */ + blkcfg.max_discard_sectors = 32768; + blkcfg.max_discard_seg = 1; + blkcfg.discard_sector_alignment = blk_size / 512; + } + if (bdev && spdk_bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_WRITE_ZEROES)) { + blkcfg.max_write_zeroes_sectors = 32768; + blkcfg.max_write_zeroes_seg = 1; + } + + memcpy(config, &blkcfg, spdk_min(len, sizeof(blkcfg))); + + return 0; +} + +static const struct spdk_vhost_dev_backend vhost_blk_device_backend = { + .session_ctx_size = sizeof(struct spdk_vhost_blk_session) - sizeof(struct spdk_vhost_session), + .start_session = vhost_blk_start, + .stop_session = vhost_blk_stop, + .vhost_get_config = vhost_blk_get_config, + .dump_info_json = vhost_blk_dump_info_json, + .write_config_json = vhost_blk_write_config_json, + .remove_device = vhost_blk_destroy, +}; + +int +vhost_blk_controller_construct(void) +{ + struct spdk_conf_section *sp; + unsigned ctrlr_num; + char *bdev_name; + char *cpumask; + char *name; + bool readonly; + bool packed_ring; + + for (sp = spdk_conf_first_section(NULL); sp != NULL; sp = spdk_conf_next_section(sp)) { + if (!spdk_conf_section_match_prefix(sp, "VhostBlk")) { + continue; + } + + if (sscanf(spdk_conf_section_get_name(sp), "VhostBlk%u", &ctrlr_num) != 1) { + SPDK_ERRLOG("Section '%s' has non-numeric suffix.\n", + spdk_conf_section_get_name(sp)); + return -1; + } + + name = spdk_conf_section_get_val(sp, "Name"); + if (name == NULL) { + SPDK_ERRLOG("VhostBlk%u: missing Name\n", ctrlr_num); + return -1; + } + + cpumask = spdk_conf_section_get_val(sp, "Cpumask"); + readonly = spdk_conf_section_get_boolval(sp, "ReadOnly", false); + packed_ring = spdk_conf_section_get_boolval(sp, "PackedRing", false); + + bdev_name = spdk_conf_section_get_val(sp, "Dev"); + if (bdev_name == NULL) { + continue; + } + + if (spdk_vhost_blk_construct(name, cpumask, bdev_name, + readonly, packed_ring) < 0) { + return -1; + } + } + + return 0; +} + +int +spdk_vhost_blk_construct(const char *name, const char *cpumask, const char *dev_name, + bool readonly, bool packed_ring) +{ + struct spdk_vhost_blk_dev *bvdev = NULL; + struct spdk_vhost_dev *vdev; + struct spdk_bdev *bdev; + int ret = 0; + + spdk_vhost_lock(); + bdev = spdk_bdev_get_by_name(dev_name); + if (bdev == NULL) { + SPDK_ERRLOG("%s: bdev '%s' not found\n", + name, dev_name); + ret = -ENODEV; + goto out; + } + + bvdev = calloc(1, sizeof(*bvdev)); + if (bvdev == NULL) { + ret = -ENOMEM; + goto out; + } + + vdev = &bvdev->vdev; + vdev->virtio_features = SPDK_VHOST_BLK_FEATURES_BASE; + vdev->disabled_features = SPDK_VHOST_BLK_DISABLED_FEATURES; + vdev->protocol_features = SPDK_VHOST_BLK_PROTOCOL_FEATURES; + + vdev->virtio_features |= (uint64_t)packed_ring << VIRTIO_F_RING_PACKED; + + if (spdk_bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_UNMAP)) { + vdev->virtio_features |= (1ULL << VIRTIO_BLK_F_DISCARD); + } + if (spdk_bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_WRITE_ZEROES)) { + vdev->virtio_features |= (1ULL << VIRTIO_BLK_F_WRITE_ZEROES); + } + if (readonly) { + vdev->virtio_features |= (1ULL << VIRTIO_BLK_F_RO); + } + if (spdk_bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_FLUSH)) { + vdev->virtio_features |= (1ULL << VIRTIO_BLK_F_FLUSH); + } + + ret = spdk_bdev_open_ext(dev_name, true, bdev_event_cb, bvdev, &bvdev->bdev_desc); + if (ret != 0) { + SPDK_ERRLOG("%s: could not open bdev '%s', error=%d\n", + name, dev_name, ret); + goto out; + } + + /* + * When starting qemu with vhost-user-blk multiqueue, the vhost device will + * be started/stopped many times, related to the queues num, as the + * vhost-user backend doesn't know the exact number of queues used for this + * device. The target have to stop and start the device once got a valid + * IO queue. + * When stoping and starting the vhost device, the backend bdev io device + * will be deleted and created repeatedly. + * Hold a bdev reference so that in the struct spdk_vhost_blk_dev, so that + * the io device will not be deleted. + */ + bvdev->dummy_io_channel = spdk_bdev_get_io_channel(bvdev->bdev_desc); + + bvdev->bdev = bdev; + bvdev->readonly = readonly; + ret = vhost_dev_register(vdev, name, cpumask, &vhost_blk_device_backend); + if (ret != 0) { + spdk_put_io_channel(bvdev->dummy_io_channel); + spdk_bdev_close(bvdev->bdev_desc); + goto out; + } + + SPDK_INFOLOG(SPDK_LOG_VHOST, "%s: using bdev '%s'\n", name, dev_name); +out: + if (ret != 0 && bvdev) { + free(bvdev); + } + spdk_vhost_unlock(); + return ret; +} + +static int +vhost_blk_destroy(struct spdk_vhost_dev *vdev) +{ + struct spdk_vhost_blk_dev *bvdev = to_blk_dev(vdev); + int rc; + + assert(bvdev != NULL); + + rc = vhost_dev_unregister(&bvdev->vdev); + if (rc != 0) { + return rc; + } + + /* if the bdev is removed, don't need call spdk_put_io_channel. */ + if (bvdev->bdev) { + spdk_put_io_channel(bvdev->dummy_io_channel); + } + + if (bvdev->bdev_desc) { + spdk_bdev_close(bvdev->bdev_desc); + bvdev->bdev_desc = NULL; + } + bvdev->bdev = NULL; + + free(bvdev); + return 0; +} + +SPDK_LOG_REGISTER_COMPONENT("vhost_blk", SPDK_LOG_VHOST_BLK) +SPDK_LOG_REGISTER_COMPONENT("vhost_blk_data", SPDK_LOG_VHOST_BLK_DATA) diff --git a/src/spdk/lib/vhost/vhost_internal.h b/src/spdk/lib/vhost/vhost_internal.h new file mode 100644 index 000000000..3aa89768a --- /dev/null +++ b/src/spdk/lib/vhost/vhost_internal.h @@ -0,0 +1,496 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef SPDK_VHOST_INTERNAL_H +#define SPDK_VHOST_INTERNAL_H +#include <linux/virtio_config.h> + +#include "spdk/stdinc.h" + +#include <rte_vhost.h> + +#include "spdk_internal/vhost_user.h" +#include "spdk_internal/log.h" +#include "spdk/util.h" +#include "spdk/rpc.h" +#include "spdk/config.h" + +#define SPDK_VHOST_MAX_VQUEUES 256 +#define SPDK_VHOST_MAX_VQ_SIZE 1024 + +#define SPDK_VHOST_SCSI_CTRLR_MAX_DEVS 8 + +#define SPDK_VHOST_IOVS_MAX 129 + +#define SPDK_VHOST_VQ_MAX_SUBMISSIONS 32 + +/* + * Rate at which stats are checked for interrupt coalescing. + */ +#define SPDK_VHOST_STATS_CHECK_INTERVAL_MS 10 +/* + * Default threshold at which interrupts start to be coalesced. + */ +#define SPDK_VHOST_VQ_IOPS_COALESCING_THRESHOLD 60000 + +/* + * Currently coalescing is not used by default. + * Setting this to value > 0 here or by RPC will enable coalescing. + */ +#define SPDK_VHOST_COALESCING_DELAY_BASE_US 0 + +#define SPDK_VHOST_FEATURES ((1ULL << VHOST_F_LOG_ALL) | \ + (1ULL << VHOST_USER_F_PROTOCOL_FEATURES) | \ + (1ULL << VIRTIO_F_VERSION_1) | \ + (1ULL << VIRTIO_F_NOTIFY_ON_EMPTY) | \ + (1ULL << VIRTIO_RING_F_EVENT_IDX) | \ + (1ULL << VIRTIO_RING_F_INDIRECT_DESC) | \ + (1ULL << VIRTIO_F_RING_PACKED)) + +#define SPDK_VHOST_DISABLED_FEATURES ((1ULL << VIRTIO_RING_F_EVENT_IDX) | \ + (1ULL << VIRTIO_F_NOTIFY_ON_EMPTY)) + +#define VRING_DESC_F_AVAIL (1ULL << VRING_PACKED_DESC_F_AVAIL) +#define VRING_DESC_F_USED (1ULL << VRING_PACKED_DESC_F_USED) +#define VRING_DESC_F_AVAIL_USED (VRING_DESC_F_AVAIL | VRING_DESC_F_USED) + +typedef struct rte_vhost_resubmit_desc spdk_vhost_resubmit_desc; +typedef struct rte_vhost_resubmit_info spdk_vhost_resubmit_info; + +struct spdk_vhost_virtqueue { + struct rte_vhost_vring vring; + struct rte_vhost_ring_inflight vring_inflight; + uint16_t last_avail_idx; + uint16_t last_used_idx; + + struct { + /* To mark a descriptor as available in packed ring + * Equal to avail_wrap_counter in spec. + */ + uint8_t avail_phase : 1; + /* To mark a descriptor as used in packed ring + * Equal to used_wrap_counter in spec. + */ + uint8_t used_phase : 1; + uint8_t padding : 5; + bool packed_ring : 1; + } packed; + + void *tasks; + + /* Request count from last stats check */ + uint32_t req_cnt; + + /* Request count from last event */ + uint16_t used_req_cnt; + + /* How long interrupt is delayed */ + uint32_t irq_delay_time; + + /* Next time when we need to send event */ + uint64_t next_event_time; + + /* Associated vhost_virtqueue in the virtio device's virtqueue list */ + uint32_t vring_idx; +} __attribute((aligned(SPDK_CACHE_LINE_SIZE))); + +struct spdk_vhost_session { + struct spdk_vhost_dev *vdev; + + /* rte_vhost connection ID. */ + int vid; + + /* Unique session ID. */ + uint64_t id; + /* Unique session name. */ + char *name; + + bool initialized; + bool started; + bool needs_restart; + bool forced_polling; + + struct rte_vhost_memory *mem; + + int task_cnt; + + uint16_t max_queues; + + uint64_t negotiated_features; + + /* Local copy of device coalescing settings. */ + uint32_t coalescing_delay_time_base; + uint32_t coalescing_io_rate_threshold; + + /* Next time when stats for event coalescing will be checked. */ + uint64_t next_stats_check_time; + + /* Interval used for event coalescing checking. */ + uint64_t stats_check_interval; + + struct spdk_vhost_virtqueue virtqueue[SPDK_VHOST_MAX_VQUEUES]; + + TAILQ_ENTRY(spdk_vhost_session) tailq; +}; + +struct spdk_vhost_dev { + char *name; + char *path; + + struct spdk_thread *thread; + bool registered; + + uint64_t virtio_features; + uint64_t disabled_features; + uint64_t protocol_features; + + const struct spdk_vhost_dev_backend *backend; + + /* Saved orginal values used to setup coalescing to avoid integer + * rounding issues during save/load config. + */ + uint32_t coalescing_delay_us; + uint32_t coalescing_iops_threshold; + + /* Current connections to the device */ + TAILQ_HEAD(, spdk_vhost_session) vsessions; + + /* Increment-only session counter */ + uint64_t vsessions_num; + + /* Number of started and actively polled sessions */ + uint32_t active_session_num; + + /* Number of pending asynchronous operations */ + uint32_t pending_async_op_num; + + TAILQ_ENTRY(spdk_vhost_dev) tailq; +}; + +/** + * \param vdev vhost device. + * \param vsession vhost session. + * \param arg user-provided parameter. + * + * \return negative values will break the foreach call, meaning + * the function won't be called again. Return codes zero and + * positive don't have any effect. + */ +typedef int (*spdk_vhost_session_fn)(struct spdk_vhost_dev *vdev, + struct spdk_vhost_session *vsession, + void *arg); + +/** + * \param vdev vhost device. + * \param arg user-provided parameter. + */ +typedef void (*spdk_vhost_dev_fn)(struct spdk_vhost_dev *vdev, void *arg); + +struct spdk_vhost_dev_backend { + /** + * Size of additional per-session context data + * allocated whenever a new client connects. + */ + size_t session_ctx_size; + + int (*start_session)(struct spdk_vhost_session *vsession); + int (*stop_session)(struct spdk_vhost_session *vsession); + + int (*vhost_get_config)(struct spdk_vhost_dev *vdev, uint8_t *config, uint32_t len); + int (*vhost_set_config)(struct spdk_vhost_dev *vdev, uint8_t *config, + uint32_t offset, uint32_t size, uint32_t flags); + + void (*dump_info_json)(struct spdk_vhost_dev *vdev, struct spdk_json_write_ctx *w); + void (*write_config_json)(struct spdk_vhost_dev *vdev, struct spdk_json_write_ctx *w); + int (*remove_device)(struct spdk_vhost_dev *vdev); +}; + +void *vhost_gpa_to_vva(struct spdk_vhost_session *vsession, uint64_t addr, uint64_t len); + +uint16_t vhost_vq_avail_ring_get(struct spdk_vhost_virtqueue *vq, uint16_t *reqs, + uint16_t reqs_len); + +/** + * Get a virtio split descriptor at given index in given virtqueue. + * The descriptor will provide access to the entire descriptor + * chain. The subsequent descriptors are accesible via + * \c spdk_vhost_vring_desc_get_next. + * \param vsession vhost session + * \param vq virtqueue + * \param req_idx descriptor index + * \param desc pointer to be set to the descriptor + * \param desc_table descriptor table to be used with + * \c spdk_vhost_vring_desc_get_next. This might be either + * default virtqueue descriptor table or per-chain indirect + * table. + * \param desc_table_size size of the *desc_table* + * \return 0 on success, -1 if given index is invalid. + * If -1 is returned, the content of params is undefined. + */ +int vhost_vq_get_desc(struct spdk_vhost_session *vsession, struct spdk_vhost_virtqueue *vq, + uint16_t req_idx, struct vring_desc **desc, struct vring_desc **desc_table, + uint32_t *desc_table_size); + +/** + * Get a virtio packed descriptor at given index in given virtqueue. + * The descriptor will provide access to the entire descriptor + * chain. The subsequent descriptors are accesible via + * \c vhost_vring_packed_desc_get_next. + * \param vsession vhost session + * \param vq virtqueue + * \param req_idx descriptor index + * \param desc pointer to be set to the descriptor + * \param desc_table descriptor table to be used with + * \c spdk_vhost_vring_desc_get_next. This might be either + * \c NULL or per-chain indirect table. + * \param desc_table_size size of the *desc_table* + * \return 0 on success, -1 if given index is invalid. + * If -1 is returned, the content of params is undefined. + */ +int vhost_vq_get_desc_packed(struct spdk_vhost_session *vsession, + struct spdk_vhost_virtqueue *virtqueue, + uint16_t req_idx, struct vring_packed_desc **desc, + struct vring_packed_desc **desc_table, uint32_t *desc_table_size); + +/** + * Send IRQ/call client (if pending) for \c vq. + * \param vsession vhost session + * \param vq virtqueue + * \return + * 0 - if no interrupt was signalled + * 1 - if interrupt was signalled + */ +int vhost_vq_used_signal(struct spdk_vhost_session *vsession, struct spdk_vhost_virtqueue *vq); + + +/** + * Send IRQs for all queues that need to be signaled. + * \param vsession vhost session + * \param vq virtqueue + */ +void vhost_session_used_signal(struct spdk_vhost_session *vsession); + +void vhost_vq_used_ring_enqueue(struct spdk_vhost_session *vsession, + struct spdk_vhost_virtqueue *vq, + uint16_t id, uint32_t len); + +/** + * Enqueue the entry to the used ring when device complete the request. + * \param vsession vhost session + * \param vq virtqueue + * \req_idx descriptor index. It's the first index of this descriptor chain. + * \num_descs descriptor count. It's the count of the number of buffers in the chain. + * \buffer_id descriptor buffer ID. + * \length device write length. Specify the length of the buffer that has been initialized + * (written to) by the device + */ +void vhost_vq_packed_ring_enqueue(struct spdk_vhost_session *vsession, + struct spdk_vhost_virtqueue *virtqueue, + uint16_t num_descs, uint16_t buffer_id, + uint32_t length); + +/** + * Get subsequent descriptor from given table. + * \param desc current descriptor, will be set to the + * next descriptor (NULL in case this is the last + * descriptor in the chain or the next desc is invalid) + * \param desc_table descriptor table + * \param desc_table_size size of the *desc_table* + * \return 0 on success, -1 if given index is invalid + * The *desc* param will be set regardless of the + * return value. + */ +int vhost_vring_desc_get_next(struct vring_desc **desc, + struct vring_desc *desc_table, uint32_t desc_table_size); +static inline bool +vhost_vring_desc_is_wr(struct vring_desc *cur_desc) +{ + return !!(cur_desc->flags & VRING_DESC_F_WRITE); +} + +int vhost_vring_desc_to_iov(struct spdk_vhost_session *vsession, struct iovec *iov, + uint16_t *iov_index, const struct vring_desc *desc); + +bool vhost_vq_packed_ring_is_avail(struct spdk_vhost_virtqueue *virtqueue); + +/** + * Get subsequent descriptor from vq or desc table. + * \param desc current descriptor, will be set to the + * next descriptor (NULL in case this is the last + * descriptor in the chain or the next desc is invalid) + * \req_idx index of current desc, will be set to the next + * index. If desc_table != NULL the req_idx is the the vring index + * or the req_idx is the desc_table index. + * \param desc_table descriptor table + * \param desc_table_size size of the *desc_table* + * \return 0 on success, -1 if given index is invalid + * The *desc* param will be set regardless of the + * return value. + */ +int vhost_vring_packed_desc_get_next(struct vring_packed_desc **desc, uint16_t *req_idx, + struct spdk_vhost_virtqueue *vq, + struct vring_packed_desc *desc_table, + uint32_t desc_table_size); + +bool vhost_vring_packed_desc_is_wr(struct vring_packed_desc *cur_desc); + +int vhost_vring_packed_desc_to_iov(struct spdk_vhost_session *vsession, struct iovec *iov, + uint16_t *iov_index, const struct vring_packed_desc *desc); + +uint16_t vhost_vring_packed_desc_get_buffer_id(struct spdk_vhost_virtqueue *vq, uint16_t req_idx, + uint16_t *num_descs); + +static inline bool __attribute__((always_inline)) +vhost_dev_has_feature(struct spdk_vhost_session *vsession, unsigned feature_id) +{ + return vsession->negotiated_features & (1ULL << feature_id); +} + +int vhost_dev_register(struct spdk_vhost_dev *vdev, const char *name, const char *mask_str, + const struct spdk_vhost_dev_backend *backend); +int vhost_dev_unregister(struct spdk_vhost_dev *vdev); + +int vhost_scsi_controller_construct(void); +int vhost_blk_controller_construct(void); +void vhost_dump_info_json(struct spdk_vhost_dev *vdev, struct spdk_json_write_ctx *w); + +/* + * Vhost callbacks for vhost_device_ops interface + */ + +int vhost_new_connection_cb(int vid, const char *ifname); +int vhost_start_device_cb(int vid); +int vhost_stop_device_cb(int vid); +int vhost_destroy_connection_cb(int vid); + +#ifdef SPDK_CONFIG_VHOST_INTERNAL_LIB +int vhost_get_config_cb(int vid, uint8_t *config, uint32_t len); +int vhost_set_config_cb(int vid, uint8_t *config, uint32_t offset, + uint32_t size, uint32_t flags); +#endif + +/* + * Memory registration functions used in start/stop device callbacks + */ +void vhost_session_mem_register(struct rte_vhost_memory *mem); +void vhost_session_mem_unregister(struct rte_vhost_memory *mem); + +/* + * Call a function for each session of the provided vhost device. + * The function will be called one-by-one on each session's thread. + * + * \param vdev vhost device + * \param fn function to call on each session's thread + * \param cpl_fn function to be called at the end of the iteration on + * the vhost management thread. + * Optional, can be NULL. + * \param arg additional argument to the both callbacks + */ +void vhost_dev_foreach_session(struct spdk_vhost_dev *dev, + spdk_vhost_session_fn fn, + spdk_vhost_dev_fn cpl_fn, + void *arg); + +/** + * Call a function on the provided lcore and block until either + * spdk_vhost_session_start_done() or spdk_vhost_session_stop_done() + * is called. + * + * This must be called under the global vhost mutex, which this function + * will unlock for the time it's waiting. It's meant to be called only + * from start/stop session callbacks. + * + * \param vsession vhost session + * \param cb_fn the function to call. The void *arg parameter in cb_fn + * is always NULL. + * \param timeout_sec timeout in seconds. This function will still + * block after the timeout expires, but will print the provided errmsg. + * \param errmsg error message to print once the timeout expires + * \return return the code passed to spdk_vhost_session_event_done(). + */ +int vhost_session_send_event(struct spdk_vhost_session *vsession, + spdk_vhost_session_fn cb_fn, unsigned timeout_sec, + const char *errmsg); + +/** + * Finish a blocking spdk_vhost_session_send_event() call and finally + * start the session. This must be called on the target lcore, which + * will now receive all session-related messages (e.g. from + * spdk_vhost_dev_foreach_session()). + * + * Must be called under the global vhost lock. + * + * \param vsession vhost session + * \param response return code + */ +void vhost_session_start_done(struct spdk_vhost_session *vsession, int response); + +/** + * Finish a blocking spdk_vhost_session_send_event() call and finally + * stop the session. This must be called on the session's lcore which + * used to receive all session-related messages (e.g. from + * spdk_vhost_dev_foreach_session()). After this call, the session- + * related messages will be once again processed by any arbitrary thread. + * + * Must be called under the global vhost lock. + * + * Must be called under the global vhost mutex. + * + * \param vsession vhost session + * \param response return code + */ +void vhost_session_stop_done(struct spdk_vhost_session *vsession, int response); + +struct spdk_vhost_session *vhost_session_find_by_vid(int vid); +void vhost_session_install_rte_compat_hooks(struct spdk_vhost_session *vsession); +int vhost_register_unix_socket(const char *path, const char *ctrl_name, + uint64_t virtio_features, uint64_t disabled_features, uint64_t protocol_features); +int vhost_driver_unregister(const char *path); +int vhost_get_mem_table(int vid, struct rte_vhost_memory **mem); +int vhost_get_negotiated_features(int vid, uint64_t *negotiated_features); + +int remove_vhost_controller(struct spdk_vhost_dev *vdev); + +#ifdef SPDK_CONFIG_VHOST_INTERNAL_LIB +int vhost_nvme_admin_passthrough(int vid, void *cmd, void *cqe, void *buf); +int vhost_nvme_set_cq_call(int vid, uint16_t qid, int fd); +int vhost_nvme_set_bar_mr(int vid, void *bar_addr, uint64_t bar_size); +int vhost_nvme_get_cap(int vid, uint64_t *cap); +int vhost_nvme_controller_construct(void); +int vhost_nvme_dev_construct(const char *name, const char *cpumask, uint32_t io_queues); +int vhost_nvme_dev_remove(struct spdk_vhost_dev *vdev); +int vhost_nvme_dev_add_ns(struct spdk_vhost_dev *vdev, + const char *bdev_name); +#endif + +#endif /* SPDK_VHOST_INTERNAL_H */ diff --git a/src/spdk/lib/vhost/vhost_nvme.c b/src/spdk/lib/vhost/vhost_nvme.c new file mode 100644 index 000000000..10f53baf9 --- /dev/null +++ b/src/spdk/lib/vhost/vhost_nvme.c @@ -0,0 +1,1500 @@ +/*- + * BSD LICENSE + * + * Copyright(c) Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "spdk/stdinc.h" + +#include "spdk/nvme.h" +#include "spdk/env.h" +#include "spdk/conf.h" +#include "spdk/util.h" +#include "spdk/string.h" +#include "spdk/thread.h" +#include "spdk/barrier.h" +#include "spdk/vhost.h" +#include "spdk/bdev.h" +#include "spdk/version.h" +#include "spdk/nvme_spec.h" +#include "spdk/likely.h" + +#include "vhost_internal.h" + +#define MAX_IO_QUEUES 31 +#define MAX_IOVS 64 +#define MAX_NAMESPACE 8 +#define MAX_QUEUE_ENTRIES_SUPPORTED 256 +#define MAX_BATCH_IO 8 + +struct spdk_vhost_nvme_sq { + uint16_t sqid; + uint16_t size; + uint16_t cqid; + bool valid; + struct spdk_nvme_cmd *sq_cmd; + uint16_t sq_head; + uint16_t sq_tail; +}; + +struct spdk_vhost_nvme_cq { + uint8_t phase; + uint16_t size; + uint16_t cqid; + bool valid; + volatile struct spdk_nvme_cpl *cq_cqe; + uint16_t cq_head; + uint16_t guest_signaled_cq_head; + uint32_t need_signaled_cnt; + STAILQ_HEAD(, spdk_vhost_nvme_task) cq_full_waited_tasks; + bool irq_enabled; + int virq; +}; + +struct spdk_vhost_nvme_ns { + struct spdk_bdev *bdev; + uint32_t block_size; + uint64_t capacity; + uint32_t nsid; + uint32_t active_ns; + struct spdk_bdev_desc *bdev_desc; + struct spdk_io_channel *bdev_io_channel; + struct spdk_nvme_ns_data nsdata; +}; + +struct spdk_vhost_nvme_task { + struct spdk_nvme_cmd cmd; + struct spdk_vhost_nvme_dev *nvme; + uint16_t sqid; + uint16_t cqid; + + /** array of iovecs to transfer. */ + struct iovec iovs[MAX_IOVS]; + + /** Number of iovecs in iovs array. */ + int iovcnt; + + /** Current iovec position. */ + int iovpos; + + /** Offset in current iovec. */ + uint32_t iov_offset; + + /* for bdev_io_wait */ + struct spdk_bdev_io_wait_entry bdev_io_wait; + struct spdk_vhost_nvme_sq *sq; + struct spdk_vhost_nvme_ns *ns; + + /* parent pointer. */ + struct spdk_vhost_nvme_task *parent; + uint8_t dnr; + uint8_t sct; + uint8_t sc; + uint32_t num_children; + STAILQ_ENTRY(spdk_vhost_nvme_task) stailq; +}; + +struct spdk_vhost_nvme_dev { + struct spdk_vhost_dev vdev; + + uint32_t num_io_queues; + union spdk_nvme_cap_register cap; + union spdk_nvme_cc_register cc; + union spdk_nvme_csts_register csts; + struct spdk_nvme_ctrlr_data cdata; + + uint32_t num_sqs; + uint32_t num_cqs; + + uint32_t num_ns; + struct spdk_vhost_nvme_ns ns[MAX_NAMESPACE]; + + volatile uint32_t *bar; + volatile uint32_t *bar_db; + uint64_t bar_size; + bool dataplane_started; + + volatile uint32_t *dbbuf_dbs; + volatile uint32_t *dbbuf_eis; + struct spdk_vhost_nvme_sq sq_queue[MAX_IO_QUEUES + 1]; + struct spdk_vhost_nvme_cq cq_queue[MAX_IO_QUEUES + 1]; + + /* The one and only session associated with this device */ + struct spdk_vhost_session *vsession; + + TAILQ_ENTRY(spdk_vhost_nvme_dev) tailq; + STAILQ_HEAD(, spdk_vhost_nvme_task) free_tasks; + struct spdk_poller *requestq_poller; + struct spdk_poller *stop_poller; +}; + +static const struct spdk_vhost_dev_backend spdk_vhost_nvme_device_backend; + +/* + * Report the SPDK version as the firmware revision. + * SPDK_VERSION_STRING won't fit into FR (only 8 bytes), so try to fit the most important parts. + */ +#define FW_VERSION SPDK_VERSION_MAJOR_STRING SPDK_VERSION_MINOR_STRING SPDK_VERSION_PATCH_STRING + +static int +nvme_process_sq(struct spdk_vhost_nvme_dev *nvme, struct spdk_vhost_nvme_sq *sq, + struct spdk_vhost_nvme_task *task); + +static struct spdk_vhost_nvme_dev * +to_nvme_dev(struct spdk_vhost_dev *vdev) +{ + if (vdev->backend != &spdk_vhost_nvme_device_backend) { + SPDK_ERRLOG("%s: not a vhost-nvme device\n", vdev->name); + return NULL; + } + + return SPDK_CONTAINEROF(vdev, struct spdk_vhost_nvme_dev, vdev); +} + +static TAILQ_HEAD(, spdk_vhost_nvme_dev) g_nvme_ctrlrs = TAILQ_HEAD_INITIALIZER(g_nvme_ctrlrs); + +static inline unsigned int sq_offset(unsigned int qid, uint32_t db_stride) +{ + return qid * 2 * db_stride; +} + +static inline unsigned int cq_offset(unsigned int qid, uint32_t db_stride) +{ + return (qid * 2 + 1) * db_stride; +} + +static void +nvme_inc_cq_head(struct spdk_vhost_nvme_cq *cq) +{ + cq->cq_head++; + if (cq->cq_head >= cq->size) { + cq->cq_head = 0; + cq->phase = !cq->phase; + } +} + +static bool +nvme_cq_is_full(struct spdk_vhost_nvme_cq *cq) +{ + return ((cq->cq_head + 1) % cq->size == cq->guest_signaled_cq_head); +} + +static void +nvme_inc_sq_head(struct spdk_vhost_nvme_sq *sq) +{ + sq->sq_head = (sq->sq_head + 1) % sq->size; +} + +static struct spdk_vhost_nvme_sq * +vhost_nvme_get_sq_from_qid(struct spdk_vhost_nvme_dev *dev, uint16_t qid) +{ + if (spdk_unlikely(!qid || qid > MAX_IO_QUEUES)) { + return NULL; + } + + return &dev->sq_queue[qid]; +} + +static struct spdk_vhost_nvme_cq * +vhost_nvme_get_cq_from_qid(struct spdk_vhost_nvme_dev *dev, uint16_t qid) +{ + if (spdk_unlikely(!qid || qid > MAX_IO_QUEUES)) { + return NULL; + } + + return &dev->cq_queue[qid]; +} + +static inline uint32_t +vhost_nvme_get_queue_head(struct spdk_vhost_nvme_dev *nvme, uint32_t offset) +{ + if (nvme->dataplane_started) { + return nvme->dbbuf_dbs[offset]; + + } else if (nvme->bar) { + return nvme->bar_db[offset]; + } + + assert(0); + + return 0; +} + +static void * +vhost_nvme_gpa_to_vva(void *priv, uint64_t addr, uint64_t len) +{ + struct spdk_vhost_session *vsession = priv; + + return vhost_gpa_to_vva(vsession, addr, len); +} + +static int +vhost_nvme_map_prps(struct spdk_vhost_nvme_dev *nvme, struct spdk_nvme_cmd *cmd, + struct spdk_vhost_nvme_task *task, uint32_t len) +{ + int err; + + err = spdk_nvme_map_prps(nvme->vsession, cmd, task->iovs, len, 4096, + vhost_nvme_gpa_to_vva); + if (spdk_unlikely(err < 0)) { + return err; + } + task->iovcnt = err; + return 0; +} + +static void +nvme_cq_signal_fd(struct spdk_vhost_nvme_dev *nvme) +{ + struct spdk_vhost_nvme_cq *cq; + uint32_t qid, cq_head; + + assert(nvme != NULL); + + for (qid = 1; qid <= MAX_IO_QUEUES; qid++) { + cq = vhost_nvme_get_cq_from_qid(nvme, qid); + if (!cq || !cq->valid) { + continue; + } + + cq_head = vhost_nvme_get_queue_head(nvme, cq_offset(qid, 1)); + if (cq->irq_enabled && cq->need_signaled_cnt && (cq->cq_head != cq_head)) { + eventfd_write(cq->virq, (eventfd_t)1); + cq->need_signaled_cnt = 0; + } + } +} + +static void +vhost_nvme_task_complete(struct spdk_vhost_nvme_task *task) +{ + struct spdk_vhost_nvme_dev *nvme = task->nvme; + struct spdk_nvme_cpl cqe = {0}; + struct spdk_vhost_nvme_cq *cq; + struct spdk_vhost_nvme_sq *sq; + struct spdk_nvme_cmd *cmd = &task->cmd; + uint16_t cqid = task->cqid; + uint16_t sqid = task->sqid; + + cq = vhost_nvme_get_cq_from_qid(nvme, cqid); + sq = vhost_nvme_get_sq_from_qid(nvme, sqid); + if (spdk_unlikely(!cq || !sq)) { + return; + } + + cq->guest_signaled_cq_head = vhost_nvme_get_queue_head(nvme, cq_offset(cqid, 1)); + if (spdk_unlikely(nvme_cq_is_full(cq))) { + STAILQ_INSERT_TAIL(&cq->cq_full_waited_tasks, task, stailq); + return; + } + + cqe.sqid = sqid; + cqe.sqhd = sq->sq_head; + cqe.cid = cmd->cid; + cqe.status.dnr = task->dnr; + cqe.status.sct = task->sct; + cqe.status.sc = task->sc; + cqe.status.p = !cq->phase; + cq->cq_cqe[cq->cq_head] = cqe; + spdk_smp_wmb(); + cq->cq_cqe[cq->cq_head].status.p = cq->phase; + + nvme_inc_cq_head(cq); + cq->need_signaled_cnt++; + + /* MMIO Controll */ + if (nvme->dataplane_started) { + nvme->dbbuf_eis[cq_offset(cqid, 1)] = (uint32_t)(cq->guest_signaled_cq_head - 1); + } + + STAILQ_INSERT_TAIL(&nvme->free_tasks, task, stailq); +} + +static void +blk_request_complete_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) +{ + struct spdk_vhost_nvme_task *task = cb_arg; + struct spdk_nvme_cmd *cmd = &task->cmd; + int sc, sct; + uint32_t cdw0; + + assert(bdev_io != NULL); + + spdk_bdev_io_get_nvme_status(bdev_io, &cdw0, &sct, &sc); + spdk_bdev_free_io(bdev_io); + + task->dnr = !success; + task->sct = sct; + task->sc = sc; + + if (spdk_unlikely(!success)) { + SPDK_ERRLOG("I/O error, sector %u\n", cmd->cdw10); + } + + vhost_nvme_task_complete(task); +} + +static void +blk_unmap_complete_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) +{ + struct spdk_vhost_nvme_task *child = cb_arg; + struct spdk_vhost_nvme_task *task = child->parent; + struct spdk_vhost_nvme_dev *nvme = task->nvme; + int sct, sc; + uint32_t cdw0; + + assert(bdev_io != NULL); + + task->num_children--; + if (!success) { + task->dnr = 1; + spdk_bdev_io_get_nvme_status(bdev_io, &cdw0, &sct, &sc); + task->sct = sct; + task->sc = sc; + } + + spdk_bdev_free_io(bdev_io); + + if (!task->num_children) { + vhost_nvme_task_complete(task); + } + + STAILQ_INSERT_TAIL(&nvme->free_tasks, child, stailq); +} + +static struct spdk_vhost_nvme_ns * +vhost_nvme_get_ns_from_nsid(struct spdk_vhost_nvme_dev *dev, uint32_t nsid) +{ + if (spdk_unlikely(!nsid || nsid > dev->num_ns)) { + return NULL; + } + + return &dev->ns[nsid - 1]; +} + +static void +vhost_nvme_resubmit_task(void *arg) +{ + struct spdk_vhost_nvme_task *task = (struct spdk_vhost_nvme_task *)arg; + int rc; + + rc = nvme_process_sq(task->nvme, task->sq, task); + if (rc) { + SPDK_DEBUGLOG(SPDK_LOG_VHOST_NVME, "vhost_nvme: task resubmit failed, rc = %d.\n", rc); + } +} + +static int +vhost_nvme_queue_task(struct spdk_vhost_nvme_task *task) +{ + int rc; + + task->bdev_io_wait.bdev = task->ns->bdev; + task->bdev_io_wait.cb_fn = vhost_nvme_resubmit_task; + task->bdev_io_wait.cb_arg = task; + + rc = spdk_bdev_queue_io_wait(task->ns->bdev, task->ns->bdev_io_channel, &task->bdev_io_wait); + if (rc != 0) { + SPDK_ERRLOG("Queue io failed in vhost_nvme_queue_task, rc=%d.\n", rc); + task->dnr = 1; + task->sct = SPDK_NVME_SCT_GENERIC; + task->sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; + vhost_nvme_task_complete(task); + } + + return rc; +} + +static int +nvme_process_sq(struct spdk_vhost_nvme_dev *nvme, struct spdk_vhost_nvme_sq *sq, + struct spdk_vhost_nvme_task *task) +{ + struct spdk_vhost_nvme_task *child; + struct spdk_nvme_cmd *cmd = &task->cmd; + struct spdk_vhost_nvme_ns *ns; + int ret = -1; + uint32_t len, nlba, block_size; + uint64_t slba; + struct spdk_nvme_dsm_range *range; + uint16_t i, num_ranges = 0; + + task->nvme = nvme; + task->dnr = 0; + task->sct = 0; + task->sc = 0; + + ns = vhost_nvme_get_ns_from_nsid(nvme, cmd->nsid); + if (spdk_unlikely(!ns)) { + task->dnr = 1; + task->sct = SPDK_NVME_SCT_GENERIC; + task->sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; + vhost_nvme_task_complete(task); + return -1; + } + + block_size = ns->block_size; + task->num_children = 0; + task->cqid = sq->cqid; + task->sqid = sq->sqid; + + task->ns = ns; + + if (spdk_unlikely(!ns->active_ns)) { + task->dnr = 1; + task->sct = SPDK_NVME_SCT_GENERIC; + task->sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; + vhost_nvme_task_complete(task); + return -1; + } + + /* valid only for Read/Write commands */ + nlba = (cmd->cdw12 & 0xffff) + 1; + slba = cmd->cdw11; + slba = (slba << 32) | cmd->cdw10; + + if (cmd->opc == SPDK_NVME_OPC_READ || cmd->opc == SPDK_NVME_OPC_WRITE || + cmd->opc == SPDK_NVME_OPC_DATASET_MANAGEMENT) { + if (cmd->psdt != SPDK_NVME_PSDT_PRP) { + SPDK_DEBUGLOG(SPDK_LOG_VHOST_NVME, "Invalid PSDT %u%ub in command\n", + cmd->psdt >> 1, cmd->psdt & 1u); + task->dnr = 1; + task->sct = SPDK_NVME_SCT_GENERIC; + task->sc = SPDK_NVME_SC_INVALID_FIELD; + vhost_nvme_task_complete(task); + return -1; + } + + if (cmd->opc == SPDK_NVME_OPC_DATASET_MANAGEMENT) { + num_ranges = (cmd->cdw10 & 0xff) + 1; + len = num_ranges * sizeof(struct spdk_nvme_dsm_range); + } else { + len = nlba * block_size; + } + + ret = vhost_nvme_map_prps(nvme, cmd, task, len); + if (spdk_unlikely(ret != 0)) { + SPDK_ERRLOG("nvme command map prps failed\n"); + task->dnr = 1; + task->sct = SPDK_NVME_SCT_GENERIC; + task->sc = SPDK_NVME_SC_INVALID_FIELD; + vhost_nvme_task_complete(task); + return -1; + } + } + + switch (cmd->opc) { + case SPDK_NVME_OPC_READ: + ret = spdk_bdev_readv(ns->bdev_desc, ns->bdev_io_channel, + task->iovs, task->iovcnt, slba * block_size, + nlba * block_size, blk_request_complete_cb, task); + break; + case SPDK_NVME_OPC_WRITE: + ret = spdk_bdev_writev(ns->bdev_desc, ns->bdev_io_channel, + task->iovs, task->iovcnt, slba * block_size, + nlba * block_size, blk_request_complete_cb, task); + break; + case SPDK_NVME_OPC_FLUSH: + ret = spdk_bdev_flush(ns->bdev_desc, ns->bdev_io_channel, + 0, ns->capacity, + blk_request_complete_cb, task); + break; + case SPDK_NVME_OPC_DATASET_MANAGEMENT: + range = (struct spdk_nvme_dsm_range *)task->iovs[0].iov_base; + for (i = 0; i < num_ranges; i++) { + if (!STAILQ_EMPTY(&nvme->free_tasks)) { + child = STAILQ_FIRST(&nvme->free_tasks); + STAILQ_REMOVE_HEAD(&nvme->free_tasks, stailq); + } else { + SPDK_ERRLOG("No free task now\n"); + ret = -1; + break; + } + task->num_children++; + child->parent = task; + ret = spdk_bdev_unmap(ns->bdev_desc, ns->bdev_io_channel, + range[i].starting_lba * block_size, + range[i].length * block_size, + blk_unmap_complete_cb, child); + if (ret) { + STAILQ_INSERT_TAIL(&nvme->free_tasks, child, stailq); + break; + } + } + break; + default: + ret = -1; + break; + } + + if (spdk_unlikely(ret)) { + if (ret == -ENOMEM) { + SPDK_DEBUGLOG(SPDK_LOG_VHOST_NVME, "No memory, start to queue io.\n"); + task->sq = sq; + ret = vhost_nvme_queue_task(task); + } else { + /* post error status to cqe */ + SPDK_ERRLOG("Error Submission For Command %u, ret %d\n", cmd->opc, ret); + task->dnr = 1; + task->sct = SPDK_NVME_SCT_GENERIC; + task->sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; + vhost_nvme_task_complete(task); + } + } + + return ret; +} + +static int +nvme_worker(void *arg) +{ + struct spdk_vhost_nvme_dev *nvme = (struct spdk_vhost_nvme_dev *)arg; + struct spdk_vhost_nvme_sq *sq; + struct spdk_vhost_nvme_cq *cq; + struct spdk_vhost_nvme_task *task; + uint32_t qid, dbbuf_sq; + int ret; + int count = -1; + + if (spdk_unlikely(!nvme->num_sqs)) { + return SPDK_POLLER_IDLE; + } + + if (spdk_unlikely(!nvme->dataplane_started && !nvme->bar)) { + return SPDK_POLLER_IDLE; + } + + for (qid = 1; qid <= MAX_IO_QUEUES; qid++) { + + sq = vhost_nvme_get_sq_from_qid(nvme, qid); + if (!sq->valid) { + continue; + } + cq = vhost_nvme_get_cq_from_qid(nvme, sq->cqid); + if (spdk_unlikely(!cq)) { + return SPDK_POLLER_BUSY; + } + cq->guest_signaled_cq_head = vhost_nvme_get_queue_head(nvme, cq_offset(sq->cqid, 1)); + if (spdk_unlikely(!STAILQ_EMPTY(&cq->cq_full_waited_tasks) && + !nvme_cq_is_full(cq))) { + task = STAILQ_FIRST(&cq->cq_full_waited_tasks); + STAILQ_REMOVE_HEAD(&cq->cq_full_waited_tasks, stailq); + vhost_nvme_task_complete(task); + } + + dbbuf_sq = vhost_nvme_get_queue_head(nvme, sq_offset(qid, 1)); + sq->sq_tail = (uint16_t)dbbuf_sq; + count = 0; + + while (sq->sq_head != sq->sq_tail) { + if (spdk_unlikely(!sq->sq_cmd)) { + break; + } + if (spdk_likely(!STAILQ_EMPTY(&nvme->free_tasks))) { + task = STAILQ_FIRST(&nvme->free_tasks); + STAILQ_REMOVE_HEAD(&nvme->free_tasks, stailq); + } else { + return SPDK_POLLER_BUSY; + } + + task->cmd = sq->sq_cmd[sq->sq_head]; + nvme_inc_sq_head(sq); + + /* processing IO */ + ret = nvme_process_sq(nvme, sq, task); + if (spdk_unlikely(ret)) { + SPDK_ERRLOG("QID %u CID %u, SQ HEAD %u, DBBUF SQ TAIL %u\n", qid, task->cmd.cid, sq->sq_head, + sq->sq_tail); + } + + /* MMIO Control */ + if (nvme->dataplane_started) { + nvme->dbbuf_eis[sq_offset(qid, 1)] = (uint32_t)(sq->sq_head - 1); + } + + /* Maximum batch I/Os to pick up at once */ + if (count++ == MAX_BATCH_IO) { + break; + } + } + } + + /* Completion Queue */ + nvme_cq_signal_fd(nvme); + + return count; +} + +static int +vhost_nvme_doorbell_buffer_config(struct spdk_vhost_nvme_dev *nvme, + struct spdk_nvme_cmd *cmd, struct spdk_nvme_cpl *cpl) +{ + struct spdk_vhost_session *vsession = nvme->vsession; + uint64_t dbs_dma_addr, eis_dma_addr; + + dbs_dma_addr = cmd->dptr.prp.prp1; + eis_dma_addr = cmd->dptr.prp.prp2; + + if ((dbs_dma_addr % 4096) || (eis_dma_addr % 4096)) { + return -1; + } + /* Guest Physical Address to Host Virtual Address */ + nvme->dbbuf_dbs = vhost_gpa_to_vva(vsession, dbs_dma_addr, 4096); + nvme->dbbuf_eis = vhost_gpa_to_vva(vsession, eis_dma_addr, 4096); + if (!nvme->dbbuf_dbs || !nvme->dbbuf_eis) { + return -1; + } + /* zeroed the doorbell buffer memory */ + memset((void *)nvme->dbbuf_dbs, 0, 4096); + memset((void *)nvme->dbbuf_eis, 0, 4096); + + cpl->status.sc = 0; + cpl->status.sct = 0; + + /* Data plane started */ + nvme->dataplane_started = true; + + return 0; +} + +static int +vhost_nvme_create_io_sq(struct spdk_vhost_nvme_dev *nvme, + struct spdk_nvme_cmd *cmd, struct spdk_nvme_cpl *cpl) +{ + uint16_t qid, qsize, cqid; + uint64_t dma_addr; + uint64_t requested_len; + struct spdk_vhost_nvme_cq *cq; + struct spdk_vhost_nvme_sq *sq; + + /* physical contiguous */ + if (!(cmd->cdw11 & 0x1)) { + return -1; + } + + cqid = (cmd->cdw11 >> 16) & 0xffff; + qid = cmd->cdw10 & 0xffff; + qsize = (cmd->cdw10 >> 16) & 0xffff; + dma_addr = cmd->dptr.prp.prp1; + if (!dma_addr || dma_addr % 4096) { + return -1; + } + + sq = vhost_nvme_get_sq_from_qid(nvme, qid); + cq = vhost_nvme_get_cq_from_qid(nvme, cqid); + if (!sq || !cq) { + SPDK_DEBUGLOG(SPDK_LOG_VHOST_NVME, "User requested invalid QID %u or CQID %u\n", + qid, cqid); + cpl->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; + cpl->status.sc = SPDK_NVME_SC_INVALID_QUEUE_IDENTIFIER; + return -1; + } + + sq->sqid = qid; + sq->cqid = cqid; + sq->size = qsize + 1; + sq->sq_head = sq->sq_tail = 0; + requested_len = sizeof(struct spdk_nvme_cmd) * sq->size; + sq->sq_cmd = vhost_gpa_to_vva(nvme->vsession, dma_addr, requested_len); + if (!sq->sq_cmd) { + return -1; + } + nvme->num_sqs++; + sq->valid = true; + if (nvme->bar) { + nvme->bar_db[sq_offset(qid, 1)] = 0; + } + + cpl->status.sc = 0; + cpl->status.sct = 0; + return 0; +} + +static int +vhost_nvme_delete_io_sq(struct spdk_vhost_nvme_dev *nvme, + struct spdk_nvme_cmd *cmd, struct spdk_nvme_cpl *cpl) +{ + uint16_t qid; + struct spdk_vhost_nvme_sq *sq; + + qid = cmd->cdw10 & 0xffff; + sq = vhost_nvme_get_sq_from_qid(nvme, qid); + if (!sq) { + return -1; + } + + /* We didn't see scenarios when deleting submission + * queue while I/O is running against the submisson + * queue for now, otherwise, we must ensure the poller + * will not run with this submission queue. + */ + nvme->num_sqs--; + sq->valid = false; + + memset(sq, 0, sizeof(*sq)); + sq->sq_cmd = NULL; + + cpl->status.sc = 0; + cpl->status.sct = 0; + + return 0; +} + +static int +vhost_nvme_create_io_cq(struct spdk_vhost_nvme_dev *nvme, + struct spdk_nvme_cmd *cmd, struct spdk_nvme_cpl *cpl) +{ + uint16_t qsize, qid; + uint64_t dma_addr; + struct spdk_vhost_nvme_cq *cq; + uint64_t requested_len; + + /* physical contiguous */ + if (!(cmd->cdw11 & 0x1)) { + return -1; + } + + qid = cmd->cdw10 & 0xffff; + qsize = (cmd->cdw10 >> 16) & 0xffff; + dma_addr = cmd->dptr.prp.prp1; + if (!dma_addr || dma_addr % 4096) { + return -1; + } + + cq = vhost_nvme_get_cq_from_qid(nvme, qid); + if (!cq) { + SPDK_DEBUGLOG(SPDK_LOG_VHOST_NVME, "User requested invalid QID %u\n", qid); + cpl->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; + cpl->status.sc = SPDK_NVME_SC_INVALID_QUEUE_IDENTIFIER; + return -1; + } + cq->cqid = qid; + cq->size = qsize + 1; + cq->phase = 1; + cq->irq_enabled = (cmd->cdw11 >> 1) & 0x1; + /* Setup virq through vhost messages */ + cq->virq = -1; + cq->cq_head = 0; + cq->guest_signaled_cq_head = 0; + cq->need_signaled_cnt = 0; + requested_len = sizeof(struct spdk_nvme_cpl) * cq->size; + cq->cq_cqe = vhost_gpa_to_vva(nvme->vsession, dma_addr, requested_len); + if (!cq->cq_cqe) { + return -1; + } + nvme->num_cqs++; + cq->valid = true; + if (nvme->bar) { + nvme->bar_db[cq_offset(qid, 1)] = 0; + } + STAILQ_INIT(&cq->cq_full_waited_tasks); + + cpl->status.sc = 0; + cpl->status.sct = 0; + return 0; +} + +static int +vhost_nvme_delete_io_cq(struct spdk_vhost_nvme_dev *nvme, + struct spdk_nvme_cmd *cmd, struct spdk_nvme_cpl *cpl) +{ + uint16_t qid; + struct spdk_vhost_nvme_cq *cq; + + qid = cmd->cdw10 & 0xffff; + cq = vhost_nvme_get_cq_from_qid(nvme, qid); + if (!cq) { + return -1; + } + nvme->num_cqs--; + cq->valid = false; + + memset(cq, 0, sizeof(*cq)); + cq->cq_cqe = NULL; + + cpl->status.sc = 0; + cpl->status.sct = 0; + return 0; +} + +static struct spdk_vhost_nvme_dev * +vhost_nvme_get_by_name(int vid) +{ + struct spdk_vhost_nvme_dev *nvme; + struct spdk_vhost_dev *vdev; + struct spdk_vhost_session *vsession; + + TAILQ_FOREACH(nvme, &g_nvme_ctrlrs, tailq) { + vdev = &nvme->vdev; + TAILQ_FOREACH(vsession, &vdev->vsessions, tailq) { + if (vsession->vid == vid) { + return nvme; + } + } + } + + return NULL; +} + +int +vhost_nvme_get_cap(int vid, uint64_t *cap) +{ + struct spdk_vhost_nvme_dev *nvme; + + nvme = vhost_nvme_get_by_name(vid); + if (!nvme) { + return -1; + } + + *cap = nvme->cap.raw; + return 0; +} + +int +vhost_nvme_admin_passthrough(int vid, void *cmd, void *cqe, void *buf) +{ + struct spdk_nvme_cmd *req = (struct spdk_nvme_cmd *)cmd; + struct spdk_nvme_cpl *cpl = (struct spdk_nvme_cpl *)cqe; + struct spdk_vhost_nvme_ns *ns; + int ret = 0; + struct spdk_vhost_nvme_dev *nvme; + + nvme = vhost_nvme_get_by_name(vid); + if (!nvme) { + return -1; + } + + SPDK_DEBUGLOG(SPDK_LOG_VHOST_NVME, "Admin Command Opcode %u\n", req->opc); + switch (req->opc) { + case SPDK_NVME_OPC_IDENTIFY: + if (req->cdw10 == SPDK_NVME_IDENTIFY_CTRLR) { + memcpy(buf, &nvme->cdata, sizeof(struct spdk_nvme_ctrlr_data)); + + } else if (req->cdw10 == SPDK_NVME_IDENTIFY_NS) { + ns = vhost_nvme_get_ns_from_nsid(nvme, req->nsid); + if (!ns) { + cpl->status.sc = SPDK_NVME_SC_NAMESPACE_ID_UNAVAILABLE; + cpl->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; + break; + } + memcpy(buf, &ns->nsdata, sizeof(struct spdk_nvme_ns_data)); + } + /* successfully */ + cpl->status.sc = 0; + cpl->status.sct = 0; + break; + case SPDK_NVME_OPC_CREATE_IO_CQ: + ret = vhost_nvme_create_io_cq(nvme, req, cpl); + break; + case SPDK_NVME_OPC_DELETE_IO_CQ: + ret = vhost_nvme_delete_io_cq(nvme, req, cpl); + break; + case SPDK_NVME_OPC_CREATE_IO_SQ: + ret = vhost_nvme_create_io_sq(nvme, req, cpl); + break; + case SPDK_NVME_OPC_DELETE_IO_SQ: + ret = vhost_nvme_delete_io_sq(nvme, req, cpl); + break; + case SPDK_NVME_OPC_GET_FEATURES: + case SPDK_NVME_OPC_SET_FEATURES: + if (req->cdw10 == SPDK_NVME_FEAT_NUMBER_OF_QUEUES) { + cpl->status.sc = 0; + cpl->status.sct = 0; + cpl->cdw0 = (nvme->num_io_queues - 1) | ((nvme->num_io_queues - 1) << 16); + } else { + cpl->status.sc = SPDK_NVME_SC_INVALID_FIELD; + cpl->status.sct = SPDK_NVME_SCT_GENERIC; + } + break; + case SPDK_NVME_OPC_DOORBELL_BUFFER_CONFIG: + ret = vhost_nvme_doorbell_buffer_config(nvme, req, cpl); + break; + case SPDK_NVME_OPC_ABORT: + /* TODO: ABORT failed fow now */ + cpl->cdw0 = 1; + cpl->status.sc = 0; + cpl->status.sct = 0; + break; + } + + if (ret) { + SPDK_ERRLOG("Admin Passthrough Failed with %u\n", req->opc); + } + + return 0; +} + +int +vhost_nvme_set_bar_mr(int vid, void *bar_addr, uint64_t bar_size) +{ + struct spdk_vhost_nvme_dev *nvme; + + nvme = vhost_nvme_get_by_name(vid); + if (!nvme) { + return -1; + } + + nvme->bar = (volatile uint32_t *)(uintptr_t)(bar_addr); + /* BAR0 SQ/CQ doorbell registers start from offset 0x1000 */ + nvme->bar_db = (volatile uint32_t *)(uintptr_t)(bar_addr + 0x1000ull); + nvme->bar_size = bar_size; + + return 0; +} + +int +vhost_nvme_set_cq_call(int vid, uint16_t qid, int fd) +{ + struct spdk_vhost_nvme_dev *nvme; + struct spdk_vhost_nvme_cq *cq; + + nvme = vhost_nvme_get_by_name(vid); + if (!nvme) { + return -1; + } + + cq = vhost_nvme_get_cq_from_qid(nvme, qid); + if (!cq) { + return -1; + } + if (cq->irq_enabled) { + cq->virq = fd; + } else { + SPDK_ERRLOG("NVMe Qid %d Disabled IRQ\n", qid); + } + + return 0; +} + +static void +free_task_pool(struct spdk_vhost_nvme_dev *nvme) +{ + struct spdk_vhost_nvme_task *task; + + while (!STAILQ_EMPTY(&nvme->free_tasks)) { + task = STAILQ_FIRST(&nvme->free_tasks); + STAILQ_REMOVE_HEAD(&nvme->free_tasks, stailq); + spdk_free(task); + } +} + +static int +alloc_task_pool(struct spdk_vhost_nvme_dev *nvme) +{ + uint32_t entries, i; + struct spdk_vhost_nvme_task *task; + + entries = nvme->num_io_queues * MAX_QUEUE_ENTRIES_SUPPORTED; + + for (i = 0; i < entries; i++) { + task = spdk_zmalloc(sizeof(struct spdk_vhost_nvme_task), + SPDK_CACHE_LINE_SIZE, NULL, + SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA); + if (task == NULL) { + SPDK_ERRLOG("Controller %s alloc task pool failed\n", + nvme->vdev.name); + free_task_pool(nvme); + return -1; + } + STAILQ_INSERT_TAIL(&nvme->free_tasks, task, stailq); + } + + return 0; +} + +static int +vhost_nvme_start_cb(struct spdk_vhost_dev *vdev, + struct spdk_vhost_session *vsession, void *unused) +{ + struct spdk_vhost_nvme_dev *nvme = to_nvme_dev(vdev); + struct spdk_vhost_nvme_ns *ns_dev; + uint32_t i; + int rc = 0; + + if (nvme == NULL) { + rc = -1; + goto out; + } + + rc = alloc_task_pool(nvme); + if (rc) { + goto out; + } + + SPDK_NOTICELOG("Start Device %u, Path %s, lcore %d\n", vsession->vid, + vdev->path, spdk_env_get_current_core()); + + for (i = 0; i < nvme->num_ns; i++) { + ns_dev = &nvme->ns[i]; + ns_dev->bdev_io_channel = spdk_bdev_get_io_channel(ns_dev->bdev_desc); + if (!ns_dev->bdev_io_channel) { + rc = -1; + goto out; + } + } + + nvme->vsession = vsession; + /* Start the NVMe Poller */ + nvme->requestq_poller = SPDK_POLLER_REGISTER(nvme_worker, nvme, 0); + +out: + vhost_session_start_done(vsession, rc); + return rc; +} + +static int +vhost_nvme_start(struct spdk_vhost_session *vsession) +{ + if (vsession->vdev->active_session_num > 0) { + /* We're trying to start a second session */ + SPDK_ERRLOG("Vhost-NVMe devices can support only one simultaneous connection.\n"); + return -1; + } + + return vhost_session_send_event(vsession, vhost_nvme_start_cb, + 3, "start session"); +} + +static void +vhost_nvme_deactive_ns(struct spdk_vhost_nvme_ns *ns) +{ + ns->active_ns = 0; + spdk_bdev_close(ns->bdev_desc); + ns->bdev_desc = NULL; + ns->bdev = NULL; +} + +static void +bdev_remove_cb(void *remove_ctx) +{ + struct spdk_vhost_nvme_ns *ns = remove_ctx; + + SPDK_NOTICELOG("Removing NS %u, Block Device %s\n", + ns->nsid, spdk_bdev_get_name(ns->bdev)); + + vhost_nvme_deactive_ns(ns); +} + +static int +destroy_device_poller_cb(void *arg) +{ + struct spdk_vhost_nvme_dev *nvme = arg; + struct spdk_vhost_nvme_ns *ns_dev; + uint32_t i; + + SPDK_DEBUGLOG(SPDK_LOG_VHOST_NVME, "Destroy device poller callback\n"); + + /* FIXME wait for pending I/Os to complete */ + + if (spdk_vhost_trylock() != 0) { + return SPDK_POLLER_BUSY; + } + + for (i = 0; i < nvme->num_ns; i++) { + ns_dev = &nvme->ns[i]; + if (ns_dev->bdev_io_channel) { + spdk_put_io_channel(ns_dev->bdev_io_channel); + ns_dev->bdev_io_channel = NULL; + } + } + /* Clear BAR space */ + if (nvme->bar) { + memset((void *)nvme->bar, 0, nvme->bar_size); + } + nvme->num_sqs = 0; + nvme->num_cqs = 0; + nvme->dbbuf_dbs = NULL; + nvme->dbbuf_eis = NULL; + nvme->dataplane_started = false; + + spdk_poller_unregister(&nvme->stop_poller); + vhost_session_stop_done(nvme->vsession, 0); + + spdk_vhost_unlock(); + return SPDK_POLLER_BUSY; +} + +static int +vhost_nvme_stop_cb(struct spdk_vhost_dev *vdev, + struct spdk_vhost_session *vsession, void *unused) +{ + struct spdk_vhost_nvme_dev *nvme = to_nvme_dev(vdev); + + if (nvme == NULL) { + vhost_session_stop_done(vsession, -1); + return -1; + } + + free_task_pool(nvme); + SPDK_NOTICELOG("Stopping Device %u, Path %s\n", vsession->vid, vdev->path); + + spdk_poller_unregister(&nvme->requestq_poller); + nvme->stop_poller = SPDK_POLLER_REGISTER(destroy_device_poller_cb, nvme, 1000); + + return 0; +} + +static int +vhost_nvme_stop(struct spdk_vhost_session *vsession) +{ + return vhost_session_send_event(vsession, vhost_nvme_stop_cb, + 3, "start session"); +} + +static void +vhost_nvme_dump_info_json(struct spdk_vhost_dev *vdev, struct spdk_json_write_ctx *w) +{ + struct spdk_vhost_nvme_dev *nvme = to_nvme_dev(vdev); + struct spdk_vhost_nvme_ns *ns_dev; + uint32_t i; + + if (nvme == NULL) { + return; + } + + spdk_json_write_named_array_begin(w, "namespaces"); + + for (i = 0; i < nvme->num_ns; i++) { + ns_dev = &nvme->ns[i]; + if (!ns_dev->active_ns) { + continue; + } + + spdk_json_write_object_begin(w); + spdk_json_write_named_uint32(w, "nsid", ns_dev->nsid); + spdk_json_write_named_string(w, "bdev", spdk_bdev_get_name(ns_dev->bdev)); + spdk_json_write_object_end(w); + } + + spdk_json_write_array_end(w); +} + +static void +vhost_nvme_write_config_json(struct spdk_vhost_dev *vdev, struct spdk_json_write_ctx *w) +{ + struct spdk_vhost_nvme_dev *nvme = to_nvme_dev(vdev); + struct spdk_vhost_nvme_ns *ns_dev; + uint32_t i; + + if (nvme == NULL) { + return; + } + + spdk_json_write_object_begin(w); + spdk_json_write_named_string(w, "method", "vhost_create_nvme_controller"); + + spdk_json_write_named_object_begin(w, "params"); + spdk_json_write_named_string(w, "ctrlr", nvme->vdev.name); + spdk_json_write_named_uint32(w, "io_queues", nvme->num_io_queues); + spdk_json_write_named_string(w, "cpumask", + spdk_cpuset_fmt(spdk_thread_get_cpumask(nvme->vdev.thread))); + spdk_json_write_object_end(w); + + spdk_json_write_object_end(w); + + for (i = 0; i < nvme->num_ns; i++) { + ns_dev = &nvme->ns[i]; + if (!ns_dev->active_ns) { + continue; + } + + spdk_json_write_object_begin(w); + spdk_json_write_named_string(w, "method", "vhost_nvme_controller_add_ns"); + + spdk_json_write_named_object_begin(w, "params"); + spdk_json_write_named_string(w, "ctrlr", nvme->vdev.name); + spdk_json_write_named_string(w, "bdev_name", spdk_bdev_get_name(ns_dev->bdev)); + spdk_json_write_object_end(w); + + spdk_json_write_object_end(w); + } +} + +static const struct spdk_vhost_dev_backend spdk_vhost_nvme_device_backend = { + .session_ctx_size = 0, + .start_session = vhost_nvme_start, + .stop_session = vhost_nvme_stop, + .dump_info_json = vhost_nvme_dump_info_json, + .write_config_json = vhost_nvme_write_config_json, + .remove_device = vhost_nvme_dev_remove, +}; + +static int +vhost_nvme_ns_identify_update(struct spdk_vhost_nvme_dev *dev) +{ + struct spdk_nvme_ctrlr_data *cdata = &dev->cdata; + struct spdk_nvme_ns_data *nsdata; + uint64_t num_blocks; + uint32_t i; + + /* Identify Namespace */ + cdata->nn = dev->num_ns; + for (i = 0; i < dev->num_ns; i++) { + nsdata = &dev->ns[i].nsdata; + if (dev->ns[i].active_ns) { + num_blocks = spdk_bdev_get_num_blocks(dev->ns[i].bdev); + nsdata->nsze = num_blocks; + /* ncap must be non-zero for active Namespace */ + nsdata->ncap = num_blocks; + nsdata->nuse = num_blocks; + nsdata->nlbaf = 0; + nsdata->flbas.format = 0; + nsdata->lbaf[0].lbads = spdk_u32log2(spdk_bdev_get_block_size(dev->ns[i].bdev)); + nsdata->noiob = spdk_bdev_get_optimal_io_boundary(dev->ns[i].bdev); + dev->ns[i].block_size = spdk_bdev_get_block_size(dev->ns[i].bdev); + dev->ns[i].capacity = num_blocks * dev->ns[i].block_size; + } else { + memset(nsdata, 0, sizeof(*nsdata)); + } + } + return 0; +} + +static int +vhost_nvme_ctrlr_identify_update(struct spdk_vhost_nvme_dev *dev) +{ + struct spdk_nvme_ctrlr_data *cdata = &dev->cdata; + char sn[20]; + + /* Controller Capabilities */ + dev->cap.bits.cqr = 1; + dev->cap.bits.to = 1; + dev->cap.bits.dstrd = 0; + dev->cap.bits.css = SPDK_NVME_CAP_CSS_NVM; + dev->cap.bits.mpsmin = 0; + dev->cap.bits.mpsmax = 0; + /* MQES is 0 based value */ + dev->cap.bits.mqes = MAX_QUEUE_ENTRIES_SUPPORTED - 1; + + /* Controller Configuration */ + dev->cc.bits.en = 0; + + /* Controller Status */ + dev->csts.bits.rdy = 0; + + /* Identify Controller */ + spdk_strcpy_pad(cdata->fr, FW_VERSION, sizeof(cdata->fr), ' '); + cdata->vid = 0x8086; + cdata->ssvid = 0x8086; + spdk_strcpy_pad(cdata->mn, "SPDK Virtual NVMe Controller", sizeof(cdata->mn), ' '); + snprintf(sn, sizeof(sn), "NVMe_%s", dev->vdev.name); + spdk_strcpy_pad(cdata->sn, sn, sizeof(cdata->sn), ' '); + cdata->ieee[0] = 0xe4; + cdata->ieee[1] = 0xd2; + cdata->ieee[2] = 0x5c; + cdata->ver.bits.mjr = 1; + cdata->ver.bits.mnr = 0; + cdata->mdts = 5; /* 128 KiB */ + cdata->rab = 6; + cdata->sqes.min = 6; + cdata->sqes.max = 6; + cdata->cqes.min = 4; + cdata->cqes.max = 4; + cdata->oncs.dsm = 1; + /* Emulated NVMe controller */ + cdata->oacs.doorbell_buffer_config = 1; + + vhost_nvme_ns_identify_update(dev); + + return 0; +} + +int +vhost_nvme_dev_construct(const char *name, const char *cpumask, uint32_t num_io_queues) +{ + struct spdk_vhost_nvme_dev *dev; + int rc; + + if (posix_memalign((void **)&dev, SPDK_CACHE_LINE_SIZE, sizeof(*dev))) { + return -ENOMEM; + } + memset(dev, 0, sizeof(*dev)); + + if (num_io_queues < 1 || num_io_queues > MAX_IO_QUEUES) { + free(dev); + return -EINVAL; + } + + spdk_vhost_lock(); + rc = vhost_dev_register(&dev->vdev, name, cpumask, + &spdk_vhost_nvme_device_backend); + + if (rc) { + free(dev); + spdk_vhost_unlock(); + return rc; + } + + dev->num_io_queues = num_io_queues; + STAILQ_INIT(&dev->free_tasks); + TAILQ_INSERT_TAIL(&g_nvme_ctrlrs, dev, tailq); + + vhost_nvme_ctrlr_identify_update(dev); + + SPDK_NOTICELOG("Controller %s: Constructed\n", name); + spdk_vhost_unlock(); + return rc; +} + +int +vhost_nvme_dev_remove(struct spdk_vhost_dev *vdev) +{ + struct spdk_vhost_nvme_dev *nvme = to_nvme_dev(vdev); + struct spdk_vhost_nvme_ns *ns; + int rc; + uint32_t i; + + if (nvme == NULL) { + return -EINVAL; + } + + TAILQ_REMOVE(&g_nvme_ctrlrs, nvme, tailq); + for (i = 0; i < nvme->num_ns; i++) { + ns = &nvme->ns[i]; + if (ns->active_ns) { + vhost_nvme_deactive_ns(ns); + } + } + + rc = vhost_dev_unregister(vdev); + if (rc != 0) { + return rc; + } + + free(nvme); + return 0; +} + +int +vhost_nvme_dev_add_ns(struct spdk_vhost_dev *vdev, const char *bdev_name) +{ + struct spdk_vhost_nvme_dev *nvme = to_nvme_dev(vdev); + struct spdk_vhost_nvme_ns *ns; + struct spdk_bdev *bdev; + int rc = -1; + + if (nvme == NULL) { + return -ENODEV; + } + + if (nvme->num_ns == MAX_NAMESPACE) { + SPDK_ERRLOG("Can't support %d Namespaces\n", nvme->num_ns); + return -ENOSPC; + } + + bdev = spdk_bdev_get_by_name(bdev_name); + if (!bdev) { + SPDK_ERRLOG("could not find bdev %s\n", bdev_name); + return -ENODEV; + } + + ns = &nvme->ns[nvme->num_ns]; + rc = spdk_bdev_open(bdev, true, bdev_remove_cb, ns, &nvme->ns[nvme->num_ns].bdev_desc); + if (rc != 0) { + SPDK_ERRLOG("Could not open bdev '%s', error=%d\n", + bdev_name, rc); + return rc; + } + + nvme->ns[nvme->num_ns].bdev = bdev; + nvme->ns[nvme->num_ns].active_ns = 1; + nvme->ns[nvme->num_ns].nsid = nvme->num_ns + 1; + nvme->num_ns++; + + vhost_nvme_ns_identify_update(nvme); + + return rc; +} + +int +vhost_nvme_controller_construct(void) +{ + struct spdk_conf_section *sp; + const char *name; + const char *bdev_name; + const char *cpumask; + int rc, i = 0; + struct spdk_vhost_dev *vdev; + uint32_t ctrlr_num, io_queues; + + for (sp = spdk_conf_first_section(NULL); sp != NULL; sp = spdk_conf_next_section(sp)) { + if (!spdk_conf_section_match_prefix(sp, "VhostNvme")) { + continue; + } + + if (sscanf(spdk_conf_section_get_name(sp), "VhostNvme%u", &ctrlr_num) != 1) { + SPDK_ERRLOG("Section '%s' has non-numeric suffix.\n", + spdk_conf_section_get_name(sp)); + return -1; + } + + name = spdk_conf_section_get_val(sp, "Name"); + if (name == NULL) { + SPDK_ERRLOG("VhostNvme%u: missing Name\n", ctrlr_num); + return -1; + } + + cpumask = spdk_conf_section_get_val(sp, "Cpumask"); + rc = spdk_conf_section_get_intval(sp, "NumberOfQueues"); + if (rc > 0) { + io_queues = rc; + } else { + io_queues = 1; + } + + rc = vhost_nvme_dev_construct(name, cpumask, io_queues); + if (rc < 0) { + SPDK_ERRLOG("VhostNvme%u: Construct failed\n", ctrlr_num); + return -1; + } + + vdev = spdk_vhost_dev_find(name); + if (!vdev) { + return -1; + } + + for (i = 0; spdk_conf_section_get_nval(sp, "Namespace", i) != NULL; i++) { + bdev_name = spdk_conf_section_get_nmval(sp, "Namespace", i, 0); + if (!bdev_name) { + SPDK_ERRLOG("namespace configuration missing bdev name\n"); + break; + } + rc = vhost_nvme_dev_add_ns(vdev, bdev_name); + if (rc < 0) { + SPDK_WARNLOG("VhostNvme%u: Construct Namespace with %s failed\n", + ctrlr_num, bdev_name); + break; + } + } + } + + return 0; +} + +SPDK_LOG_REGISTER_COMPONENT("vhost_nvme", SPDK_LOG_VHOST_NVME) diff --git a/src/spdk/lib/vhost/vhost_rpc.c b/src/spdk/lib/vhost/vhost_rpc.c new file mode 100644 index 000000000..196d75918 --- /dev/null +++ b/src/spdk/lib/vhost/vhost_rpc.c @@ -0,0 +1,652 @@ +/*- + * BSD LICENSE + * + * Copyright(c) Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "spdk/stdinc.h" + +#include "spdk_internal/log.h" +#include "spdk/rpc.h" +#include "spdk/util.h" +#include "spdk/string.h" +#include "spdk/env.h" + +#include "spdk/scsi.h" +#include "spdk/vhost.h" +#include "vhost_internal.h" +#include "spdk/bdev.h" + +struct rpc_vhost_scsi_ctrlr { + char *ctrlr; + char *cpumask; +}; + +static void +free_rpc_vhost_scsi_ctrlr(struct rpc_vhost_scsi_ctrlr *req) +{ + free(req->ctrlr); + free(req->cpumask); +} + +static const struct spdk_json_object_decoder rpc_vhost_create_scsi_ctrlr[] = { + {"ctrlr", offsetof(struct rpc_vhost_scsi_ctrlr, ctrlr), spdk_json_decode_string }, + {"cpumask", offsetof(struct rpc_vhost_scsi_ctrlr, cpumask), spdk_json_decode_string, true}, +}; + +static void +rpc_vhost_create_scsi_controller(struct spdk_jsonrpc_request *request, + const struct spdk_json_val *params) +{ + struct rpc_vhost_scsi_ctrlr req = {0}; + struct spdk_json_write_ctx *w; + int rc; + + if (spdk_json_decode_object(params, rpc_vhost_create_scsi_ctrlr, + SPDK_COUNTOF(rpc_vhost_create_scsi_ctrlr), + &req)) { + SPDK_DEBUGLOG(SPDK_LOG_VHOST_RPC, "spdk_json_decode_object failed\n"); + rc = -EINVAL; + goto invalid; + } + + rc = spdk_vhost_scsi_dev_construct(req.ctrlr, req.cpumask); + if (rc < 0) { + goto invalid; + } + + free_rpc_vhost_scsi_ctrlr(&req); + + w = spdk_jsonrpc_begin_result(request); + spdk_json_write_bool(w, true); + spdk_jsonrpc_end_result(request, w); + return; + +invalid: + free_rpc_vhost_scsi_ctrlr(&req); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + spdk_strerror(-rc)); +} +SPDK_RPC_REGISTER("vhost_create_scsi_controller", rpc_vhost_create_scsi_controller, + SPDK_RPC_RUNTIME) +SPDK_RPC_REGISTER_ALIAS_DEPRECATED(vhost_create_scsi_controller, construct_vhost_scsi_controller) + +struct rpc_vhost_scsi_ctrlr_add_target { + char *ctrlr; + int32_t scsi_target_num; + char *bdev_name; +}; + +static void +free_rpc_vhost_scsi_ctrlr_add_target(struct rpc_vhost_scsi_ctrlr_add_target *req) +{ + free(req->ctrlr); + free(req->bdev_name); +} + +static const struct spdk_json_object_decoder rpc_vhost_scsi_ctrlr_add_target[] = { + {"ctrlr", offsetof(struct rpc_vhost_scsi_ctrlr_add_target, ctrlr), spdk_json_decode_string }, + {"scsi_target_num", offsetof(struct rpc_vhost_scsi_ctrlr_add_target, scsi_target_num), spdk_json_decode_int32}, + {"bdev_name", offsetof(struct rpc_vhost_scsi_ctrlr_add_target, bdev_name), spdk_json_decode_string }, +}; + +static void +rpc_vhost_scsi_controller_add_target(struct spdk_jsonrpc_request *request, + const struct spdk_json_val *params) +{ + struct rpc_vhost_scsi_ctrlr_add_target req = {0}; + struct spdk_json_write_ctx *w; + struct spdk_vhost_dev *vdev; + int rc; + + if (spdk_json_decode_object(params, rpc_vhost_scsi_ctrlr_add_target, + SPDK_COUNTOF(rpc_vhost_scsi_ctrlr_add_target), + &req)) { + SPDK_DEBUGLOG(SPDK_LOG_VHOST_RPC, "spdk_json_decode_object failed\n"); + rc = -EINVAL; + goto invalid; + } + + spdk_vhost_lock(); + vdev = spdk_vhost_dev_find(req.ctrlr); + if (vdev == NULL) { + spdk_vhost_unlock(); + rc = -ENODEV; + goto invalid; + } + + rc = spdk_vhost_scsi_dev_add_tgt(vdev, req.scsi_target_num, req.bdev_name); + spdk_vhost_unlock(); + if (rc < 0) { + goto invalid; + } + + free_rpc_vhost_scsi_ctrlr_add_target(&req); + + w = spdk_jsonrpc_begin_result(request); + spdk_json_write_int32(w, rc); + spdk_jsonrpc_end_result(request, w); + return; + +invalid: + free_rpc_vhost_scsi_ctrlr_add_target(&req); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + spdk_strerror(-rc)); +} +SPDK_RPC_REGISTER("vhost_scsi_controller_add_target", rpc_vhost_scsi_controller_add_target, + SPDK_RPC_RUNTIME) +SPDK_RPC_REGISTER_ALIAS_DEPRECATED(vhost_scsi_controller_add_target, add_vhost_scsi_lun) + +struct rpc_remove_vhost_scsi_ctrlr_target { + char *ctrlr; + uint32_t scsi_target_num; +}; + +static void +free_rpc_remove_vhost_scsi_ctrlr_target(struct rpc_remove_vhost_scsi_ctrlr_target *req) +{ + free(req->ctrlr); +} + +static const struct spdk_json_object_decoder rpc_vhost_remove_target[] = { + {"ctrlr", offsetof(struct rpc_remove_vhost_scsi_ctrlr_target, ctrlr), spdk_json_decode_string }, + {"scsi_target_num", offsetof(struct rpc_remove_vhost_scsi_ctrlr_target, scsi_target_num), spdk_json_decode_uint32}, +}; + +static int +rpc_vhost_scsi_controller_remove_target_finish_cb(struct spdk_vhost_dev *vdev, void *arg) +{ + struct spdk_jsonrpc_request *request = arg; + struct spdk_json_write_ctx *w; + + w = spdk_jsonrpc_begin_result(request); + spdk_json_write_bool(w, true); + spdk_jsonrpc_end_result(request, w); + return 0; +} + +static void +rpc_vhost_scsi_controller_remove_target(struct spdk_jsonrpc_request *request, + const struct spdk_json_val *params) +{ + struct rpc_remove_vhost_scsi_ctrlr_target req = {0}; + struct spdk_vhost_dev *vdev; + int rc; + + if (spdk_json_decode_object(params, rpc_vhost_remove_target, + SPDK_COUNTOF(rpc_vhost_remove_target), + &req)) { + SPDK_DEBUGLOG(SPDK_LOG_VHOST_RPC, "spdk_json_decode_object failed\n"); + rc = -EINVAL; + goto invalid; + } + + spdk_vhost_lock(); + vdev = spdk_vhost_dev_find(req.ctrlr); + if (vdev == NULL) { + spdk_vhost_unlock(); + rc = -ENODEV; + goto invalid; + } + + rc = spdk_vhost_scsi_dev_remove_tgt(vdev, req.scsi_target_num, + rpc_vhost_scsi_controller_remove_target_finish_cb, + request); + spdk_vhost_unlock(); + if (rc < 0) { + goto invalid; + } + + free_rpc_remove_vhost_scsi_ctrlr_target(&req); + return; + +invalid: + free_rpc_remove_vhost_scsi_ctrlr_target(&req); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + spdk_strerror(-rc)); +} + +SPDK_RPC_REGISTER("vhost_scsi_controller_remove_target", + rpc_vhost_scsi_controller_remove_target, SPDK_RPC_RUNTIME) +SPDK_RPC_REGISTER_ALIAS_DEPRECATED(vhost_scsi_controller_remove_target, remove_vhost_scsi_target) + +struct rpc_vhost_blk_ctrlr { + char *ctrlr; + char *dev_name; + char *cpumask; + bool readonly; + bool packed_ring; +}; + +static const struct spdk_json_object_decoder rpc_construct_vhost_blk_ctrlr[] = { + {"ctrlr", offsetof(struct rpc_vhost_blk_ctrlr, ctrlr), spdk_json_decode_string }, + {"dev_name", offsetof(struct rpc_vhost_blk_ctrlr, dev_name), spdk_json_decode_string }, + {"cpumask", offsetof(struct rpc_vhost_blk_ctrlr, cpumask), spdk_json_decode_string, true}, + {"readonly", offsetof(struct rpc_vhost_blk_ctrlr, readonly), spdk_json_decode_bool, true}, + {"packed_ring", offsetof(struct rpc_vhost_blk_ctrlr, packed_ring), spdk_json_decode_bool, true}, +}; + +static void +free_rpc_vhost_blk_ctrlr(struct rpc_vhost_blk_ctrlr *req) +{ + free(req->ctrlr); + free(req->dev_name); + free(req->cpumask); +} + +static void +rpc_vhost_create_blk_controller(struct spdk_jsonrpc_request *request, + const struct spdk_json_val *params) +{ + struct rpc_vhost_blk_ctrlr req = {0}; + struct spdk_json_write_ctx *w; + int rc; + + if (spdk_json_decode_object(params, rpc_construct_vhost_blk_ctrlr, + SPDK_COUNTOF(rpc_construct_vhost_blk_ctrlr), + &req)) { + SPDK_DEBUGLOG(SPDK_LOG_VHOST_RPC, "spdk_json_decode_object failed\n"); + rc = -EINVAL; + goto invalid; + } + + rc = spdk_vhost_blk_construct(req.ctrlr, req.cpumask, req.dev_name, + req.readonly, req.packed_ring); + if (rc < 0) { + goto invalid; + } + + free_rpc_vhost_blk_ctrlr(&req); + + w = spdk_jsonrpc_begin_result(request); + spdk_json_write_bool(w, true); + spdk_jsonrpc_end_result(request, w); + return; + +invalid: + free_rpc_vhost_blk_ctrlr(&req); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + spdk_strerror(-rc)); + +} +SPDK_RPC_REGISTER("vhost_create_blk_controller", rpc_vhost_create_blk_controller, + SPDK_RPC_RUNTIME) +SPDK_RPC_REGISTER_ALIAS_DEPRECATED(vhost_create_blk_controller, construct_vhost_blk_controller) + +struct rpc_delete_vhost_ctrlr { + char *ctrlr; +}; + +static const struct spdk_json_object_decoder rpc_delete_vhost_ctrlr_decoder[] = { + {"ctrlr", offsetof(struct rpc_delete_vhost_ctrlr, ctrlr), spdk_json_decode_string }, +}; + +static void +free_rpc_delete_vhost_ctrlr(struct rpc_delete_vhost_ctrlr *req) +{ + free(req->ctrlr); +} + +static void +rpc_vhost_delete_controller(struct spdk_jsonrpc_request *request, + const struct spdk_json_val *params) +{ + struct rpc_delete_vhost_ctrlr req = {0}; + struct spdk_json_write_ctx *w; + struct spdk_vhost_dev *vdev; + int rc; + + if (spdk_json_decode_object(params, rpc_delete_vhost_ctrlr_decoder, + SPDK_COUNTOF(rpc_delete_vhost_ctrlr_decoder), &req)) { + SPDK_DEBUGLOG(SPDK_LOG_VHOST_RPC, "spdk_json_decode_object failed\n"); + rc = -EINVAL; + goto invalid; + } + + spdk_vhost_lock(); + vdev = spdk_vhost_dev_find(req.ctrlr); + if (vdev == NULL) { + spdk_vhost_unlock(); + rc = -ENODEV; + goto invalid; + } + + rc = spdk_vhost_dev_remove(vdev); + spdk_vhost_unlock(); + if (rc < 0) { + goto invalid; + } + + free_rpc_delete_vhost_ctrlr(&req); + + w = spdk_jsonrpc_begin_result(request); + spdk_json_write_bool(w, true); + spdk_jsonrpc_end_result(request, w); + + return; + +invalid: + free_rpc_delete_vhost_ctrlr(&req); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + spdk_strerror(-rc)); + +} +SPDK_RPC_REGISTER("vhost_delete_controller", rpc_vhost_delete_controller, SPDK_RPC_RUNTIME) +SPDK_RPC_REGISTER_ALIAS_DEPRECATED(vhost_delete_controller, remove_vhost_controller) + +struct rpc_get_vhost_ctrlrs { + char *name; +}; + +static void +_rpc_get_vhost_controller(struct spdk_json_write_ctx *w, struct spdk_vhost_dev *vdev) +{ + uint32_t delay_base_us, iops_threshold; + + spdk_vhost_get_coalescing(vdev, &delay_base_us, &iops_threshold); + + spdk_json_write_object_begin(w); + + spdk_json_write_named_string(w, "ctrlr", spdk_vhost_dev_get_name(vdev)); + spdk_json_write_named_string_fmt(w, "cpumask", "0x%s", + spdk_cpuset_fmt(spdk_thread_get_cpumask(vdev->thread))); + spdk_json_write_named_uint32(w, "delay_base_us", delay_base_us); + spdk_json_write_named_uint32(w, "iops_threshold", iops_threshold); + spdk_json_write_named_string(w, "socket", vdev->path); + + spdk_json_write_named_object_begin(w, "backend_specific"); + vhost_dump_info_json(vdev, w); + spdk_json_write_object_end(w); + + spdk_json_write_object_end(w); +} + +static const struct spdk_json_object_decoder rpc_get_vhost_ctrlrs_decoders[] = { + {"name", offsetof(struct rpc_get_vhost_ctrlrs, name), spdk_json_decode_string, true}, +}; + +static void +free_rpc_get_vhost_ctrlrs(struct rpc_get_vhost_ctrlrs *req) +{ + free(req->name); +} + +static void +rpc_vhost_get_controllers(struct spdk_jsonrpc_request *request, + const struct spdk_json_val *params) +{ + struct rpc_get_vhost_ctrlrs req = {0}; + struct spdk_json_write_ctx *w; + struct spdk_vhost_dev *vdev; + int rc; + + if (params && spdk_json_decode_object(params, rpc_get_vhost_ctrlrs_decoders, + SPDK_COUNTOF(rpc_get_vhost_ctrlrs_decoders), &req)) { + SPDK_ERRLOG("spdk_json_decode_object failed\n"); + rc = -EINVAL; + goto invalid; + } + + spdk_vhost_lock(); + if (req.name != NULL) { + vdev = spdk_vhost_dev_find(req.name); + if (vdev == NULL) { + spdk_vhost_unlock(); + rc = -ENODEV; + goto invalid; + } + + free_rpc_get_vhost_ctrlrs(&req); + + w = spdk_jsonrpc_begin_result(request); + spdk_json_write_array_begin(w); + + _rpc_get_vhost_controller(w, vdev); + spdk_vhost_unlock(); + + spdk_json_write_array_end(w); + spdk_jsonrpc_end_result(request, w); + return; + } + + free_rpc_get_vhost_ctrlrs(&req); + + w = spdk_jsonrpc_begin_result(request); + spdk_json_write_array_begin(w); + + vdev = spdk_vhost_dev_next(NULL); + while (vdev != NULL) { + _rpc_get_vhost_controller(w, vdev); + vdev = spdk_vhost_dev_next(vdev); + } + spdk_vhost_unlock(); + + spdk_json_write_array_end(w); + spdk_jsonrpc_end_result(request, w); + return; + +invalid: + free_rpc_get_vhost_ctrlrs(&req); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, + spdk_strerror(-rc)); +} +SPDK_RPC_REGISTER("vhost_get_controllers", rpc_vhost_get_controllers, SPDK_RPC_RUNTIME) +SPDK_RPC_REGISTER_ALIAS_DEPRECATED(vhost_get_controllers, get_vhost_controllers) + + +struct rpc_vhost_ctrlr_coalescing { + char *ctrlr; + uint32_t delay_base_us; + uint32_t iops_threshold; +}; + +static const struct spdk_json_object_decoder rpc_set_vhost_ctrlr_coalescing[] = { + {"ctrlr", offsetof(struct rpc_vhost_ctrlr_coalescing, ctrlr), spdk_json_decode_string }, + {"delay_base_us", offsetof(struct rpc_vhost_ctrlr_coalescing, delay_base_us), spdk_json_decode_uint32}, + {"iops_threshold", offsetof(struct rpc_vhost_ctrlr_coalescing, iops_threshold), spdk_json_decode_uint32}, +}; + +static void +free_rpc_set_vhost_controllers_event_coalescing(struct rpc_vhost_ctrlr_coalescing *req) +{ + free(req->ctrlr); +} + +static void +rpc_vhost_controller_set_coalescing(struct spdk_jsonrpc_request *request, + const struct spdk_json_val *params) +{ + struct rpc_vhost_ctrlr_coalescing req = {0}; + struct spdk_json_write_ctx *w; + struct spdk_vhost_dev *vdev; + int rc; + + if (spdk_json_decode_object(params, rpc_set_vhost_ctrlr_coalescing, + SPDK_COUNTOF(rpc_set_vhost_ctrlr_coalescing), &req)) { + SPDK_DEBUGLOG(SPDK_LOG_VHOST_RPC, "spdk_json_decode_object failed\n"); + rc = -EINVAL; + goto invalid; + } + + spdk_vhost_lock(); + vdev = spdk_vhost_dev_find(req.ctrlr); + if (vdev == NULL) { + spdk_vhost_unlock(); + rc = -ENODEV; + goto invalid; + } + + rc = spdk_vhost_set_coalescing(vdev, req.delay_base_us, req.iops_threshold); + spdk_vhost_unlock(); + if (rc) { + goto invalid; + } + + free_rpc_set_vhost_controllers_event_coalescing(&req); + + w = spdk_jsonrpc_begin_result(request); + spdk_json_write_bool(w, true); + spdk_jsonrpc_end_result(request, w); + + return; + +invalid: + free_rpc_set_vhost_controllers_event_coalescing(&req); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + spdk_strerror(-rc)); +} +SPDK_RPC_REGISTER("vhost_controller_set_coalescing", rpc_vhost_controller_set_coalescing, + SPDK_RPC_RUNTIME) +SPDK_RPC_REGISTER_ALIAS_DEPRECATED(vhost_controller_set_coalescing, set_vhost_controller_coalescing) + +#ifdef SPDK_CONFIG_VHOST_INTERNAL_LIB + +struct rpc_vhost_nvme_ctrlr { + char *ctrlr; + uint32_t io_queues; + char *cpumask; +}; + +static const struct spdk_json_object_decoder rpc_construct_vhost_nvme_ctrlr[] = { + {"ctrlr", offsetof(struct rpc_vhost_nvme_ctrlr, ctrlr), spdk_json_decode_string }, + {"io_queues", offsetof(struct rpc_vhost_nvme_ctrlr, io_queues), spdk_json_decode_uint32}, + {"cpumask", offsetof(struct rpc_vhost_nvme_ctrlr, cpumask), spdk_json_decode_string, true}, +}; + +static void +free_rpc_vhost_nvme_ctrlr(struct rpc_vhost_nvme_ctrlr *req) +{ + free(req->ctrlr); + free(req->cpumask); +} + +static void +rpc_vhost_create_nvme_controller(struct spdk_jsonrpc_request *request, + const struct spdk_json_val *params) +{ + struct rpc_vhost_nvme_ctrlr req = {}; + struct spdk_json_write_ctx *w; + int rc; + + if (spdk_json_decode_object(params, rpc_construct_vhost_nvme_ctrlr, + SPDK_COUNTOF(rpc_construct_vhost_nvme_ctrlr), + &req)) { + rc = -EINVAL; + goto invalid; + } + + rc = vhost_nvme_dev_construct(req.ctrlr, req.cpumask, req.io_queues); + if (rc < 0) { + goto invalid; + } + + free_rpc_vhost_nvme_ctrlr(&req); + + w = spdk_jsonrpc_begin_result(request); + spdk_json_write_bool(w, true); + spdk_jsonrpc_end_result(request, w); + return; + +invalid: + free_rpc_vhost_nvme_ctrlr(&req); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + spdk_strerror(-rc)); + +} +SPDK_RPC_REGISTER("vhost_create_nvme_controller", rpc_vhost_create_nvme_controller, + SPDK_RPC_RUNTIME) +SPDK_RPC_REGISTER_ALIAS_DEPRECATED(vhost_create_nvme_controller, construct_vhost_nvme_controller) + +struct rpc_vhost_nvme_ctrlr_add_ns { + char *ctrlr; + char *bdev_name; +}; + +static void +free_rpc_vhost_nvme_ctrlr_add_ns(struct rpc_vhost_nvme_ctrlr_add_ns *req) +{ + free(req->ctrlr); + free(req->bdev_name); +} + +static const struct spdk_json_object_decoder rpc_vhost_nvme_add_ns[] = { + {"ctrlr", offsetof(struct rpc_vhost_nvme_ctrlr_add_ns, ctrlr), spdk_json_decode_string }, + {"bdev_name", offsetof(struct rpc_vhost_nvme_ctrlr_add_ns, bdev_name), spdk_json_decode_string }, +}; + +static void +rpc_vhost_nvme_controller_add_ns(struct spdk_jsonrpc_request *request, + const struct spdk_json_val *params) +{ + struct rpc_vhost_nvme_ctrlr_add_ns req = {0}; + struct spdk_json_write_ctx *w; + struct spdk_vhost_dev *vdev; + int rc; + + if (spdk_json_decode_object(params, rpc_vhost_nvme_add_ns, + SPDK_COUNTOF(rpc_vhost_nvme_add_ns), + &req)) { + SPDK_DEBUGLOG(SPDK_LOG_VHOST_RPC, "spdk_json_decode_object failed\n"); + rc = -EINVAL; + goto invalid; + } + + spdk_vhost_lock(); + vdev = spdk_vhost_dev_find(req.ctrlr); + if (vdev == NULL) { + spdk_vhost_unlock(); + rc = -ENODEV; + goto invalid; + } + + rc = vhost_nvme_dev_add_ns(vdev, req.bdev_name); + spdk_vhost_unlock(); + if (rc < 0) { + goto invalid; + } + free_rpc_vhost_nvme_ctrlr_add_ns(&req); + + w = spdk_jsonrpc_begin_result(request); + spdk_json_write_bool(w, true); + spdk_jsonrpc_end_result(request, w); + return; + +invalid: + free_rpc_vhost_nvme_ctrlr_add_ns(&req); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + spdk_strerror(-rc)); +} +SPDK_RPC_REGISTER("vhost_nvme_controller_add_ns", rpc_vhost_nvme_controller_add_ns, + SPDK_RPC_RUNTIME) +SPDK_RPC_REGISTER_ALIAS_DEPRECATED(vhost_nvme_controller_add_ns, add_vhost_nvme_ns) + +#endif /* SPDK_CONFIG_VHOST_INTERNAL_LIB */ + +SPDK_LOG_REGISTER_COMPONENT("vhost_rpc", SPDK_LOG_VHOST_RPC) diff --git a/src/spdk/lib/vhost/vhost_scsi.c b/src/spdk/lib/vhost/vhost_scsi.c new file mode 100644 index 000000000..49e49dc76 --- /dev/null +++ b/src/spdk/lib/vhost/vhost_scsi.c @@ -0,0 +1,1536 @@ +/*- + * BSD LICENSE + * + * Copyright(c) Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "spdk/stdinc.h" + +#include <linux/virtio_scsi.h> + +#include "spdk/env.h" +#include "spdk/thread.h" +#include "spdk/scsi.h" +#include "spdk/scsi_spec.h" +#include "spdk/conf.h" +#include "spdk/util.h" +#include "spdk/likely.h" + +#include "spdk/vhost.h" +#include "vhost_internal.h" + +/* Features supported by SPDK VHOST lib. */ +#define SPDK_VHOST_SCSI_FEATURES (SPDK_VHOST_FEATURES | \ + (1ULL << VIRTIO_SCSI_F_INOUT) | \ + (1ULL << VIRTIO_SCSI_F_HOTPLUG) | \ + (1ULL << VIRTIO_SCSI_F_CHANGE ) | \ + (1ULL << VIRTIO_SCSI_F_T10_PI )) + +/* Features that are specified in VIRTIO SCSI but currently not supported: + * - Live migration not supported yet + * - T10 PI + */ +#define SPDK_VHOST_SCSI_DISABLED_FEATURES (SPDK_VHOST_DISABLED_FEATURES | \ + (1ULL << VIRTIO_SCSI_F_T10_PI )) + +#define MGMT_POLL_PERIOD_US (1000 * 5) + +#define VIRTIO_SCSI_CONTROLQ 0 +#define VIRTIO_SCSI_EVENTQ 1 +#define VIRTIO_SCSI_REQUESTQ 2 + +enum spdk_scsi_dev_vhost_status { + /* Target ID is empty. */ + VHOST_SCSI_DEV_EMPTY, + + /* Target is still being added. */ + VHOST_SCSI_DEV_ADDING, + + /* Target ID occupied. */ + VHOST_SCSI_DEV_PRESENT, + + /* Target ID is occupied but removal is in progress. */ + VHOST_SCSI_DEV_REMOVING, + + /* In session - device (SCSI target) seen but removed. */ + VHOST_SCSI_DEV_REMOVED, +}; + +/** Context for a SCSI target in a vhost device */ +struct spdk_scsi_dev_vhost_state { + struct spdk_scsi_dev *dev; + enum spdk_scsi_dev_vhost_status status; + spdk_vhost_event_fn remove_cb; + void *remove_ctx; +}; + +struct spdk_vhost_scsi_dev { + int ref; + bool registered; + struct spdk_vhost_dev vdev; + struct spdk_scsi_dev_vhost_state scsi_dev_state[SPDK_VHOST_SCSI_CTRLR_MAX_DEVS]; +}; + +/** Context for a SCSI target in a vhost session */ +struct spdk_scsi_dev_session_state { + struct spdk_scsi_dev *dev; + enum spdk_scsi_dev_vhost_status status; +}; + +struct spdk_vhost_scsi_session { + struct spdk_vhost_session vsession; + + struct spdk_vhost_scsi_dev *svdev; + /** Local copy of the device state */ + struct spdk_scsi_dev_session_state scsi_dev_state[SPDK_VHOST_SCSI_CTRLR_MAX_DEVS]; + struct spdk_poller *requestq_poller; + struct spdk_poller *mgmt_poller; + struct spdk_poller *stop_poller; +}; + +struct spdk_vhost_scsi_task { + struct spdk_scsi_task scsi; + struct iovec iovs[SPDK_VHOST_IOVS_MAX]; + + union { + struct virtio_scsi_cmd_resp *resp; + struct virtio_scsi_ctrl_tmf_resp *tmf_resp; + }; + + struct spdk_vhost_scsi_session *svsession; + struct spdk_scsi_dev *scsi_dev; + + /** Number of bytes that were written. */ + uint32_t used_len; + + int req_idx; + + /* If set, the task is currently used for I/O processing. */ + bool used; + + struct spdk_vhost_virtqueue *vq; +}; + +static int vhost_scsi_start(struct spdk_vhost_session *vsession); +static int vhost_scsi_stop(struct spdk_vhost_session *vsession); +static void vhost_scsi_dump_info_json(struct spdk_vhost_dev *vdev, + struct spdk_json_write_ctx *w); +static void vhost_scsi_write_config_json(struct spdk_vhost_dev *vdev, + struct spdk_json_write_ctx *w); +static int vhost_scsi_dev_remove(struct spdk_vhost_dev *vdev); + +static const struct spdk_vhost_dev_backend spdk_vhost_scsi_device_backend = { + .session_ctx_size = sizeof(struct spdk_vhost_scsi_session) - sizeof(struct spdk_vhost_session), + .start_session = vhost_scsi_start, + .stop_session = vhost_scsi_stop, + .dump_info_json = vhost_scsi_dump_info_json, + .write_config_json = vhost_scsi_write_config_json, + .remove_device = vhost_scsi_dev_remove, +}; + +static inline void +scsi_task_init(struct spdk_vhost_scsi_task *task) +{ + memset(&task->scsi, 0, sizeof(task->scsi)); + /* Tmf_resp pointer and resp pointer are in a union. + * Here means task->tmf_resp = task->resp = NULL. + */ + task->resp = NULL; + task->used = true; + task->used_len = 0; +} + +static void +vhost_scsi_task_put(struct spdk_vhost_scsi_task *task) +{ + spdk_scsi_task_put(&task->scsi); +} + +static void +vhost_scsi_task_free_cb(struct spdk_scsi_task *scsi_task) +{ + struct spdk_vhost_scsi_task *task = SPDK_CONTAINEROF(scsi_task, struct spdk_vhost_scsi_task, scsi); + struct spdk_vhost_session *vsession = &task->svsession->vsession; + + assert(vsession->task_cnt > 0); + vsession->task_cnt--; + task->used = false; +} + +static void +remove_scsi_tgt(struct spdk_vhost_scsi_dev *svdev, + unsigned scsi_tgt_num) +{ + struct spdk_scsi_dev_vhost_state *state; + struct spdk_scsi_dev *dev; + + state = &svdev->scsi_dev_state[scsi_tgt_num]; + dev = state->dev; + state->dev = NULL; + assert(state->status == VHOST_SCSI_DEV_REMOVING); + state->status = VHOST_SCSI_DEV_EMPTY; + spdk_scsi_dev_destruct(dev, NULL, NULL); + if (state->remove_cb) { + state->remove_cb(&svdev->vdev, state->remove_ctx); + state->remove_cb = NULL; + } + SPDK_INFOLOG(SPDK_LOG_VHOST, "%s: removed target 'Target %u'\n", + svdev->vdev.name, scsi_tgt_num); + + if (--svdev->ref == 0 && svdev->registered == false) { + free(svdev); + } +} + +static void +vhost_scsi_dev_process_removed_cpl_cb(struct spdk_vhost_dev *vdev, void *ctx) +{ + unsigned scsi_tgt_num = (unsigned)(uintptr_t)ctx; + struct spdk_vhost_scsi_dev *svdev = SPDK_CONTAINEROF(vdev, + struct spdk_vhost_scsi_dev, vdev); + + /* all sessions have already detached the device */ + if (svdev->scsi_dev_state[scsi_tgt_num].status != VHOST_SCSI_DEV_REMOVING) { + /* device was already removed in the meantime */ + return; + } + + remove_scsi_tgt(svdev, scsi_tgt_num); +} + +static int +vhost_scsi_session_process_removed(struct spdk_vhost_dev *vdev, + struct spdk_vhost_session *vsession, void *ctx) +{ + unsigned scsi_tgt_num = (unsigned)(uintptr_t)ctx; + struct spdk_vhost_scsi_session *svsession = (struct spdk_vhost_scsi_session *)vsession; + struct spdk_scsi_dev_session_state *state = &svsession->scsi_dev_state[scsi_tgt_num]; + + if (state->dev != NULL) { + /* there's still a session that references this device, + * so abort our foreach chain here. We'll be called + * again from this session's management poller after it + * is removed in there + */ + return -1; + } + + return 0; +} + +static void +process_removed_devs(struct spdk_vhost_scsi_session *svsession) +{ + struct spdk_scsi_dev *dev; + struct spdk_scsi_dev_session_state *state; + int i; + + for (i = 0; i < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; ++i) { + state = &svsession->scsi_dev_state[i]; + dev = state->dev; + + if (dev && state->status == VHOST_SCSI_DEV_REMOVING && + !spdk_scsi_dev_has_pending_tasks(dev, NULL)) { + /* detach the device from this session */ + spdk_scsi_dev_free_io_channels(dev); + state->dev = NULL; + state->status = VHOST_SCSI_DEV_REMOVED; + /* try to detach it globally */ + spdk_vhost_lock(); + vhost_dev_foreach_session(&svsession->svdev->vdev, + vhost_scsi_session_process_removed, + vhost_scsi_dev_process_removed_cpl_cb, + (void *)(uintptr_t)i); + spdk_vhost_unlock(); + } + } +} + +static void +eventq_enqueue(struct spdk_vhost_scsi_session *svsession, unsigned scsi_dev_num, + uint32_t event, uint32_t reason) +{ + struct spdk_vhost_session *vsession = &svsession->vsession; + struct spdk_vhost_virtqueue *vq; + struct vring_desc *desc, *desc_table; + struct virtio_scsi_event *desc_ev; + uint32_t desc_table_size, req_size = 0; + uint16_t req; + int rc; + + assert(scsi_dev_num < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS); + vq = &vsession->virtqueue[VIRTIO_SCSI_EVENTQ]; + + if (vq->vring.desc == NULL || vhost_vq_avail_ring_get(vq, &req, 1) != 1) { + SPDK_ERRLOG("%s: failed to send virtio event (no avail ring entries?).\n", + vsession->name); + return; + } + + rc = vhost_vq_get_desc(vsession, vq, req, &desc, &desc_table, &desc_table_size); + if (rc != 0 || desc->len < sizeof(*desc_ev)) { + SPDK_ERRLOG("%s: invalid eventq descriptor at index %"PRIu16".\n", + vsession->name, req); + goto out; + } + + desc_ev = vhost_gpa_to_vva(vsession, desc->addr, sizeof(*desc_ev)); + if (desc_ev == NULL) { + SPDK_ERRLOG("%s: eventq descriptor at index %"PRIu16" points " + "to unmapped guest memory address %p.\n", + vsession->name, req, (void *)(uintptr_t)desc->addr); + goto out; + } + + desc_ev->event = event; + desc_ev->lun[0] = 1; + desc_ev->lun[1] = scsi_dev_num; + /* virtio LUN id 0 can refer either to the entire device + * or actual LUN 0 (the only supported by vhost for now) + */ + desc_ev->lun[2] = 0 >> 8; + desc_ev->lun[3] = 0 & 0xFF; + /* virtio doesn't specify any strict format for LUN id (bytes 2 and 3) + * current implementation relies on linux kernel sources + */ + memset(&desc_ev->lun[4], 0, 4); + desc_ev->reason = reason; + req_size = sizeof(*desc_ev); + +out: + vhost_vq_used_ring_enqueue(vsession, vq, req, req_size); +} + +static void +submit_completion(struct spdk_vhost_scsi_task *task) +{ + struct spdk_vhost_session *vsession = &task->svsession->vsession; + + vhost_vq_used_ring_enqueue(vsession, task->vq, task->req_idx, + task->used_len); + SPDK_DEBUGLOG(SPDK_LOG_VHOST_SCSI, "Finished task (%p) req_idx=%d\n", task, task->req_idx); + + vhost_scsi_task_put(task); +} + +static void +vhost_scsi_task_mgmt_cpl(struct spdk_scsi_task *scsi_task) +{ + struct spdk_vhost_scsi_task *task = SPDK_CONTAINEROF(scsi_task, struct spdk_vhost_scsi_task, scsi); + + submit_completion(task); +} + +static void +vhost_scsi_task_cpl(struct spdk_scsi_task *scsi_task) +{ + struct spdk_vhost_scsi_task *task = SPDK_CONTAINEROF(scsi_task, struct spdk_vhost_scsi_task, scsi); + + /* The SCSI task has completed. Do final processing and then post + notification to the virtqueue's "used" ring. + */ + task->resp->status = task->scsi.status; + + if (task->scsi.status != SPDK_SCSI_STATUS_GOOD) { + memcpy(task->resp->sense, task->scsi.sense_data, task->scsi.sense_data_len); + task->resp->sense_len = task->scsi.sense_data_len; + SPDK_DEBUGLOG(SPDK_LOG_VHOST_SCSI, "Task (%p) req_idx=%d failed - status=%u\n", task, task->req_idx, + task->scsi.status); + } + assert(task->scsi.transfer_len == task->scsi.length); + task->resp->resid = task->scsi.length - task->scsi.data_transferred; + + submit_completion(task); +} + +static void +task_submit(struct spdk_vhost_scsi_task *task) +{ + task->resp->response = VIRTIO_SCSI_S_OK; + spdk_scsi_dev_queue_task(task->scsi_dev, &task->scsi); +} + +static void +mgmt_task_submit(struct spdk_vhost_scsi_task *task, enum spdk_scsi_task_func func) +{ + task->tmf_resp->response = VIRTIO_SCSI_S_OK; + task->scsi.function = func; + spdk_scsi_dev_queue_mgmt_task(task->scsi_dev, &task->scsi); +} + +static void +invalid_request(struct spdk_vhost_scsi_task *task) +{ + struct spdk_vhost_session *vsession = &task->svsession->vsession; + + vhost_vq_used_ring_enqueue(vsession, task->vq, task->req_idx, + task->used_len); + vhost_scsi_task_put(task); + + SPDK_DEBUGLOG(SPDK_LOG_VHOST_SCSI, "Invalid request (status=%" PRIu8")\n", + task->resp ? task->resp->response : -1); +} + +static int +vhost_scsi_task_init_target(struct spdk_vhost_scsi_task *task, const __u8 *lun) +{ + struct spdk_vhost_scsi_session *svsession = task->svsession; + struct spdk_scsi_dev_session_state *state; + uint16_t lun_id = (((uint16_t)lun[2] << 8) | lun[3]) & 0x3FFF; + + SPDK_LOGDUMP(SPDK_LOG_VHOST_SCSI_QUEUE, "LUN", lun, 8); + + /* First byte must be 1 and second is target */ + if (lun[0] != 1 || lun[1] >= SPDK_VHOST_SCSI_CTRLR_MAX_DEVS) { + return -1; + } + + state = &svsession->scsi_dev_state[lun[1]]; + task->scsi_dev = state->dev; + if (state->dev == NULL || state->status != VHOST_SCSI_DEV_PRESENT) { + /* If dev has been hotdetached, return 0 to allow sending + * additional hotremove event via sense codes. + */ + return state->status != VHOST_SCSI_DEV_EMPTY ? 0 : -1; + } + + task->scsi.target_port = spdk_scsi_dev_find_port_by_id(task->scsi_dev, 0); + task->scsi.lun = spdk_scsi_dev_get_lun(state->dev, lun_id); + return 0; +} + +static void +process_ctrl_request(struct spdk_vhost_scsi_task *task) +{ + struct spdk_vhost_session *vsession = &task->svsession->vsession; + struct vring_desc *desc, *desc_table; + struct virtio_scsi_ctrl_tmf_req *ctrl_req; + struct virtio_scsi_ctrl_an_resp *an_resp; + uint32_t desc_table_size, used_len = 0; + int rc; + + spdk_scsi_task_construct(&task->scsi, vhost_scsi_task_mgmt_cpl, vhost_scsi_task_free_cb); + rc = vhost_vq_get_desc(vsession, task->vq, task->req_idx, &desc, &desc_table, + &desc_table_size); + if (spdk_unlikely(rc != 0)) { + SPDK_ERRLOG("%s: invalid controlq descriptor at index %d.\n", + vsession->name, task->req_idx); + goto out; + } + + ctrl_req = vhost_gpa_to_vva(vsession, desc->addr, sizeof(*ctrl_req)); + if (ctrl_req == NULL) { + SPDK_ERRLOG("%s: invalid task management request at index %d.\n", + vsession->name, task->req_idx); + goto out; + } + + SPDK_DEBUGLOG(SPDK_LOG_VHOST_SCSI_QUEUE, + "Processing controlq descriptor: desc %d/%p, desc_addr %p, len %d, flags %d, last_used_idx %d; kickfd %d; size %d\n", + task->req_idx, desc, (void *)desc->addr, desc->len, desc->flags, task->vq->last_used_idx, + task->vq->vring.kickfd, task->vq->vring.size); + SPDK_LOGDUMP(SPDK_LOG_VHOST_SCSI_QUEUE, "Request descriptor", (uint8_t *)ctrl_req, desc->len); + + vhost_scsi_task_init_target(task, ctrl_req->lun); + + vhost_vring_desc_get_next(&desc, desc_table, desc_table_size); + if (spdk_unlikely(desc == NULL)) { + SPDK_ERRLOG("%s: no response descriptor for controlq request %d.\n", + vsession->name, task->req_idx); + goto out; + } + + /* Process the TMF request */ + switch (ctrl_req->type) { + case VIRTIO_SCSI_T_TMF: + task->tmf_resp = vhost_gpa_to_vva(vsession, desc->addr, sizeof(*task->tmf_resp)); + if (spdk_unlikely(desc->len < sizeof(struct virtio_scsi_ctrl_tmf_resp) || task->tmf_resp == NULL)) { + SPDK_ERRLOG("%s: TMF response descriptor at index %d points to invalid guest memory region\n", + vsession->name, task->req_idx); + goto out; + } + + /* Check if we are processing a valid request */ + if (task->scsi_dev == NULL) { + task->tmf_resp->response = VIRTIO_SCSI_S_BAD_TARGET; + break; + } + + switch (ctrl_req->subtype) { + case VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET: + /* Handle LUN reset */ + SPDK_DEBUGLOG(SPDK_LOG_VHOST_SCSI_QUEUE, "%s: LUN reset\n", vsession->name); + + mgmt_task_submit(task, SPDK_SCSI_TASK_FUNC_LUN_RESET); + return; + default: + task->tmf_resp->response = VIRTIO_SCSI_S_ABORTED; + /* Unsupported command */ + SPDK_DEBUGLOG(SPDK_LOG_VHOST_SCSI_QUEUE, "%s: unsupported TMF command %x\n", + vsession->name, ctrl_req->subtype); + break; + } + break; + case VIRTIO_SCSI_T_AN_QUERY: + case VIRTIO_SCSI_T_AN_SUBSCRIBE: { + an_resp = vhost_gpa_to_vva(vsession, desc->addr, sizeof(*an_resp)); + if (spdk_unlikely(desc->len < sizeof(struct virtio_scsi_ctrl_an_resp) || an_resp == NULL)) { + SPDK_WARNLOG("%s: asynchronous response descriptor points to invalid guest memory region\n", + vsession->name); + goto out; + } + + an_resp->response = VIRTIO_SCSI_S_ABORTED; + break; + } + default: + SPDK_DEBUGLOG(SPDK_LOG_VHOST_SCSI_QUEUE, "%s: Unsupported control command %x\n", + vsession->name, ctrl_req->type); + break; + } + + used_len = sizeof(struct virtio_scsi_ctrl_tmf_resp); +out: + vhost_vq_used_ring_enqueue(vsession, task->vq, task->req_idx, used_len); + vhost_scsi_task_put(task); +} + +/* + * Process task's descriptor chain and setup data related fields. + * Return + * -1 if request is invalid and must be aborted, + * 0 if all data are set. + */ +static int +task_data_setup(struct spdk_vhost_scsi_task *task, + struct virtio_scsi_cmd_req **req) +{ + struct spdk_vhost_session *vsession = &task->svsession->vsession; + struct vring_desc *desc, *desc_table; + struct iovec *iovs = task->iovs; + uint16_t iovcnt = 0; + uint32_t desc_table_len, len = 0; + int rc; + + spdk_scsi_task_construct(&task->scsi, vhost_scsi_task_cpl, vhost_scsi_task_free_cb); + + rc = vhost_vq_get_desc(vsession, task->vq, task->req_idx, &desc, &desc_table, &desc_table_len); + /* First descriptor must be readable */ + if (spdk_unlikely(rc != 0 || vhost_vring_desc_is_wr(desc) || + desc->len < sizeof(struct virtio_scsi_cmd_req))) { + SPDK_WARNLOG("%s: invalid first request descriptor at index %"PRIu16".\n", + vsession->name, task->req_idx); + goto invalid_task; + } + + *req = vhost_gpa_to_vva(vsession, desc->addr, sizeof(**req)); + if (spdk_unlikely(*req == NULL)) { + SPDK_WARNLOG("%s: request descriptor at index %d points to invalid guest memory region\n", + vsession->name, task->req_idx); + goto invalid_task; + } + + /* Each request must have at least 2 descriptors (e.g. request and response) */ + vhost_vring_desc_get_next(&desc, desc_table, desc_table_len); + if (desc == NULL) { + SPDK_WARNLOG("%s: descriptor chain at index %d contains neither payload nor response buffer.\n", + vsession->name, task->req_idx); + goto invalid_task; + } + task->scsi.dxfer_dir = vhost_vring_desc_is_wr(desc) ? SPDK_SCSI_DIR_FROM_DEV : + SPDK_SCSI_DIR_TO_DEV; + task->scsi.iovs = iovs; + + if (task->scsi.dxfer_dir == SPDK_SCSI_DIR_FROM_DEV) { + /* + * FROM_DEV (READ): [RD_req][WR_resp][WR_buf0]...[WR_bufN] + */ + task->resp = vhost_gpa_to_vva(vsession, desc->addr, sizeof(*task->resp)); + if (spdk_unlikely(desc->len < sizeof(struct virtio_scsi_cmd_resp) || task->resp == NULL)) { + SPDK_WARNLOG("%s: response descriptor at index %d points to invalid guest memory region\n", + vsession->name, task->req_idx); + goto invalid_task; + } + rc = vhost_vring_desc_get_next(&desc, desc_table, desc_table_len); + if (spdk_unlikely(rc != 0)) { + SPDK_WARNLOG("%s: invalid descriptor chain at request index %d (descriptor id overflow?).\n", + vsession->name, task->req_idx); + goto invalid_task; + } + + if (desc == NULL) { + /* + * TEST UNIT READY command and some others might not contain any payload and this is not an error. + */ + SPDK_DEBUGLOG(SPDK_LOG_VHOST_SCSI_DATA, + "No payload descriptors for FROM DEV command req_idx=%"PRIu16".\n", task->req_idx); + SPDK_LOGDUMP(SPDK_LOG_VHOST_SCSI_DATA, "CDB=", (*req)->cdb, VIRTIO_SCSI_CDB_SIZE); + task->used_len = sizeof(struct virtio_scsi_cmd_resp); + task->scsi.iovcnt = 1; + task->scsi.iovs[0].iov_len = 0; + task->scsi.length = 0; + task->scsi.transfer_len = 0; + return 0; + } + + /* All remaining descriptors are data. */ + while (desc) { + if (spdk_unlikely(!vhost_vring_desc_is_wr(desc))) { + SPDK_WARNLOG("%s: FROM DEV cmd: descriptor nr %" PRIu16" in payload chain is read only.\n", + vsession->name, iovcnt); + goto invalid_task; + } + + if (spdk_unlikely(vhost_vring_desc_to_iov(vsession, iovs, &iovcnt, desc))) { + goto invalid_task; + } + len += desc->len; + + rc = vhost_vring_desc_get_next(&desc, desc_table, desc_table_len); + if (spdk_unlikely(rc != 0)) { + SPDK_WARNLOG("%s: invalid payload in descriptor chain starting at index %d.\n", + vsession->name, task->req_idx); + goto invalid_task; + } + } + + task->used_len = sizeof(struct virtio_scsi_cmd_resp) + len; + } else { + SPDK_DEBUGLOG(SPDK_LOG_VHOST_SCSI_DATA, "TO DEV"); + /* + * TO_DEV (WRITE):[RD_req][RD_buf0]...[RD_bufN][WR_resp] + * No need to check descriptor WR flag as this is done while setting scsi.dxfer_dir. + */ + + /* Process descriptors up to response. */ + while (!vhost_vring_desc_is_wr(desc)) { + if (spdk_unlikely(vhost_vring_desc_to_iov(vsession, iovs, &iovcnt, desc))) { + goto invalid_task; + } + len += desc->len; + + vhost_vring_desc_get_next(&desc, desc_table, desc_table_len); + if (spdk_unlikely(desc == NULL)) { + SPDK_WARNLOG("%s: TO_DEV cmd: no response descriptor.\n", vsession->name); + goto invalid_task; + } + } + + task->resp = vhost_gpa_to_vva(vsession, desc->addr, sizeof(*task->resp)); + if (spdk_unlikely(desc->len < sizeof(struct virtio_scsi_cmd_resp) || task->resp == NULL)) { + SPDK_WARNLOG("%s: response descriptor at index %d points to invalid guest memory region\n", + vsession->name, task->req_idx); + goto invalid_task; + } + + task->used_len = sizeof(struct virtio_scsi_cmd_resp); + } + + task->scsi.iovcnt = iovcnt; + task->scsi.length = len; + task->scsi.transfer_len = len; + return 0; + +invalid_task: + SPDK_DEBUGLOG(SPDK_LOG_VHOST_SCSI_DATA, "%s: Invalid task at index %"PRIu16".\n", + vsession->name, task->req_idx); + return -1; +} + +static int +process_request(struct spdk_vhost_scsi_task *task) +{ + struct virtio_scsi_cmd_req *req; + int result; + + result = task_data_setup(task, &req); + if (result) { + return result; + } + + result = vhost_scsi_task_init_target(task, req->lun); + if (spdk_unlikely(result != 0)) { + task->resp->response = VIRTIO_SCSI_S_BAD_TARGET; + return -1; + } + + task->scsi.cdb = req->cdb; + SPDK_LOGDUMP(SPDK_LOG_VHOST_SCSI_DATA, "request CDB", req->cdb, VIRTIO_SCSI_CDB_SIZE); + + if (spdk_unlikely(task->scsi.lun == NULL)) { + spdk_scsi_task_process_null_lun(&task->scsi); + task->resp->response = VIRTIO_SCSI_S_OK; + return 1; + } + + return 0; +} + +static void +process_scsi_task(struct spdk_vhost_session *vsession, + struct spdk_vhost_virtqueue *vq, + uint16_t req_idx) +{ + struct spdk_vhost_scsi_task *task; + int result; + + task = &((struct spdk_vhost_scsi_task *)vq->tasks)[req_idx]; + if (spdk_unlikely(task->used)) { + SPDK_ERRLOG("%s: request with idx '%"PRIu16"' is already pending.\n", + vsession->name, req_idx); + vhost_vq_used_ring_enqueue(vsession, vq, req_idx, 0); + return; + } + + vsession->task_cnt++; + scsi_task_init(task); + + if (spdk_unlikely(vq->vring_idx == VIRTIO_SCSI_CONTROLQ)) { + process_ctrl_request(task); + } else { + result = process_request(task); + if (likely(result == 0)) { + task_submit(task); + SPDK_DEBUGLOG(SPDK_LOG_VHOST_SCSI, "====== Task %p req_idx %d submitted ======\n", task, + task->req_idx); + } else if (result > 0) { + vhost_scsi_task_cpl(&task->scsi); + SPDK_DEBUGLOG(SPDK_LOG_VHOST_SCSI, "====== Task %p req_idx %d finished early ======\n", task, + task->req_idx); + } else { + invalid_request(task); + SPDK_DEBUGLOG(SPDK_LOG_VHOST_SCSI, "====== Task %p req_idx %d failed ======\n", task, + task->req_idx); + } + } +} + +static void +process_vq(struct spdk_vhost_scsi_session *svsession, struct spdk_vhost_virtqueue *vq) +{ + struct spdk_vhost_session *vsession = &svsession->vsession; + uint16_t reqs[32]; + uint16_t reqs_cnt, i; + + reqs_cnt = vhost_vq_avail_ring_get(vq, reqs, SPDK_COUNTOF(reqs)); + assert(reqs_cnt <= 32); + + for (i = 0; i < reqs_cnt; i++) { + SPDK_DEBUGLOG(SPDK_LOG_VHOST_SCSI, "====== Starting processing request idx %"PRIu16"======\n", + reqs[i]); + + if (spdk_unlikely(reqs[i] >= vq->vring.size)) { + SPDK_ERRLOG("%s: request idx '%"PRIu16"' exceeds virtqueue size (%"PRIu16").\n", + vsession->name, reqs[i], vq->vring.size); + vhost_vq_used_ring_enqueue(vsession, vq, reqs[i], 0); + continue; + } + + process_scsi_task(vsession, vq, reqs[i]); + } +} + +static int +vdev_mgmt_worker(void *arg) +{ + struct spdk_vhost_scsi_session *svsession = arg; + struct spdk_vhost_session *vsession = &svsession->vsession; + + process_removed_devs(svsession); + vhost_vq_used_signal(vsession, &vsession->virtqueue[VIRTIO_SCSI_EVENTQ]); + + process_vq(svsession, &vsession->virtqueue[VIRTIO_SCSI_CONTROLQ]); + vhost_vq_used_signal(vsession, &vsession->virtqueue[VIRTIO_SCSI_CONTROLQ]); + + return SPDK_POLLER_BUSY; +} + +static int +vdev_worker(void *arg) +{ + struct spdk_vhost_scsi_session *svsession = arg; + struct spdk_vhost_session *vsession = &svsession->vsession; + uint32_t q_idx; + + for (q_idx = VIRTIO_SCSI_REQUESTQ; q_idx < vsession->max_queues; q_idx++) { + process_vq(svsession, &vsession->virtqueue[q_idx]); + } + + vhost_session_used_signal(vsession); + + return SPDK_POLLER_BUSY; +} + +static struct spdk_vhost_scsi_dev * +to_scsi_dev(struct spdk_vhost_dev *ctrlr) +{ + if (ctrlr == NULL) { + return NULL; + } + + if (ctrlr->backend != &spdk_vhost_scsi_device_backend) { + SPDK_ERRLOG("%s: not a vhost-scsi device.\n", ctrlr->name); + return NULL; + } + + return SPDK_CONTAINEROF(ctrlr, struct spdk_vhost_scsi_dev, vdev); +} + +static struct spdk_vhost_scsi_session * +to_scsi_session(struct spdk_vhost_session *vsession) +{ + assert(vsession->vdev->backend == &spdk_vhost_scsi_device_backend); + return (struct spdk_vhost_scsi_session *)vsession; +} + +int +spdk_vhost_scsi_dev_construct(const char *name, const char *cpumask) +{ + struct spdk_vhost_scsi_dev *svdev = calloc(1, sizeof(*svdev)); + int rc; + + if (svdev == NULL) { + return -ENOMEM; + } + + svdev->vdev.virtio_features = SPDK_VHOST_SCSI_FEATURES; + svdev->vdev.disabled_features = SPDK_VHOST_SCSI_DISABLED_FEATURES; + + spdk_vhost_lock(); + rc = vhost_dev_register(&svdev->vdev, name, cpumask, + &spdk_vhost_scsi_device_backend); + + if (rc) { + free(svdev); + spdk_vhost_unlock(); + return rc; + } + + svdev->registered = true; + + spdk_vhost_unlock(); + return rc; +} + +static int +vhost_scsi_dev_remove(struct spdk_vhost_dev *vdev) +{ + struct spdk_vhost_scsi_dev *svdev = to_scsi_dev(vdev); + int rc, i; + + assert(svdev != NULL); + for (i = 0; i < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; ++i) { + if (svdev->scsi_dev_state[i].dev) { + if (vdev->registered) { + SPDK_ERRLOG("%s: SCSI target %d is still present.\n", vdev->name, i); + return -EBUSY; + } + + rc = spdk_vhost_scsi_dev_remove_tgt(vdev, i, NULL, NULL); + if (rc != 0) { + SPDK_ERRLOG("%s: failed to force-remove target %d\n", vdev->name, i); + return rc; + } + } + } + + rc = vhost_dev_unregister(vdev); + if (rc != 0) { + return rc; + } + svdev->registered = false; + + if (svdev->ref == 0) { + free(svdev); + } + + return 0; +} + +struct spdk_scsi_dev * +spdk_vhost_scsi_dev_get_tgt(struct spdk_vhost_dev *vdev, uint8_t num) +{ + struct spdk_vhost_scsi_dev *svdev; + + assert(num < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS); + svdev = to_scsi_dev(vdev); + assert(svdev != NULL); + if (svdev->scsi_dev_state[num].status != VHOST_SCSI_DEV_PRESENT) { + return NULL; + } + + assert(svdev->scsi_dev_state[num].dev != NULL); + return svdev->scsi_dev_state[num].dev; +} + +static void +vhost_scsi_lun_hotremove(const struct spdk_scsi_lun *lun, void *arg) +{ + struct spdk_vhost_scsi_dev *svdev = arg; + const struct spdk_scsi_dev *scsi_dev; + unsigned scsi_dev_num; + + assert(lun != NULL); + assert(svdev != NULL); + scsi_dev = spdk_scsi_lun_get_dev(lun); + for (scsi_dev_num = 0; scsi_dev_num < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; scsi_dev_num++) { + if (svdev->scsi_dev_state[scsi_dev_num].dev == scsi_dev) { + break; + } + } + + if (scsi_dev_num == SPDK_VHOST_SCSI_CTRLR_MAX_DEVS) { + /* The entire device has been already removed. */ + return; + } + + /* remove entire device */ + spdk_vhost_scsi_dev_remove_tgt(&svdev->vdev, scsi_dev_num, NULL, NULL); +} + +static void +vhost_scsi_dev_add_tgt_cpl_cb(struct spdk_vhost_dev *vdev, void *ctx) +{ + unsigned scsi_tgt_num = (unsigned)(uintptr_t)ctx; + struct spdk_vhost_scsi_dev *svdev = SPDK_CONTAINEROF(vdev, + struct spdk_vhost_scsi_dev, vdev); + struct spdk_scsi_dev_vhost_state *vhost_sdev; + + vhost_sdev = &svdev->scsi_dev_state[scsi_tgt_num]; + + /* All sessions have added the target */ + assert(vhost_sdev->status == VHOST_SCSI_DEV_ADDING); + vhost_sdev->status = VHOST_SCSI_DEV_PRESENT; + svdev->ref++; +} + +static int +vhost_scsi_session_add_tgt(struct spdk_vhost_dev *vdev, + struct spdk_vhost_session *vsession, void *ctx) +{ + unsigned scsi_tgt_num = (unsigned)(uintptr_t)ctx; + struct spdk_vhost_scsi_session *svsession = (struct spdk_vhost_scsi_session *)vsession; + struct spdk_scsi_dev_session_state *session_sdev = &svsession->scsi_dev_state[scsi_tgt_num]; + struct spdk_scsi_dev_vhost_state *vhost_sdev; + int rc; + + if (!vsession->started || session_sdev->dev != NULL) { + /* Nothing to do. */ + return 0; + } + + vhost_sdev = &svsession->svdev->scsi_dev_state[scsi_tgt_num]; + session_sdev->dev = vhost_sdev->dev; + session_sdev->status = VHOST_SCSI_DEV_PRESENT; + + rc = spdk_scsi_dev_allocate_io_channels(svsession->scsi_dev_state[scsi_tgt_num].dev); + if (rc != 0) { + SPDK_ERRLOG("%s: Couldn't allocate io channnel for SCSI target %u.\n", + vsession->name, scsi_tgt_num); + + /* unset the SCSI target so that all I/O to it will be rejected */ + session_sdev->dev = NULL; + /* Set status to EMPTY so that we won't reply with SCSI hotremove + * sense codes - the device hasn't ever been added. + */ + session_sdev->status = VHOST_SCSI_DEV_EMPTY; + + /* Return with no error. We'll continue allocating io_channels for + * other sessions on this device in hopes they succeed. The sessions + * that failed to allocate io_channels simply won't be able to + * detect the SCSI target, nor do any I/O to it. + */ + return 0; + } + + if (vhost_dev_has_feature(vsession, VIRTIO_SCSI_F_HOTPLUG)) { + eventq_enqueue(svsession, scsi_tgt_num, + VIRTIO_SCSI_T_TRANSPORT_RESET, VIRTIO_SCSI_EVT_RESET_RESCAN); + } else { + SPDK_NOTICELOG("%s: driver does not support hotplug. " + "Please restart it or perform a rescan.\n", + vsession->name); + } + + return 0; +} + +int +spdk_vhost_scsi_dev_add_tgt(struct spdk_vhost_dev *vdev, int scsi_tgt_num, + const char *bdev_name) +{ + struct spdk_vhost_scsi_dev *svdev; + struct spdk_scsi_dev_vhost_state *state; + char target_name[SPDK_SCSI_DEV_MAX_NAME]; + int lun_id_list[1]; + const char *bdev_names_list[1]; + + svdev = to_scsi_dev(vdev); + assert(svdev != NULL); + if (scsi_tgt_num < 0) { + for (scsi_tgt_num = 0; scsi_tgt_num < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; scsi_tgt_num++) { + if (svdev->scsi_dev_state[scsi_tgt_num].dev == NULL) { + break; + } + } + + if (scsi_tgt_num == SPDK_VHOST_SCSI_CTRLR_MAX_DEVS) { + SPDK_ERRLOG("%s: all SCSI target slots are already in use.\n", vdev->name); + return -ENOSPC; + } + } else { + if (scsi_tgt_num >= SPDK_VHOST_SCSI_CTRLR_MAX_DEVS) { + SPDK_ERRLOG("%s: SCSI target number is too big (got %d, max %d)\n", + vdev->name, scsi_tgt_num, SPDK_VHOST_SCSI_CTRLR_MAX_DEVS); + return -EINVAL; + } + } + + if (bdev_name == NULL) { + SPDK_ERRLOG("No lun name specified\n"); + return -EINVAL; + } + + state = &svdev->scsi_dev_state[scsi_tgt_num]; + if (state->dev != NULL) { + SPDK_ERRLOG("%s: SCSI target %u already occupied\n", vdev->name, scsi_tgt_num); + return -EEXIST; + } + + /* + * At this stage only one LUN per target + */ + snprintf(target_name, sizeof(target_name), "Target %u", scsi_tgt_num); + lun_id_list[0] = 0; + bdev_names_list[0] = (char *)bdev_name; + + state->status = VHOST_SCSI_DEV_ADDING; + state->dev = spdk_scsi_dev_construct(target_name, bdev_names_list, lun_id_list, 1, + SPDK_SPC_PROTOCOL_IDENTIFIER_SAS, + vhost_scsi_lun_hotremove, svdev); + + if (state->dev == NULL) { + state->status = VHOST_SCSI_DEV_EMPTY; + SPDK_ERRLOG("%s: couldn't create SCSI target %u using bdev '%s'\n", + vdev->name, scsi_tgt_num, bdev_name); + return -EINVAL; + } + spdk_scsi_dev_add_port(state->dev, 0, "vhost"); + + SPDK_INFOLOG(SPDK_LOG_VHOST, "%s: added SCSI target %u using bdev '%s'\n", + vdev->name, scsi_tgt_num, bdev_name); + + vhost_dev_foreach_session(vdev, vhost_scsi_session_add_tgt, + vhost_scsi_dev_add_tgt_cpl_cb, + (void *)(uintptr_t)scsi_tgt_num); + return scsi_tgt_num; +} + +struct scsi_tgt_hotplug_ctx { + unsigned scsi_tgt_num; + bool async_fini; +}; + +static void +vhost_scsi_dev_remove_tgt_cpl_cb(struct spdk_vhost_dev *vdev, void *_ctx) +{ + struct scsi_tgt_hotplug_ctx *ctx = _ctx; + struct spdk_vhost_scsi_dev *svdev = SPDK_CONTAINEROF(vdev, + struct spdk_vhost_scsi_dev, vdev); + + if (!ctx->async_fini) { + /* there aren't any active sessions, so remove the dev and exit */ + remove_scsi_tgt(svdev, ctx->scsi_tgt_num); + } + + free(ctx); +} + +static int +vhost_scsi_session_remove_tgt(struct spdk_vhost_dev *vdev, + struct spdk_vhost_session *vsession, void *_ctx) +{ + struct scsi_tgt_hotplug_ctx *ctx = _ctx; + unsigned scsi_tgt_num = ctx->scsi_tgt_num; + struct spdk_vhost_scsi_session *svsession = (struct spdk_vhost_scsi_session *)vsession; + struct spdk_scsi_dev_session_state *state = &svsession->scsi_dev_state[scsi_tgt_num]; + + if (!vsession->started || state->dev == NULL) { + /* Nothing to do */ + return 0; + } + + /* Mark the target for removal */ + assert(state->status == VHOST_SCSI_DEV_PRESENT); + state->status = VHOST_SCSI_DEV_REMOVING; + + /* Send a hotremove Virtio event */ + if (vhost_dev_has_feature(vsession, VIRTIO_SCSI_F_HOTPLUG)) { + eventq_enqueue(svsession, scsi_tgt_num, + VIRTIO_SCSI_T_TRANSPORT_RESET, VIRTIO_SCSI_EVT_RESET_REMOVED); + } + + /* Wait for the session's management poller to remove the target after + * all its pending I/O has finished. + */ + ctx->async_fini = true; + return 0; +} + +int +spdk_vhost_scsi_dev_remove_tgt(struct spdk_vhost_dev *vdev, unsigned scsi_tgt_num, + spdk_vhost_event_fn cb_fn, void *cb_arg) +{ + struct spdk_vhost_scsi_dev *svdev; + struct spdk_scsi_dev_vhost_state *scsi_dev_state; + struct scsi_tgt_hotplug_ctx *ctx; + + if (scsi_tgt_num >= SPDK_VHOST_SCSI_CTRLR_MAX_DEVS) { + SPDK_ERRLOG("%s: invalid SCSI target number %d\n", vdev->name, scsi_tgt_num); + return -EINVAL; + } + + svdev = to_scsi_dev(vdev); + assert(svdev != NULL); + scsi_dev_state = &svdev->scsi_dev_state[scsi_tgt_num]; + + if (scsi_dev_state->status != VHOST_SCSI_DEV_PRESENT) { + return -EBUSY; + } + + if (scsi_dev_state->dev == NULL || scsi_dev_state->status == VHOST_SCSI_DEV_ADDING) { + SPDK_ERRLOG("%s: SCSI target %u is not occupied\n", vdev->name, scsi_tgt_num); + return -ENODEV; + } + + assert(scsi_dev_state->status != VHOST_SCSI_DEV_EMPTY); + ctx = calloc(1, sizeof(*ctx)); + if (ctx == NULL) { + SPDK_ERRLOG("calloc failed\n"); + return -ENOMEM; + } + + ctx->scsi_tgt_num = scsi_tgt_num; + ctx->async_fini = false; + + scsi_dev_state->remove_cb = cb_fn; + scsi_dev_state->remove_ctx = cb_arg; + scsi_dev_state->status = VHOST_SCSI_DEV_REMOVING; + + vhost_dev_foreach_session(vdev, vhost_scsi_session_remove_tgt, + vhost_scsi_dev_remove_tgt_cpl_cb, ctx); + return 0; +} + +int +vhost_scsi_controller_construct(void) +{ + struct spdk_conf_section *sp = spdk_conf_first_section(NULL); + struct spdk_vhost_dev *vdev; + int i, dev_num; + unsigned ctrlr_num = 0; + char *bdev_name, *tgt_num_str; + char *cpumask; + char *name; + char *tgt = NULL; + + while (sp != NULL) { + if (!spdk_conf_section_match_prefix(sp, "VhostScsi")) { + sp = spdk_conf_next_section(sp); + continue; + } + + if (sscanf(spdk_conf_section_get_name(sp), "VhostScsi%u", &ctrlr_num) != 1) { + SPDK_ERRLOG("Section '%s' has non-numeric suffix.\n", + spdk_conf_section_get_name(sp)); + return -1; + } + + name = spdk_conf_section_get_val(sp, "Name"); + cpumask = spdk_conf_section_get_val(sp, "Cpumask"); + + if (spdk_vhost_scsi_dev_construct(name, cpumask) < 0) { + return -1; + } + + vdev = spdk_vhost_dev_find(name); + assert(vdev); + + for (i = 0; ; i++) { + + tgt = spdk_conf_section_get_nval(sp, "Target", i); + if (tgt == NULL) { + break; + } + + tgt_num_str = spdk_conf_section_get_nmval(sp, "Target", i, 0); + if (tgt_num_str == NULL) { + SPDK_ERRLOG("%s: invalid or missing SCSI target number\n", name); + return -1; + } + + dev_num = (int)strtol(tgt_num_str, NULL, 10); + bdev_name = spdk_conf_section_get_nmval(sp, "Target", i, 1); + if (bdev_name == NULL) { + SPDK_ERRLOG("%s: invalid or missing bdev name for SCSI target %d\n", name, dev_num); + return -1; + } else if (spdk_conf_section_get_nmval(sp, "Target", i, 2)) { + SPDK_ERRLOG("%s: only one LUN per SCSI target is supported\n", name); + return -1; + } + + if (spdk_vhost_scsi_dev_add_tgt(vdev, dev_num, bdev_name) < 0) { + return -1; + } + } + + sp = spdk_conf_next_section(sp); + } + + return 0; +} + +static void +free_task_pool(struct spdk_vhost_scsi_session *svsession) +{ + struct spdk_vhost_session *vsession = &svsession->vsession; + struct spdk_vhost_virtqueue *vq; + uint16_t i; + + for (i = 0; i < vsession->max_queues; i++) { + vq = &vsession->virtqueue[i]; + if (vq->tasks == NULL) { + continue; + } + + spdk_free(vq->tasks); + vq->tasks = NULL; + } +} + +static int +alloc_task_pool(struct spdk_vhost_scsi_session *svsession) +{ + struct spdk_vhost_session *vsession = &svsession->vsession; + struct spdk_vhost_virtqueue *vq; + struct spdk_vhost_scsi_task *task; + uint32_t task_cnt; + uint16_t i; + uint32_t j; + + for (i = 0; i < vsession->max_queues; i++) { + vq = &vsession->virtqueue[i]; + if (vq->vring.desc == NULL) { + continue; + } + + task_cnt = vq->vring.size; + if (task_cnt > SPDK_VHOST_MAX_VQ_SIZE) { + /* sanity check */ + SPDK_ERRLOG("%s: virtuque %"PRIu16" is too big. (size = %"PRIu32", max = %"PRIu32")\n", + vsession->name, i, task_cnt, SPDK_VHOST_MAX_VQ_SIZE); + free_task_pool(svsession); + return -1; + } + vq->tasks = spdk_zmalloc(sizeof(struct spdk_vhost_scsi_task) * task_cnt, + SPDK_CACHE_LINE_SIZE, NULL, + SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA); + if (vq->tasks == NULL) { + SPDK_ERRLOG("%s: failed to allocate %"PRIu32" tasks for virtqueue %"PRIu16"\n", + vsession->name, task_cnt, i); + free_task_pool(svsession); + return -1; + } + + for (j = 0; j < task_cnt; j++) { + task = &((struct spdk_vhost_scsi_task *)vq->tasks)[j]; + task->svsession = svsession; + task->vq = vq; + task->req_idx = j; + } + } + + return 0; +} + +static int +vhost_scsi_start_cb(struct spdk_vhost_dev *vdev, + struct spdk_vhost_session *vsession, void *unused) +{ + struct spdk_vhost_scsi_session *svsession = to_scsi_session(vsession); + struct spdk_vhost_scsi_dev *svdev = svsession->svdev; + struct spdk_scsi_dev_vhost_state *state; + uint32_t i; + int rc; + + /* validate all I/O queues are in a contiguous index range */ + for (i = VIRTIO_SCSI_REQUESTQ; i < vsession->max_queues; i++) { + if (vsession->virtqueue[i].vring.desc == NULL) { + SPDK_ERRLOG("%s: queue %"PRIu32" is empty\n", vsession->name, i); + rc = -1; + goto out; + } + } + + rc = alloc_task_pool(svsession); + if (rc != 0) { + SPDK_ERRLOG("%s: failed to alloc task pool.\n", vsession->name); + goto out; + } + + for (i = 0; i < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; i++) { + state = &svdev->scsi_dev_state[i]; + if (state->dev == NULL || state->status == VHOST_SCSI_DEV_REMOVING) { + continue; + } + + assert(svsession->scsi_dev_state[i].status == VHOST_SCSI_DEV_EMPTY); + svsession->scsi_dev_state[i].dev = state->dev; + svsession->scsi_dev_state[i].status = VHOST_SCSI_DEV_PRESENT; + rc = spdk_scsi_dev_allocate_io_channels(state->dev); + if (rc != 0) { + SPDK_ERRLOG("%s: failed to alloc io_channel for SCSI target %"PRIu32"\n", + vsession->name, i); + /* unset the SCSI target so that all I/O to it will be rejected */ + svsession->scsi_dev_state[i].dev = NULL; + /* set EMPTY state so that we won't reply with SCSI hotremove + * sense codes - the device hasn't ever been added. + */ + svsession->scsi_dev_state[i].status = VHOST_SCSI_DEV_EMPTY; + continue; + } + } + SPDK_INFOLOG(SPDK_LOG_VHOST, "%s: started poller on lcore %d\n", + vsession->name, spdk_env_get_current_core()); + + svsession->requestq_poller = SPDK_POLLER_REGISTER(vdev_worker, svsession, 0); + if (vsession->virtqueue[VIRTIO_SCSI_CONTROLQ].vring.desc && + vsession->virtqueue[VIRTIO_SCSI_EVENTQ].vring.desc) { + svsession->mgmt_poller = SPDK_POLLER_REGISTER(vdev_mgmt_worker, svsession, + MGMT_POLL_PERIOD_US); + } +out: + vhost_session_start_done(vsession, rc); + return rc; +} + +static int +vhost_scsi_start(struct spdk_vhost_session *vsession) +{ + struct spdk_vhost_scsi_session *svsession = to_scsi_session(vsession); + struct spdk_vhost_scsi_dev *svdev; + + svdev = to_scsi_dev(vsession->vdev); + assert(svdev != NULL); + svsession->svdev = svdev; + + return vhost_session_send_event(vsession, vhost_scsi_start_cb, + 3, "start session"); +} + +static int +destroy_session_poller_cb(void *arg) +{ + struct spdk_vhost_scsi_session *svsession = arg; + struct spdk_vhost_session *vsession = &svsession->vsession; + struct spdk_scsi_dev_session_state *state; + uint32_t i; + + if (vsession->task_cnt > 0) { + return SPDK_POLLER_BUSY; + } + + if (spdk_vhost_trylock() != 0) { + return SPDK_POLLER_BUSY; + } + + for (i = 0; i < vsession->max_queues; i++) { + vhost_vq_used_signal(vsession, &vsession->virtqueue[i]); + } + + for (i = 0; i < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; i++) { + enum spdk_scsi_dev_vhost_status prev_status; + + state = &svsession->scsi_dev_state[i]; + /* clear the REMOVED status so that we won't send hotremove events anymore */ + prev_status = state->status; + state->status = VHOST_SCSI_DEV_EMPTY; + if (state->dev == NULL) { + continue; + } + + spdk_scsi_dev_free_io_channels(state->dev); + + state->dev = NULL; + + if (prev_status == VHOST_SCSI_DEV_REMOVING) { + /* try to detach it globally */ + vhost_dev_foreach_session(vsession->vdev, + vhost_scsi_session_process_removed, + vhost_scsi_dev_process_removed_cpl_cb, + (void *)(uintptr_t)i); + } + } + + SPDK_INFOLOG(SPDK_LOG_VHOST, "%s: stopping poller on lcore %d\n", + vsession->name, spdk_env_get_current_core()); + + free_task_pool(svsession); + + spdk_poller_unregister(&svsession->stop_poller); + vhost_session_stop_done(vsession, 0); + + spdk_vhost_unlock(); + return SPDK_POLLER_BUSY; +} + +static int +vhost_scsi_stop_cb(struct spdk_vhost_dev *vdev, + struct spdk_vhost_session *vsession, void *unused) +{ + struct spdk_vhost_scsi_session *svsession = to_scsi_session(vsession); + + /* Stop receiving new I/O requests */ + spdk_poller_unregister(&svsession->requestq_poller); + + /* Stop receiving controlq requests, also stop processing the + * asynchronous hotremove events. All the remaining events + * will be finalized by the stop_poller below. + */ + spdk_poller_unregister(&svsession->mgmt_poller); + + /* Wait for all pending I/Os to complete, then process all the + * remaining hotremove events one last time. + */ + svsession->stop_poller = SPDK_POLLER_REGISTER(destroy_session_poller_cb, + svsession, 1000); + + return 0; +} + +static int +vhost_scsi_stop(struct spdk_vhost_session *vsession) +{ + return vhost_session_send_event(vsession, vhost_scsi_stop_cb, + 3, "stop session"); +} + +static void +vhost_scsi_dump_info_json(struct spdk_vhost_dev *vdev, struct spdk_json_write_ctx *w) +{ + struct spdk_scsi_dev *sdev; + struct spdk_scsi_lun *lun; + uint32_t dev_idx; + uint32_t lun_idx; + + assert(vdev != NULL); + spdk_json_write_named_array_begin(w, "scsi"); + for (dev_idx = 0; dev_idx < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; dev_idx++) { + sdev = spdk_vhost_scsi_dev_get_tgt(vdev, dev_idx); + if (!sdev) { + continue; + } + + spdk_json_write_object_begin(w); + + spdk_json_write_named_uint32(w, "scsi_dev_num", dev_idx); + + spdk_json_write_named_uint32(w, "id", spdk_scsi_dev_get_id(sdev)); + + spdk_json_write_named_string(w, "target_name", spdk_scsi_dev_get_name(sdev)); + + spdk_json_write_named_array_begin(w, "luns"); + + for (lun_idx = 0; lun_idx < SPDK_SCSI_DEV_MAX_LUN; lun_idx++) { + lun = spdk_scsi_dev_get_lun(sdev, lun_idx); + if (!lun) { + continue; + } + + spdk_json_write_object_begin(w); + + spdk_json_write_named_int32(w, "id", spdk_scsi_lun_get_id(lun)); + + spdk_json_write_named_string(w, "bdev_name", spdk_scsi_lun_get_bdev_name(lun)); + + spdk_json_write_object_end(w); + } + + spdk_json_write_array_end(w); + spdk_json_write_object_end(w); + } + + spdk_json_write_array_end(w); +} + +static void +vhost_scsi_write_config_json(struct spdk_vhost_dev *vdev, struct spdk_json_write_ctx *w) +{ + struct spdk_scsi_dev *scsi_dev; + struct spdk_scsi_lun *lun; + uint32_t i; + + spdk_json_write_object_begin(w); + spdk_json_write_named_string(w, "method", "vhost_create_scsi_controller"); + + spdk_json_write_named_object_begin(w, "params"); + spdk_json_write_named_string(w, "ctrlr", vdev->name); + spdk_json_write_named_string(w, "cpumask", + spdk_cpuset_fmt(spdk_thread_get_cpumask(vdev->thread))); + spdk_json_write_object_end(w); + + spdk_json_write_object_end(w); + + for (i = 0; i < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; i++) { + scsi_dev = spdk_vhost_scsi_dev_get_tgt(vdev, i); + if (scsi_dev == NULL) { + continue; + } + + lun = spdk_scsi_dev_get_lun(scsi_dev, 0); + assert(lun != NULL); + + spdk_json_write_object_begin(w); + spdk_json_write_named_string(w, "method", "vhost_scsi_controller_add_target"); + + spdk_json_write_named_object_begin(w, "params"); + spdk_json_write_named_string(w, "ctrlr", vdev->name); + spdk_json_write_named_uint32(w, "scsi_target_num", i); + + spdk_json_write_named_string(w, "bdev_name", spdk_scsi_lun_get_bdev_name(lun)); + spdk_json_write_object_end(w); + + spdk_json_write_object_end(w); + } +} + +SPDK_LOG_REGISTER_COMPONENT("vhost_scsi", SPDK_LOG_VHOST_SCSI) +SPDK_LOG_REGISTER_COMPONENT("vhost_scsi_queue", SPDK_LOG_VHOST_SCSI_QUEUE) +SPDK_LOG_REGISTER_COMPONENT("vhost_scsi_data", SPDK_LOG_VHOST_SCSI_DATA) |