summaryrefslogtreecommitdiffstats
path: root/src/spdk/include/spdk_internal
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-27 18:24:20 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-27 18:24:20 +0000
commit483eb2f56657e8e7f419ab1a4fab8dce9ade8609 (patch)
treee5d88d25d870d5dedacb6bbdbe2a966086a0a5cf /src/spdk/include/spdk_internal
parentInitial commit. (diff)
downloadceph-upstream.tar.xz
ceph-upstream.zip
Adding upstream version 14.2.21.upstream/14.2.21upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src/spdk/include/spdk_internal')
-rw-r--r--src/spdk/include/spdk_internal/assert.h55
-rw-r--r--src/spdk/include/spdk_internal/copy_engine.h95
-rw-r--r--src/spdk/include/spdk_internal/event.h133
-rw-r--r--src/spdk/include/spdk_internal/log.h101
-rw-r--r--src/spdk/include/spdk_internal/lvolstore.h124
-rw-r--r--src/spdk/include/spdk_internal/mock.h117
-rw-r--r--src/spdk/include/spdk_internal/sock.h109
-rw-r--r--src/spdk/include/spdk_internal/utf.h325
-rw-r--r--src/spdk/include/spdk_internal/virtio.h490
9 files changed, 1549 insertions, 0 deletions
diff --git a/src/spdk/include/spdk_internal/assert.h b/src/spdk/include/spdk_internal/assert.h
new file mode 100644
index 00000000..7e4c4507
--- /dev/null
+++ b/src/spdk/include/spdk_internal/assert.h
@@ -0,0 +1,55 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef SPDK_INTERNAL_ASSERT_H
+#define SPDK_INTERNAL_ASSERT_H
+
+#include "spdk/stdinc.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "spdk/assert.h"
+
+#if !defined(DEBUG) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5))
+#define SPDK_UNREACHABLE() __builtin_unreachable()
+#else
+#define SPDK_UNREACHABLE() abort()
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SPDK_INTERNAL_ASSERT_H */
diff --git a/src/spdk/include/spdk_internal/copy_engine.h b/src/spdk/include/spdk_internal/copy_engine.h
new file mode 100644
index 00000000..6b8a13c4
--- /dev/null
+++ b/src/spdk/include/spdk_internal/copy_engine.h
@@ -0,0 +1,95 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef SPDK_INTERNAL_COPY_ENGINE_H
+#define SPDK_INTERNAL_COPY_ENGINE_H
+
+#include "spdk/stdinc.h"
+
+#include "spdk/copy_engine.h"
+#include "spdk/queue.h"
+
+struct spdk_copy_task {
+ spdk_copy_completion_cb cb;
+ uint8_t offload_ctx[0];
+};
+
+struct spdk_copy_engine {
+ int (*copy)(void *cb_arg, struct spdk_io_channel *ch, void *dst, void *src,
+ uint64_t nbytes, spdk_copy_completion_cb cb);
+ int (*fill)(void *cb_arg, struct spdk_io_channel *ch, void *dst, uint8_t fill,
+ uint64_t nbytes, spdk_copy_completion_cb cb);
+ struct spdk_io_channel *(*get_io_channel)(void);
+};
+
+struct spdk_copy_module_if {
+ /** Initialization function for the module. Called by the spdk
+ * application during startup.
+ *
+ * Modules are required to define this function.
+ */
+ int (*module_init)(void);
+
+ /** Finish function for the module. Called by the spdk application
+ * before the spdk application exits to perform any necessary cleanup.
+ *
+ * Modules are not required to define this function.
+ */
+ void (*module_fini)(void *ctx);
+
+ /** Function called to return a text string representing the
+ * module's configuration options for inclusion in an
+ * spdk configuration file.
+ */
+ void (*config_text)(FILE *fp);
+
+ size_t (*get_ctx_size)(void);
+ TAILQ_ENTRY(spdk_copy_module_if) tailq;
+};
+
+void spdk_copy_engine_register(struct spdk_copy_engine *copy_engine);
+void spdk_copy_module_list_add(struct spdk_copy_module_if *copy_module);
+
+#define SPDK_COPY_MODULE_REGISTER(init_fn, fini_fn, config_fn, ctx_size_fn) \
+ static struct spdk_copy_module_if init_fn ## _if = { \
+ .module_init = init_fn, \
+ .module_fini = fini_fn, \
+ .config_text = config_fn, \
+ .get_ctx_size = ctx_size_fn, \
+ }; \
+ __attribute__((constructor)) static void init_fn ## _init(void) \
+ { \
+ spdk_copy_module_list_add(&init_fn ## _if); \
+ }
+
+#endif
diff --git a/src/spdk/include/spdk_internal/event.h b/src/spdk/include/spdk_internal/event.h
new file mode 100644
index 00000000..25bdc04e
--- /dev/null
+++ b/src/spdk/include/spdk_internal/event.h
@@ -0,0 +1,133 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef SPDK_INTERNAL_EVENT_H
+#define SPDK_INTERNAL_EVENT_H
+
+#include "spdk/stdinc.h"
+
+#include "spdk/event.h"
+#include "spdk/json.h"
+
+struct spdk_event {
+ uint32_t lcore;
+ spdk_event_fn fn;
+ void *arg1;
+ void *arg2;
+};
+
+int spdk_reactors_init(unsigned int max_delay_us);
+void spdk_reactors_fini(void);
+
+void spdk_reactors_start(void);
+void spdk_reactors_stop(void *arg1, void *arg2);
+
+struct spdk_subsystem {
+ const char *name;
+ /* User must call spdk_subsystem_init_next() when they are done with their initialization. */
+ void (*init)(void);
+ void (*fini)(void);
+ void (*config)(FILE *fp);
+
+ /**
+ * Write JSON configuration handler.
+ *
+ * \param w JSON write context
+ * \param done_ev Done event to be called when writing is done.
+ */
+ void (*write_config_json)(struct spdk_json_write_ctx *w, struct spdk_event *done_ev);
+ TAILQ_ENTRY(spdk_subsystem) tailq;
+};
+
+TAILQ_HEAD(spdk_subsystem_list, spdk_subsystem);
+extern struct spdk_subsystem_list g_subsystems;
+
+struct spdk_subsystem *spdk_subsystem_find(struct spdk_subsystem_list *list, const char *name);
+
+struct spdk_subsystem_depend {
+ const char *name;
+ const char *depends_on;
+ TAILQ_ENTRY(spdk_subsystem_depend) tailq;
+};
+
+TAILQ_HEAD(spdk_subsystem_depend_list, spdk_subsystem_depend);
+extern struct spdk_subsystem_depend_list g_subsystems_deps;
+
+void spdk_add_subsystem(struct spdk_subsystem *subsystem);
+void spdk_add_subsystem_depend(struct spdk_subsystem_depend *depend);
+
+void spdk_subsystem_init(struct spdk_event *app_start_event);
+void spdk_subsystem_fini(struct spdk_event *app_finish_event);
+void spdk_subsystem_init_next(int rc);
+void spdk_subsystem_fini_next(void);
+void spdk_subsystem_config(FILE *fp);
+
+/**
+ * Save pointed \c subsystem configuration to the JSON write context \c w. In case of
+ * error \c null is written to the JSON context. Writing might be done in async way
+ * so caller need to pass event that subsystem will call when it finish writing
+ * configuration.
+ *
+ * \param w JSON write context
+ * \param subsystem the subsystem to query
+ * \param done_ev event to be called when writing is done
+ */
+void spdk_subsystem_config_json(struct spdk_json_write_ctx *w, struct spdk_subsystem *subsystem,
+ struct spdk_event *done_ev);
+
+void spdk_rpc_initialize(const char *listen_addr);
+void spdk_rpc_finish(void);
+
+/**
+ * \brief Register a new subsystem
+ */
+#define SPDK_SUBSYSTEM_REGISTER(_name) \
+ __attribute__((constructor)) static void _name ## _register(void) \
+ { \
+ spdk_add_subsystem(&_name); \
+ }
+
+/**
+ * \brief Declare that a subsystem depends on another subsystem.
+ */
+#define SPDK_SUBSYSTEM_DEPEND(_name, _depends_on) \
+ static struct spdk_subsystem_depend __subsystem_ ## _name ## _depend_on ## _depends_on = { \
+ .name = #_name, \
+ .depends_on = #_depends_on, \
+ }; \
+ __attribute__((constructor)) static void _name ## _depend_on ## _depends_on(void) \
+ { \
+ spdk_add_subsystem_depend(&__subsystem_ ## _name ## _depend_on ## _depends_on); \
+ }
+
+#endif /* SPDK_INTERNAL_EVENT_H */
diff --git a/src/spdk/include/spdk_internal/log.h b/src/spdk/include/spdk_internal/log.h
new file mode 100644
index 00000000..c91116bb
--- /dev/null
+++ b/src/spdk/include/spdk_internal/log.h
@@ -0,0 +1,101 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ * Logging interfaces
+ */
+
+#ifndef SPDK_INTERNAL_LOG_H
+#define SPDK_INTERNAL_LOG_H
+
+#include "spdk/log.h"
+#include "spdk/queue.h"
+
+extern enum spdk_log_level g_spdk_log_level;
+extern enum spdk_log_level g_spdk_log_print_level;
+extern enum spdk_log_level g_spdk_log_backtrace_level;
+
+struct spdk_trace_flag {
+ TAILQ_ENTRY(spdk_trace_flag) tailq;
+ const char *name;
+ bool enabled;
+};
+
+void spdk_log_register_trace_flag(const char *name, struct spdk_trace_flag *flag);
+
+struct spdk_trace_flag *spdk_log_get_first_trace_flag(void);
+struct spdk_trace_flag *spdk_log_get_next_trace_flag(struct spdk_trace_flag *flag);
+
+#define SPDK_LOG_REGISTER_COMPONENT(str, flag) \
+struct spdk_trace_flag flag = { \
+ .enabled = false, \
+ .name = str, \
+}; \
+__attribute__((constructor)) static void register_trace_flag_##flag(void) \
+{ \
+ spdk_log_register_trace_flag(str, &flag); \
+}
+
+#define SPDK_INFOLOG(FLAG, ...) \
+ do { \
+ extern struct spdk_trace_flag FLAG; \
+ if (FLAG.enabled) { \
+ spdk_log(SPDK_LOG_INFO, __FILE__, __LINE__, __func__, __VA_ARGS__); \
+ } \
+ } while (0)
+
+#ifdef DEBUG
+
+#define SPDK_DEBUGLOG(FLAG, ...) \
+ do { \
+ extern struct spdk_trace_flag FLAG; \
+ if (FLAG.enabled) { \
+ spdk_log(SPDK_LOG_DEBUG, __FILE__, __LINE__, __func__, __VA_ARGS__); \
+ } \
+ } while (0)
+
+#define SPDK_TRACEDUMP(FLAG, LABEL, BUF, LEN) \
+ do { \
+ extern struct spdk_trace_flag FLAG; \
+ if ((FLAG.enabled) && (LEN)) { \
+ spdk_trace_dump(stderr, (LABEL), (BUF), (LEN)); \
+ } \
+ } while (0)
+
+#else
+#define SPDK_DEBUGLOG(...) do { } while (0)
+#define SPDK_TRACEDUMP(...) do { } while (0)
+#endif
+
+#endif /* SPDK_INTERNAL_LOG_H */
diff --git a/src/spdk/include/spdk_internal/lvolstore.h b/src/spdk/include/spdk_internal/lvolstore.h
new file mode 100644
index 00000000..064bad5d
--- /dev/null
+++ b/src/spdk/include/spdk_internal/lvolstore.h
@@ -0,0 +1,124 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef SPDK_INTERNAL_LVOLSTORE_H
+#define SPDK_INTERNAL_LVOLSTORE_H
+
+#include "spdk/blob.h"
+#include "spdk/lvol.h"
+#include "spdk/uuid.h"
+#include "spdk/bdev_module.h"
+
+/* Default size of blobstore cluster */
+#define SPDK_LVS_OPTS_CLUSTER_SZ (4 * 1024 * 1024)
+
+struct spdk_lvs_req {
+ spdk_lvs_op_complete cb_fn;
+ void *cb_arg;
+ struct spdk_lvol_store *lvol_store;
+ int lvserrno;
+};
+
+struct spdk_lvol_req {
+ spdk_lvol_op_complete cb_fn;
+ void *cb_arg;
+ struct spdk_lvol *lvol;
+ size_t sz;
+ struct spdk_io_channel *channel;
+ char name[SPDK_LVOL_NAME_MAX];
+};
+
+struct spdk_lvs_with_handle_req {
+ spdk_lvs_op_with_handle_complete cb_fn;
+ void *cb_arg;
+ struct spdk_lvol_store *lvol_store;
+ struct spdk_bs_dev *bs_dev;
+ struct spdk_bdev *base_bdev;
+ int lvserrno;
+};
+
+struct spdk_lvs_destroy_req {
+ spdk_lvs_op_complete cb_fn;
+ void *cb_arg;
+ struct spdk_lvol_store *lvs;
+};
+
+struct spdk_lvol_with_handle_req {
+ spdk_lvol_op_with_handle_complete cb_fn;
+ void *cb_arg;
+ struct spdk_lvol *lvol;
+};
+
+struct spdk_lvol_store {
+ struct spdk_bs_dev *bs_dev;
+ struct spdk_blob_store *blobstore;
+ struct spdk_blob *super_blob;
+ spdk_blob_id super_blob_id;
+ struct spdk_uuid uuid;
+ int lvol_count;
+ int lvols_opened;
+ bool destruct;
+ TAILQ_HEAD(, spdk_lvol) lvols;
+ TAILQ_HEAD(, spdk_lvol) pending_lvols;
+ bool on_list;
+ TAILQ_ENTRY(spdk_lvol_store) link;
+ char name[SPDK_LVS_NAME_MAX];
+ char new_name[SPDK_LVS_NAME_MAX];
+};
+
+struct spdk_lvol {
+ struct spdk_lvol_store *lvol_store;
+ struct spdk_blob *blob;
+ spdk_blob_id blob_id;
+ char *unique_id;
+ char name[SPDK_LVOL_NAME_MAX];
+ struct spdk_uuid uuid;
+ char uuid_str[SPDK_UUID_STRING_LEN];
+ bool thin_provision;
+ struct spdk_bdev *bdev;
+ int ref_count;
+ bool action_in_progress;
+ TAILQ_ENTRY(spdk_lvol) link;
+};
+
+struct lvol_task {
+ enum spdk_bdev_io_status status;
+};
+
+struct lvol_store_bdev *vbdev_lvol_store_first(void);
+struct lvol_store_bdev *vbdev_lvol_store_next(struct lvol_store_bdev *prev);
+
+void spdk_lvol_resize(struct spdk_lvol *lvol, uint64_t sz, spdk_lvol_op_complete cb_fn,
+ void *cb_arg);
+
+#endif /* SPDK_INTERNAL_LVOLSTORE_H */
diff --git a/src/spdk/include/spdk_internal/mock.h b/src/spdk/include/spdk_internal/mock.h
new file mode 100644
index 00000000..f5a366dd
--- /dev/null
+++ b/src/spdk/include/spdk_internal/mock.h
@@ -0,0 +1,117 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef SPDK_INTERNAL_MOCK_H
+#define SPDK_INTERNAL_MOCK_H
+
+#include "spdk/stdinc.h"
+
+#define MOCK_STRUCT_INIT(...) \
+ { __VA_ARGS__ }
+
+#define DEFINE_RETURN_MOCK(fn, ret) \
+ bool ut_ ## fn ## _mocked = false; \
+ ret ut_ ## fn
+
+/*
+ * For controlling mocked function behavior, setting
+ * and getting values from the stub, the _P macros are
+ * for mocking functions that return pointer values.
+ */
+#define MOCK_SET(fn, val) \
+ ut_ ## fn ## _mocked = true; \
+ ut_ ## fn = val
+
+#define MOCK_GET(fn) \
+ ut_ ## fn
+
+#define MOCK_CLEAR(fn) \
+ ut_ ## fn ## _mocked = false;
+
+#define MOCK_CLEAR_P(fn) \
+ ut_ ## fn ## _mocked = false; \
+ ut_ ## fn = NULL;
+
+/* for declaring function protoypes for wrappers */
+#define DECLARE_WRAPPER(fn, ret, args) \
+ extern bool ut_ ## fn ## _mocked; \
+ extern ret ut_ ## fn; \
+ ret __wrap_ ## fn args; ret __real_ ## fn args;
+
+/* for defining the implmentation of wrappers for syscalls */
+#define DEFINE_WRAPPER(fn, ret, dargs, pargs) \
+ DEFINE_RETURN_MOCK(fn, ret); \
+ __attribute__((used)) ret __wrap_ ## fn dargs \
+ { \
+ if (!ut_ ## fn ## _mocked) { \
+ return __real_ ## fn pargs; \
+ } else { \
+ return MOCK_GET(fn); \
+ } \
+ }
+
+/* DEFINE_STUB is for defining the implmentation of stubs for SPDK funcs. */
+#define DEFINE_STUB(fn, ret, dargs, val) \
+ bool ut_ ## fn ## _mocked = true; \
+ ret ut_ ## fn = val; \
+ ret fn dargs; \
+ ret fn dargs \
+ { \
+ return MOCK_GET(fn); \
+ }
+
+/* DEFINE_STUB_V macro is for stubs that don't have a return value */
+#define DEFINE_STUB_V(fn, dargs) \
+ void fn dargs; \
+ void fn dargs \
+ { \
+ }
+
+#define HANDLE_RETURN_MOCK(fn) \
+ if (ut_ ## fn ## _mocked) { \
+ return ut_ ## fn; \
+ }
+
+
+/* declare wrapper protos (alphabetically please) here */
+DECLARE_WRAPPER(calloc, void *, (size_t nmemb, size_t size));
+
+DECLARE_WRAPPER(pthread_mutex_init, int,
+ (pthread_mutex_t *mtx, const pthread_mutexattr_t *attr));
+
+DECLARE_WRAPPER(pthread_mutexattr_init, int,
+ (pthread_mutexattr_t *attr));
+
+DECLARE_WRAPPER(pthread_self, pthread_t, (void));
+
+#endif /* SPDK_INTERNAL_MOCK_H */
diff --git a/src/spdk/include/spdk_internal/sock.h b/src/spdk/include/spdk_internal/sock.h
new file mode 100644
index 00000000..d5542bf3
--- /dev/null
+++ b/src/spdk/include/spdk_internal/sock.h
@@ -0,0 +1,109 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * TCP network implementation abstraction layer
+ */
+
+#ifndef SPDK_INTERNAL_SOCK_H
+#define SPDK_INTERNAL_SOCK_H
+
+#include "spdk/stdinc.h"
+#include "spdk/sock.h"
+#include "spdk/queue.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define MAX_EVENTS_PER_POLL 32
+
+struct spdk_sock {
+ struct spdk_net_impl *net_impl;
+ spdk_sock_cb cb_fn;
+ void *cb_arg;
+ TAILQ_ENTRY(spdk_sock) link;
+};
+
+struct spdk_sock_group {
+ STAILQ_HEAD(, spdk_sock_group_impl) group_impls;
+};
+
+struct spdk_sock_group_impl {
+ struct spdk_net_impl *net_impl;
+ TAILQ_HEAD(, spdk_sock) socks;
+ STAILQ_ENTRY(spdk_sock_group_impl) link;
+};
+
+struct spdk_net_impl {
+ const char *name;
+
+ int (*getaddr)(struct spdk_sock *sock, char *saddr, int slen, uint16_t *sport, char *caddr,
+ int clen, uint16_t *cport);
+ struct spdk_sock *(*connect)(const char *ip, int port);
+ struct spdk_sock *(*listen)(const char *ip, int port);
+ struct spdk_sock *(*accept)(struct spdk_sock *sock);
+ int (*close)(struct spdk_sock *sock);
+ ssize_t (*recv)(struct spdk_sock *sock, void *buf, size_t len);
+ ssize_t (*writev)(struct spdk_sock *sock, struct iovec *iov, int iovcnt);
+
+ int (*set_recvlowat)(struct spdk_sock *sock, int nbytes);
+ int (*set_recvbuf)(struct spdk_sock *sock, int sz);
+ int (*set_sendbuf)(struct spdk_sock *sock, int sz);
+
+ bool (*is_ipv6)(struct spdk_sock *sock);
+ bool (*is_ipv4)(struct spdk_sock *sock);
+
+ struct spdk_sock_group_impl *(*group_impl_create)(void);
+ int (*group_impl_add_sock)(struct spdk_sock_group_impl *group, struct spdk_sock *sock);
+ int (*group_impl_remove_sock)(struct spdk_sock_group_impl *group, struct spdk_sock *sock);
+ int (*group_impl_poll)(struct spdk_sock_group_impl *group, int max_events,
+ struct spdk_sock **socks);
+ int (*group_impl_close)(struct spdk_sock_group_impl *group);
+
+ STAILQ_ENTRY(spdk_net_impl) link;
+};
+
+void spdk_net_impl_register(struct spdk_net_impl *impl);
+
+#define SPDK_NET_IMPL_REGISTER(name, impl) \
+static void __attribute__((constructor)) net_impl_register_##name(void) \
+{ \
+ spdk_net_impl_register(impl); \
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SPDK_INTERNAL_SOCK_H */
diff --git a/src/spdk/include/spdk_internal/utf.h b/src/spdk/include/spdk_internal/utf.h
new file mode 100644
index 00000000..b2b1c3c4
--- /dev/null
+++ b/src/spdk/include/spdk_internal/utf.h
@@ -0,0 +1,325 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef SPDK_UTF_H_
+#define SPDK_UTF_H_
+
+#include "spdk/stdinc.h"
+
+#include "spdk/endian.h"
+#include "spdk/likely.h"
+#include "spdk/string.h"
+
+static inline bool
+utf8_tail(uint8_t c)
+{
+ /* c >= 0x80 && c <= 0xBF, or binary 01xxxxxx */
+ return (c & 0xC0) == 0x80;
+}
+
+/*
+ * Check for a valid UTF-8 encoding of a single codepoint.
+ *
+ * \return Length of valid UTF-8 byte sequence, or negative if invalid.
+ */
+static inline int
+utf8_valid(const uint8_t *start, const uint8_t *end)
+{
+ const uint8_t *p = start;
+ uint8_t b0, b1, b2, b3;
+
+ if (p == end) {
+ return 0;
+ }
+
+ b0 = *p;
+
+ if (b0 <= 0x7F) {
+ return 1;
+ }
+
+ if (b0 <= 0xC1) {
+ /* Invalid start byte */
+ return -1;
+ }
+
+ if (++p == end) {
+ /* Not enough bytes left */
+ return -1;
+ }
+ b1 = *p;
+
+ if (b0 <= 0xDF) {
+ /* C2..DF 80..BF */
+ if (!utf8_tail(b1)) {
+ return -1;
+ }
+ return 2;
+ }
+
+ if (++p == end) {
+ /* Not enough bytes left */
+ return -1;
+ }
+ b2 = *p;
+
+ if (b0 == 0xE0) {
+ /* E0 A0..BF 80..BF */
+ if (b1 < 0xA0 || b1 > 0xBF || !utf8_tail(b2)) {
+ return -1;
+ }
+ return 3;
+ } else if (b0 == 0xED && b1 >= 0xA0) {
+ /*
+ * UTF-16 surrogate pairs use U+D800..U+DFFF, which would be encoded as
+ * ED A0..BF 80..BF in UTF-8; however, surrogate pairs are not allowed in UTF-8.
+ */
+ return -1;
+ } else if (b0 <= 0xEF) {
+ /* E1..EF 80..BF 80..BF */
+ if (!utf8_tail(b1) || !utf8_tail(b2)) {
+ return -1;
+ }
+ return 3;
+ }
+
+ if (++p == end) {
+ /* Not enough bytes left */
+ return -1;
+ }
+ b3 = *p;
+
+ if (b0 == 0xF0) {
+ /* F0 90..BF 80..BF 80..BF */
+ if (b1 < 0x90 || b1 > 0xBF || !utf8_tail(b2) || !utf8_tail(b3)) {
+ return -1;
+ }
+ return 4;
+ } else if (b0 <= 0xF3) {
+ /* F1..F3 80..BF 80..BF 80..BF */
+ if (!utf8_tail(b1) || !utf8_tail(b2) || !utf8_tail(b3)) {
+ return -1;
+ }
+ return 4;
+ } else if (b0 == 0xF4) {
+ /* F4 80..8F 80..BF 80..BF */
+ if (b1 < 0x80 || b1 > 0x8F || !utf8_tail(b2) || !utf8_tail(b3)) {
+ return -1;
+ }
+ return 4;
+ }
+
+ return -1;
+}
+
+static inline uint32_t
+utf8_decode_unsafe_1(const uint8_t *data)
+{
+ return data[0];
+}
+
+static inline uint32_t
+utf8_decode_unsafe_2(const uint8_t *data)
+{
+ uint32_t codepoint;
+
+ codepoint = ((data[0] & 0x1F) << 6);
+ codepoint |= (data[1] & 0x3F);
+
+ return codepoint;
+}
+
+static inline uint32_t
+utf8_decode_unsafe_3(const uint8_t *data)
+{
+ uint32_t codepoint;
+
+ codepoint = ((data[0] & 0x0F) << 12);
+ codepoint |= (data[1] & 0x3F) << 6;
+ codepoint |= (data[2] & 0x3F);
+
+ return codepoint;
+}
+
+static inline uint32_t
+utf8_decode_unsafe_4(const uint8_t *data)
+{
+ uint32_t codepoint;
+
+ codepoint = ((data[0] & 0x07) << 18);
+ codepoint |= (data[1] & 0x3F) << 12;
+ codepoint |= (data[2] & 0x3F) << 6;
+ codepoint |= (data[3] & 0x3F);
+
+ return codepoint;
+}
+
+/*
+ * Encode a single Unicode codepoint as UTF-8.
+ *
+ * buf must have at least 4 bytes of space available (hence unsafe).
+ *
+ * \return Number of bytes appended to buf, or negative if encoding failed.
+ */
+static inline int
+utf8_encode_unsafe(uint8_t *buf, uint32_t c)
+{
+ if (c <= 0x7F) {
+ buf[0] = c;
+ return 1;
+ } else if (c <= 0x7FF) {
+ buf[0] = 0xC0 | (c >> 6);
+ buf[1] = 0x80 | (c & 0x3F);
+ return 2;
+ } else if (c >= 0xD800 && c <= 0xDFFF) {
+ /* UTF-16 surrogate pairs - invalid in UTF-8 */
+ return -1;
+ } else if (c <= 0xFFFF) {
+ buf[0] = 0xE0 | (c >> 12);
+ buf[1] = 0x80 | ((c >> 6) & 0x3F);
+ buf[2] = 0x80 | (c & 0x3F);
+ return 3;
+ } else if (c <= 0x10FFFF) {
+ buf[0] = 0xF0 | (c >> 18);
+ buf[1] = 0x80 | ((c >> 12) & 0x3F);
+ buf[2] = 0x80 | ((c >> 6) & 0x3F);
+ buf[3] = 0x80 | (c & 0x3F);
+ return 4;
+ }
+ return -1;
+}
+
+static inline int
+utf8_codepoint_len(uint32_t c)
+{
+ if (c <= 0x7F) {
+ return 1;
+ } else if (c <= 0x7FF) {
+ return 2;
+ } else if (c >= 0xD800 && c <= 0xDFFF) {
+ /* UTF-16 surrogate pairs - invalid in UTF-8 */
+ return -1;
+ } else if (c <= 0xFFFF) {
+ return 3;
+ } else if (c <= 0x10FFFF) {
+ return 4;
+ }
+ return -1;
+}
+
+static inline bool
+utf16_valid_surrogate_high(uint32_t val)
+{
+ return val >= 0xD800 && val <= 0xDBFF;
+}
+
+static inline bool
+utf16_valid_surrogate_low(uint32_t val)
+{
+ return val >= 0xDC00 && val <= 0xDFFF;
+}
+
+/*
+ * Check for a valid UTF-16LE encoding of a single codepoint.
+ *
+ * \return Length of valid UTF-16LE sequence in 16-bit code units, or negative if invalid.
+ */
+static inline int
+utf16le_valid(const uint16_t *start, const uint16_t *end)
+{
+ const uint16_t *p = start;
+ uint16_t high, low;
+
+ if (p == end) {
+ return 0;
+ }
+
+ high = from_le16(p);
+
+ if (high <= 0xD7FF || high >= 0xE000) {
+ /* Single code unit in BMP */
+ return 1;
+ }
+
+ if (high >= 0xDC00) {
+ /* Low surrogate in first code unit - invalid */
+ return -1;
+ }
+
+ assert(utf16_valid_surrogate_high(high));
+
+ if (++p == end) {
+ /* Not enough code units left */
+ return -1;
+ }
+ low = from_le16(p);
+
+ if (!utf16_valid_surrogate_low(low)) {
+ return -1;
+ }
+
+ /* Valid surrogate pair */
+ return 2;
+}
+
+static inline uint32_t
+utf16_decode_surrogate_pair(uint32_t high, uint32_t low)
+{
+ uint32_t codepoint;
+
+ assert(utf16_valid_surrogate_high(high));
+ assert(utf16_valid_surrogate_low(low));
+
+ codepoint = low;
+ codepoint &= 0x3FF;
+ codepoint |= ((high & 0x3FF) << 10);
+ codepoint += 0x10000;
+
+ return codepoint;
+}
+
+static inline void
+utf16_encode_surrogate_pair(uint32_t codepoint, uint16_t *high, uint16_t *low)
+{
+ assert(codepoint >= 0x10000);
+ assert(codepoint <= 0x10FFFF);
+
+ codepoint -= 0x10000;
+ *high = 0xD800 | (codepoint >> 10);
+ *low = 0xDC00 | (codepoint & 0x3FF);
+
+ assert(utf16_valid_surrogate_high(*high));
+ assert(utf16_valid_surrogate_low(*low));
+}
+
+#endif
diff --git a/src/spdk/include/spdk_internal/virtio.h b/src/spdk/include/spdk_internal/virtio.h
new file mode 100644
index 00000000..77744029
--- /dev/null
+++ b/src/spdk/include/spdk_internal/virtio.h
@@ -0,0 +1,490 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef SPDK_VIRTIO_H
+#define SPDK_VIRTIO_H
+
+#include "spdk/stdinc.h"
+
+#include <linux/virtio_ring.h>
+#include <linux/virtio_pci.h>
+#include <linux/virtio_config.h>
+
+#include "spdk_internal/log.h"
+#include "spdk/likely.h"
+#include "spdk/queue.h"
+#include "spdk/json.h"
+#include "spdk/thread.h"
+#include "spdk/pci_ids.h"
+#include "spdk/env.h"
+
+#ifndef VHOST_USER_F_PROTOCOL_FEATURES
+#define VHOST_USER_F_PROTOCOL_FEATURES 30
+#endif
+
+/**
+ * The maximum virtqueue size is 2^15. Use that value as the end of
+ * descriptor chain terminator since it will never be a valid index
+ * in the descriptor table. This is used to verify we are correctly
+ * handling vq_free_cnt.
+ */
+#define VQ_RING_DESC_CHAIN_END 32768
+
+#define SPDK_VIRTIO_MAX_VIRTQUEUES 0x100
+
+/* Extra status define for readability */
+#define VIRTIO_CONFIG_S_RESET 0
+
+struct virtio_dev_ops;
+
+struct virtio_dev {
+ struct virtqueue **vqs;
+
+ /** Name of this virtio dev set by backend */
+ char *name;
+
+ /** Fixed number of backend-specific non-I/O virtqueues. */
+ uint16_t fixed_queues_num;
+
+ /** Max number of virtqueues the host supports. */
+ uint16_t max_queues;
+
+ /** Common device & guest features. */
+ uint64_t negotiated_features;
+
+ int is_hw;
+
+ /** Modern/legacy virtio device flag. */
+ uint8_t modern;
+
+ /** Mutex for asynchronous virtqueue-changing operations. */
+ pthread_mutex_t mutex;
+
+ /** Backend-specific callbacks. */
+ const struct virtio_dev_ops *backend_ops;
+
+ /** Context for the backend ops */
+ void *ctx;
+};
+
+struct virtio_dev_ops {
+ int (*read_dev_cfg)(struct virtio_dev *hw, size_t offset,
+ void *dst, int len);
+ int (*write_dev_cfg)(struct virtio_dev *hw, size_t offset,
+ const void *src, int len);
+ uint8_t (*get_status)(struct virtio_dev *hw);
+ void (*set_status)(struct virtio_dev *hw, uint8_t status);
+
+ /**
+ * Get device features. The features might be already
+ * negotiated with driver (guest) features.
+ */
+ uint64_t (*get_features)(struct virtio_dev *vdev);
+
+ /**
+ * Negotiate and set device features.
+ * The negotiation can fail with return code -1.
+ * This function should also set vdev->negotiated_features field.
+ */
+ int (*set_features)(struct virtio_dev *vdev, uint64_t features);
+
+ /** Destruct virtio device */
+ void (*destruct_dev)(struct virtio_dev *vdev);
+
+ uint16_t (*get_queue_size)(struct virtio_dev *vdev, uint16_t queue_id);
+ int (*setup_queue)(struct virtio_dev *hw, struct virtqueue *vq);
+ void (*del_queue)(struct virtio_dev *hw, struct virtqueue *vq);
+ void (*notify_queue)(struct virtio_dev *hw, struct virtqueue *vq);
+
+ void (*dump_json_info)(struct virtio_dev *hw, struct spdk_json_write_ctx *w);
+ void (*write_json_config)(struct virtio_dev *hw, struct spdk_json_write_ctx *w);
+};
+
+struct vq_desc_extra {
+ void *cookie;
+ uint16_t ndescs;
+};
+
+struct virtqueue {
+ struct virtio_dev *vdev; /**< owner of this virtqueue */
+ struct vring vq_ring; /**< vring keeping desc, used and avail */
+ /**
+ * Last consumed descriptor in the used table,
+ * trails vq_ring.used->idx.
+ */
+ uint16_t vq_used_cons_idx;
+ uint16_t vq_nentries; /**< vring desc numbers */
+ uint16_t vq_free_cnt; /**< num of desc available */
+ uint16_t vq_avail_idx; /**< sync until needed */
+
+ void *vq_ring_virt_mem; /**< virtual address of vring */
+ unsigned int vq_ring_size;
+
+ uint64_t vq_ring_mem; /**< physical address of vring */
+
+ /**
+ * Head of the free chain in the descriptor table. If
+ * there are no free descriptors, this will be set to
+ * VQ_RING_DESC_CHAIN_END.
+ */
+ uint16_t vq_desc_head_idx;
+
+ /**
+ * Tail of the free chain in desc table. If
+ * there are no free descriptors, this will be set to
+ * VQ_RING_DESC_CHAIN_END.
+ */
+ uint16_t vq_desc_tail_idx;
+ uint16_t vq_queue_index; /**< PCI queue index */
+ uint16_t *notify_addr;
+
+ /** Thread that's polling this queue. */
+ struct spdk_thread *owner_thread;
+
+ uint16_t req_start;
+ uint16_t req_end;
+ uint16_t reqs_finished;
+
+ struct vq_desc_extra vq_descx[0];
+};
+
+enum spdk_virtio_desc_type {
+ SPDK_VIRTIO_DESC_RO = 0, /**< Read only */
+ SPDK_VIRTIO_DESC_WR = VRING_DESC_F_WRITE, /**< Write only */
+ /* TODO VIRTIO_DESC_INDIRECT */
+};
+
+/** Context for creating PCI virtio_devs */
+struct virtio_pci_ctx;
+
+/**
+ * Callback for creating virtio_dev from a PCI device.
+ * \param pci_ctx PCI context to be associated with a virtio_dev
+ * \param ctx context provided by the user
+ * \return 0 on success, -1 on error.
+ */
+typedef int (*virtio_pci_create_cb)(struct virtio_pci_ctx *pci_ctx, void *ctx);
+
+uint16_t virtio_recv_pkts(struct virtqueue *vq, void **io, uint32_t *len, uint16_t io_cnt);
+
+/**
+ * Start a new request on the current vring head position and associate it
+ * with an opaque cookie object. The previous request in given vq will be
+ * made visible to the device in hopes it can be processed early, but there's
+ * no guarantee it will be until the device is notified with \c
+ * virtqueue_req_flush. This behavior is simply an optimization and virtqueues
+ * must always be flushed. Empty requests (with no descriptors added) will be
+ * ignored. The device owning given virtqueue must be started.
+ *
+ * \param vq virtio queue
+ * \param cookie opaque object to associate with this request. Once the request
+ * is sent, processed and a response is received, the same object will be
+ * returned to the user after calling the virtio poll API.
+ * \param iovcnt number of required iovectors for the request. This can be
+ * higher than than the actual number of iovectors to be added.
+ * \return 0 on success or negative errno otherwise. If the `iovcnt` is
+ * greater than virtqueue depth, -EINVAL is returned. If simply not enough
+ * iovectors are available, -ENOMEM is returned.
+ */
+int virtqueue_req_start(struct virtqueue *vq, void *cookie, int iovcnt);
+
+/**
+ * Flush a virtqueue. This will notify the device if it's required.
+ * The device owning given virtqueue must be started.
+ *
+ * \param vq virtio queue
+ */
+void virtqueue_req_flush(struct virtqueue *vq);
+
+/**
+ * Abort the very last request in a virtqueue. This will restore virtqueue
+ * state to the point before the last request was created. Note that this
+ * is only effective if a queue hasn't been flushed yet. The device owning
+ * given virtqueue must be started.
+ *
+ * \param vq virtio queue
+ */
+void virtqueue_req_abort(struct virtqueue *vq);
+
+/**
+ * Add iovec chain to the last created request. This call does not provide any
+ * error-checking. The caller has to ensure that he doesn't add more iovs than
+ * what was specified during request creation. The device owning given virtqueue
+ * must be started.
+ *
+ * \param vq virtio queue
+ * \param iovs iovec array
+ * \param iovcnt number of iovs in iovec array
+ * \param desc_type type of all given iovectors
+ */
+void virtqueue_req_add_iovs(struct virtqueue *vq, struct iovec *iovs, uint16_t iovcnt,
+ enum spdk_virtio_desc_type desc_type);
+
+/**
+ * Construct a virtio device. The device will be in stopped state by default.
+ * Before doing any I/O, it has to be manually started via \c virtio_dev_restart.
+ *
+ * \param vdev memory for virtio device, must be zeroed
+ * \param name name for the virtio device
+ * \param ops backend callbacks
+ * \param ops_ctx argument for the backend callbacks
+ * \return zero on success, or negative error code otherwise
+ */
+int virtio_dev_construct(struct virtio_dev *vdev, const char *name,
+ const struct virtio_dev_ops *ops, void *ops_ctx);
+
+/**
+ * Reset the device and prepare it to be `virtio_dev_start`ed. This call
+ * will also renegotiate feature flags.
+ *
+ * \param vdev virtio device
+ * \param req_features features this driver supports. A VIRTIO_F_VERSION_1
+ * flag will be automatically appended, as legacy devices are not supported.
+ */
+int virtio_dev_reset(struct virtio_dev *vdev, uint64_t req_features);
+
+/**
+ * Notify the host to start processing this virtio device. This is
+ * a blocking call that won't return until the host has started.
+ * This will also allocate virtqueues.
+ *
+ * \param vdev virtio device
+ * \param max_queues number of queues to allocate. The max number of
+ * usable I/O queues is also limited by the host device. `vdev` will be
+ * started successfully even if the host supports less queues than requested.
+ * \param fixed_queue_num number of queues preceeding the first
+ * request queue. For Virtio-SCSI this is equal to 2, as there are
+ * additional event and control queues.
+ */
+int virtio_dev_start(struct virtio_dev *vdev, uint16_t max_queues,
+ uint16_t fixed_queues_num);
+
+/**
+ * Stop the host from processing the device. This is a blocking call
+ * that won't return until all outstanding I/O has been processed on
+ * the host (virtio device) side. In order to re-start the device, it
+ * has to be `virtio_dev_reset` first.
+ *
+ * \param vdev virtio device
+ */
+void virtio_dev_stop(struct virtio_dev *vdev);
+
+/**
+ * Destruct a virtio device. Note that it must be in the stopped state.
+ * The virtio_dev should be manually freed afterwards.
+ *
+ * \param vdev virtio device
+ */
+void virtio_dev_destruct(struct virtio_dev *vdev);
+
+/**
+ * Bind a virtqueue with given index to the current thread;
+ *
+ * This function is thread-safe.
+ *
+ * \param vdev vhost device
+ * \param index virtqueue index
+ * \return 0 on success, -1 in case a virtqueue with given index either
+ * does not exists or is already acquired.
+ */
+int virtio_dev_acquire_queue(struct virtio_dev *vdev, uint16_t index);
+
+/**
+ * Look for unused queue and bind it to the current thread. This will
+ * scan the queues in range from *start_index* (inclusive) up to
+ * vdev->max_queues (exclusive).
+ *
+ * This function is thread-safe.
+ *
+ * \param vdev vhost device
+ * \param start_index virtqueue index to start looking from
+ * \return index of acquired queue or -1 in case no unused queue in given range
+ * has been found
+ */
+int32_t virtio_dev_find_and_acquire_queue(struct virtio_dev *vdev, uint16_t start_index);
+
+/**
+ * Get thread that acquired given virtqueue.
+ *
+ * This function is thread-safe.
+ *
+ * \param vdev vhost device
+ * \param index index of virtqueue
+ * \return thread that acquired given virtqueue. If the queue is unused
+ * or doesn't exist a NULL is returned.
+ */
+struct spdk_thread *virtio_dev_queue_get_thread(struct virtio_dev *vdev, uint16_t index);
+
+/**
+ * Check if virtqueue with given index is acquired.
+ *
+ * This function is thread-safe.
+ *
+ * \param vdev vhost device
+ * \param index index of virtqueue
+ * \return virtqueue acquire status. in case of invalid index *false* is returned.
+ */
+bool virtio_dev_queue_is_acquired(struct virtio_dev *vdev, uint16_t index);
+
+/**
+ * Release previously acquired queue.
+ *
+ * This function must be called from the thread that acquired the queue.
+ *
+ * \param vdev vhost device
+ * \param index index of virtqueue to release
+ */
+void virtio_dev_release_queue(struct virtio_dev *vdev, uint16_t index);
+
+/**
+ * Get Virtio status flags.
+ *
+ * \param vdev virtio device
+ */
+uint8_t virtio_dev_get_status(struct virtio_dev *vdev);
+
+/**
+ * Set Virtio status flag. The flags have to be set in very specific order
+ * defined the VIRTIO 1.0 spec section 3.1.1. To unset the flags, stop the
+ * device or set \c VIRTIO_CONFIG_S_RESET status flag. There is no way to
+ * unset only particular flags.
+ *
+ * \param vdev virtio device
+ * \param flag flag to set
+ */
+void virtio_dev_set_status(struct virtio_dev *vdev, uint8_t flag);
+
+/**
+ * Write raw data into the device config at given offset. This call does not
+ * provide any error checking.
+ *
+ * \param vdev virtio device
+ * \param offset offset in bytes
+ * \param src pointer to data to copy from
+ * \param len length of data to copy in bytes
+ * \return 0 on success, negative errno otherwise
+ */
+int virtio_dev_write_dev_config(struct virtio_dev *vdev, size_t offset, const void *src, int len);
+
+/**
+ * Read raw data from the device config at given offset. This call does not
+ * provide any error checking.
+ *
+ * \param vdev virtio device
+ * \param offset offset in bytes
+ * \param dst pointer to buffer to copy data into
+ * \param len length of data to copy in bytes
+ * \return 0 on success, negative errno otherwise
+ */
+int virtio_dev_read_dev_config(struct virtio_dev *vdev, size_t offset, void *dst, int len);
+
+/**
+ * Get backend-specific ops for given device.
+ *
+ * \param vdev virtio device
+ */
+const struct virtio_dev_ops *virtio_dev_backend_ops(struct virtio_dev *vdev);
+
+/**
+ * Check if the device has negotiated given feature bit.
+ *
+ * \param vdev virtio device
+ * \param bit feature bit
+ */
+static inline bool
+virtio_dev_has_feature(struct virtio_dev *vdev, uint64_t bit)
+{
+ return !!(vdev->negotiated_features & (1ULL << bit));
+}
+
+/**
+ * Dump all device specific information into given json stream.
+ *
+ * \param vdev virtio device
+ * \param w json stream
+ */
+void virtio_dev_dump_json_info(struct virtio_dev *vdev, struct spdk_json_write_ctx *w);
+
+/**
+ * Enumerate all PCI Virtio devices of given type on the system.
+ *
+ * \param enum_cb a function to be called for each valid PCI device.
+ * If a virtio_dev is has been created, the callback should return 0.
+ * Returning any other value will cause the PCI context to be freed,
+ * making it unusable.
+ * \param enum_ctx additional opaque context to be passed into `enum_cb`
+ * \param pci_device_id PCI Device ID of devices to iterate through
+ */
+int virtio_pci_dev_enumerate(virtio_pci_create_cb enum_cb, void *enum_ctx,
+ uint16_t pci_device_id);
+
+/**
+ * Attach a PCI Virtio device of given type.
+ *
+ * \param create_cb callback to create a virtio_dev.
+ * If virtio_dev is has been created, the callback should return 0.
+ * Returning any other value will cause the PCI context to be freed,
+ * making it unusable.
+ * \param enum_ctx additional opaque context to be passed into `enum_cb`
+ * \param pci_device_id PCI Device ID of devices to iterate through
+ * \param pci_addr PCI address of the device to attach
+ */
+int virtio_pci_dev_attach(virtio_pci_create_cb create_cb, void *enum_ctx,
+ uint16_t pci_device_id, struct spdk_pci_addr *pci_addr);
+
+/**
+ * Connect to a vhost-user device and init corresponding virtio_dev struct.
+ * The virtio_dev will have to be freed with \c virtio_dev_free.
+ *
+ * \param vdev preallocated vhost device struct to operate on
+ * \param name name of this virtio device
+ * \param path path to the Unix domain socket of the vhost-user device
+ * \param queue_size size of each of the queues
+ * \return virtio device
+ */
+int virtio_user_dev_init(struct virtio_dev *vdev, const char *name, const char *path,
+ uint32_t queue_size);
+
+/**
+ * Initialize virtio_dev for a given PCI device.
+ * The virtio_dev has to be freed with \c virtio_dev_destruct.
+ *
+ * \param vdev preallocated vhost device struct to operate on
+ * \param name name of this virtio device
+ * \param pci_ctx context of the PCI device
+ * \return 0 on success, -1 on error.
+ */
+int virtio_pci_dev_init(struct virtio_dev *vdev, const char *name,
+ struct virtio_pci_ctx *pci_ctx);
+
+#endif /* SPDK_VIRTIO_H */