diff options
Diffstat (limited to 'src/spdk/include/spdk_internal')
| -rw-r--r-- | src/spdk/include/spdk_internal/assert.h | 55 | ||||
| -rw-r--r-- | src/spdk/include/spdk_internal/copy_engine.h | 95 | ||||
| -rw-r--r-- | src/spdk/include/spdk_internal/event.h | 133 | ||||
| -rw-r--r-- | src/spdk/include/spdk_internal/log.h | 101 | ||||
| -rw-r--r-- | src/spdk/include/spdk_internal/lvolstore.h | 124 | ||||
| -rw-r--r-- | src/spdk/include/spdk_internal/mock.h | 117 | ||||
| -rw-r--r-- | src/spdk/include/spdk_internal/sock.h | 109 | ||||
| -rw-r--r-- | src/spdk/include/spdk_internal/utf.h | 325 | ||||
| -rw-r--r-- | src/spdk/include/spdk_internal/virtio.h | 490 |
9 files changed, 1549 insertions, 0 deletions
diff --git a/src/spdk/include/spdk_internal/assert.h b/src/spdk/include/spdk_internal/assert.h new file mode 100644 index 00000000..7e4c4507 --- /dev/null +++ b/src/spdk/include/spdk_internal/assert.h @@ -0,0 +1,55 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef SPDK_INTERNAL_ASSERT_H +#define SPDK_INTERNAL_ASSERT_H + +#include "spdk/stdinc.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#include "spdk/assert.h" + +#if !defined(DEBUG) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5)) +#define SPDK_UNREACHABLE() __builtin_unreachable() +#else +#define SPDK_UNREACHABLE() abort() +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* SPDK_INTERNAL_ASSERT_H */ diff --git a/src/spdk/include/spdk_internal/copy_engine.h b/src/spdk/include/spdk_internal/copy_engine.h new file mode 100644 index 00000000..6b8a13c4 --- /dev/null +++ b/src/spdk/include/spdk_internal/copy_engine.h @@ -0,0 +1,95 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef SPDK_INTERNAL_COPY_ENGINE_H +#define SPDK_INTERNAL_COPY_ENGINE_H + +#include "spdk/stdinc.h" + +#include "spdk/copy_engine.h" +#include "spdk/queue.h" + +struct spdk_copy_task { + spdk_copy_completion_cb cb; + uint8_t offload_ctx[0]; +}; + +struct spdk_copy_engine { + int (*copy)(void *cb_arg, struct spdk_io_channel *ch, void *dst, void *src, + uint64_t nbytes, spdk_copy_completion_cb cb); + int (*fill)(void *cb_arg, struct spdk_io_channel *ch, void *dst, uint8_t fill, + uint64_t nbytes, spdk_copy_completion_cb cb); + struct spdk_io_channel *(*get_io_channel)(void); +}; + +struct spdk_copy_module_if { + /** Initialization function for the module. Called by the spdk + * application during startup. + * + * Modules are required to define this function. + */ + int (*module_init)(void); + + /** Finish function for the module. Called by the spdk application + * before the spdk application exits to perform any necessary cleanup. + * + * Modules are not required to define this function. + */ + void (*module_fini)(void *ctx); + + /** Function called to return a text string representing the + * module's configuration options for inclusion in an + * spdk configuration file. + */ + void (*config_text)(FILE *fp); + + size_t (*get_ctx_size)(void); + TAILQ_ENTRY(spdk_copy_module_if) tailq; +}; + +void spdk_copy_engine_register(struct spdk_copy_engine *copy_engine); +void spdk_copy_module_list_add(struct spdk_copy_module_if *copy_module); + +#define SPDK_COPY_MODULE_REGISTER(init_fn, fini_fn, config_fn, ctx_size_fn) \ + static struct spdk_copy_module_if init_fn ## _if = { \ + .module_init = init_fn, \ + .module_fini = fini_fn, \ + .config_text = config_fn, \ + .get_ctx_size = ctx_size_fn, \ + }; \ + __attribute__((constructor)) static void init_fn ## _init(void) \ + { \ + spdk_copy_module_list_add(&init_fn ## _if); \ + } + +#endif diff --git a/src/spdk/include/spdk_internal/event.h b/src/spdk/include/spdk_internal/event.h new file mode 100644 index 00000000..25bdc04e --- /dev/null +++ b/src/spdk/include/spdk_internal/event.h @@ -0,0 +1,133 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef SPDK_INTERNAL_EVENT_H +#define SPDK_INTERNAL_EVENT_H + +#include "spdk/stdinc.h" + +#include "spdk/event.h" +#include "spdk/json.h" + +struct spdk_event { + uint32_t lcore; + spdk_event_fn fn; + void *arg1; + void *arg2; +}; + +int spdk_reactors_init(unsigned int max_delay_us); +void spdk_reactors_fini(void); + +void spdk_reactors_start(void); +void spdk_reactors_stop(void *arg1, void *arg2); + +struct spdk_subsystem { + const char *name; + /* User must call spdk_subsystem_init_next() when they are done with their initialization. */ + void (*init)(void); + void (*fini)(void); + void (*config)(FILE *fp); + + /** + * Write JSON configuration handler. + * + * \param w JSON write context + * \param done_ev Done event to be called when writing is done. + */ + void (*write_config_json)(struct spdk_json_write_ctx *w, struct spdk_event *done_ev); + TAILQ_ENTRY(spdk_subsystem) tailq; +}; + +TAILQ_HEAD(spdk_subsystem_list, spdk_subsystem); +extern struct spdk_subsystem_list g_subsystems; + +struct spdk_subsystem *spdk_subsystem_find(struct spdk_subsystem_list *list, const char *name); + +struct spdk_subsystem_depend { + const char *name; + const char *depends_on; + TAILQ_ENTRY(spdk_subsystem_depend) tailq; +}; + +TAILQ_HEAD(spdk_subsystem_depend_list, spdk_subsystem_depend); +extern struct spdk_subsystem_depend_list g_subsystems_deps; + +void spdk_add_subsystem(struct spdk_subsystem *subsystem); +void spdk_add_subsystem_depend(struct spdk_subsystem_depend *depend); + +void spdk_subsystem_init(struct spdk_event *app_start_event); +void spdk_subsystem_fini(struct spdk_event *app_finish_event); +void spdk_subsystem_init_next(int rc); +void spdk_subsystem_fini_next(void); +void spdk_subsystem_config(FILE *fp); + +/** + * Save pointed \c subsystem configuration to the JSON write context \c w. In case of + * error \c null is written to the JSON context. Writing might be done in async way + * so caller need to pass event that subsystem will call when it finish writing + * configuration. + * + * \param w JSON write context + * \param subsystem the subsystem to query + * \param done_ev event to be called when writing is done + */ +void spdk_subsystem_config_json(struct spdk_json_write_ctx *w, struct spdk_subsystem *subsystem, + struct spdk_event *done_ev); + +void spdk_rpc_initialize(const char *listen_addr); +void spdk_rpc_finish(void); + +/** + * \brief Register a new subsystem + */ +#define SPDK_SUBSYSTEM_REGISTER(_name) \ + __attribute__((constructor)) static void _name ## _register(void) \ + { \ + spdk_add_subsystem(&_name); \ + } + +/** + * \brief Declare that a subsystem depends on another subsystem. + */ +#define SPDK_SUBSYSTEM_DEPEND(_name, _depends_on) \ + static struct spdk_subsystem_depend __subsystem_ ## _name ## _depend_on ## _depends_on = { \ + .name = #_name, \ + .depends_on = #_depends_on, \ + }; \ + __attribute__((constructor)) static void _name ## _depend_on ## _depends_on(void) \ + { \ + spdk_add_subsystem_depend(&__subsystem_ ## _name ## _depend_on ## _depends_on); \ + } + +#endif /* SPDK_INTERNAL_EVENT_H */ diff --git a/src/spdk/include/spdk_internal/log.h b/src/spdk/include/spdk_internal/log.h new file mode 100644 index 00000000..c91116bb --- /dev/null +++ b/src/spdk/include/spdk_internal/log.h @@ -0,0 +1,101 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * \file + * Logging interfaces + */ + +#ifndef SPDK_INTERNAL_LOG_H +#define SPDK_INTERNAL_LOG_H + +#include "spdk/log.h" +#include "spdk/queue.h" + +extern enum spdk_log_level g_spdk_log_level; +extern enum spdk_log_level g_spdk_log_print_level; +extern enum spdk_log_level g_spdk_log_backtrace_level; + +struct spdk_trace_flag { + TAILQ_ENTRY(spdk_trace_flag) tailq; + const char *name; + bool enabled; +}; + +void spdk_log_register_trace_flag(const char *name, struct spdk_trace_flag *flag); + +struct spdk_trace_flag *spdk_log_get_first_trace_flag(void); +struct spdk_trace_flag *spdk_log_get_next_trace_flag(struct spdk_trace_flag *flag); + +#define SPDK_LOG_REGISTER_COMPONENT(str, flag) \ +struct spdk_trace_flag flag = { \ + .enabled = false, \ + .name = str, \ +}; \ +__attribute__((constructor)) static void register_trace_flag_##flag(void) \ +{ \ + spdk_log_register_trace_flag(str, &flag); \ +} + +#define SPDK_INFOLOG(FLAG, ...) \ + do { \ + extern struct spdk_trace_flag FLAG; \ + if (FLAG.enabled) { \ + spdk_log(SPDK_LOG_INFO, __FILE__, __LINE__, __func__, __VA_ARGS__); \ + } \ + } while (0) + +#ifdef DEBUG + +#define SPDK_DEBUGLOG(FLAG, ...) \ + do { \ + extern struct spdk_trace_flag FLAG; \ + if (FLAG.enabled) { \ + spdk_log(SPDK_LOG_DEBUG, __FILE__, __LINE__, __func__, __VA_ARGS__); \ + } \ + } while (0) + +#define SPDK_TRACEDUMP(FLAG, LABEL, BUF, LEN) \ + do { \ + extern struct spdk_trace_flag FLAG; \ + if ((FLAG.enabled) && (LEN)) { \ + spdk_trace_dump(stderr, (LABEL), (BUF), (LEN)); \ + } \ + } while (0) + +#else +#define SPDK_DEBUGLOG(...) do { } while (0) +#define SPDK_TRACEDUMP(...) do { } while (0) +#endif + +#endif /* SPDK_INTERNAL_LOG_H */ diff --git a/src/spdk/include/spdk_internal/lvolstore.h b/src/spdk/include/spdk_internal/lvolstore.h new file mode 100644 index 00000000..064bad5d --- /dev/null +++ b/src/spdk/include/spdk_internal/lvolstore.h @@ -0,0 +1,124 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef SPDK_INTERNAL_LVOLSTORE_H +#define SPDK_INTERNAL_LVOLSTORE_H + +#include "spdk/blob.h" +#include "spdk/lvol.h" +#include "spdk/uuid.h" +#include "spdk/bdev_module.h" + +/* Default size of blobstore cluster */ +#define SPDK_LVS_OPTS_CLUSTER_SZ (4 * 1024 * 1024) + +struct spdk_lvs_req { + spdk_lvs_op_complete cb_fn; + void *cb_arg; + struct spdk_lvol_store *lvol_store; + int lvserrno; +}; + +struct spdk_lvol_req { + spdk_lvol_op_complete cb_fn; + void *cb_arg; + struct spdk_lvol *lvol; + size_t sz; + struct spdk_io_channel *channel; + char name[SPDK_LVOL_NAME_MAX]; +}; + +struct spdk_lvs_with_handle_req { + spdk_lvs_op_with_handle_complete cb_fn; + void *cb_arg; + struct spdk_lvol_store *lvol_store; + struct spdk_bs_dev *bs_dev; + struct spdk_bdev *base_bdev; + int lvserrno; +}; + +struct spdk_lvs_destroy_req { + spdk_lvs_op_complete cb_fn; + void *cb_arg; + struct spdk_lvol_store *lvs; +}; + +struct spdk_lvol_with_handle_req { + spdk_lvol_op_with_handle_complete cb_fn; + void *cb_arg; + struct spdk_lvol *lvol; +}; + +struct spdk_lvol_store { + struct spdk_bs_dev *bs_dev; + struct spdk_blob_store *blobstore; + struct spdk_blob *super_blob; + spdk_blob_id super_blob_id; + struct spdk_uuid uuid; + int lvol_count; + int lvols_opened; + bool destruct; + TAILQ_HEAD(, spdk_lvol) lvols; + TAILQ_HEAD(, spdk_lvol) pending_lvols; + bool on_list; + TAILQ_ENTRY(spdk_lvol_store) link; + char name[SPDK_LVS_NAME_MAX]; + char new_name[SPDK_LVS_NAME_MAX]; +}; + +struct spdk_lvol { + struct spdk_lvol_store *lvol_store; + struct spdk_blob *blob; + spdk_blob_id blob_id; + char *unique_id; + char name[SPDK_LVOL_NAME_MAX]; + struct spdk_uuid uuid; + char uuid_str[SPDK_UUID_STRING_LEN]; + bool thin_provision; + struct spdk_bdev *bdev; + int ref_count; + bool action_in_progress; + TAILQ_ENTRY(spdk_lvol) link; +}; + +struct lvol_task { + enum spdk_bdev_io_status status; +}; + +struct lvol_store_bdev *vbdev_lvol_store_first(void); +struct lvol_store_bdev *vbdev_lvol_store_next(struct lvol_store_bdev *prev); + +void spdk_lvol_resize(struct spdk_lvol *lvol, uint64_t sz, spdk_lvol_op_complete cb_fn, + void *cb_arg); + +#endif /* SPDK_INTERNAL_LVOLSTORE_H */ diff --git a/src/spdk/include/spdk_internal/mock.h b/src/spdk/include/spdk_internal/mock.h new file mode 100644 index 00000000..f5a366dd --- /dev/null +++ b/src/spdk/include/spdk_internal/mock.h @@ -0,0 +1,117 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef SPDK_INTERNAL_MOCK_H +#define SPDK_INTERNAL_MOCK_H + +#include "spdk/stdinc.h" + +#define MOCK_STRUCT_INIT(...) \ + { __VA_ARGS__ } + +#define DEFINE_RETURN_MOCK(fn, ret) \ + bool ut_ ## fn ## _mocked = false; \ + ret ut_ ## fn + +/* + * For controlling mocked function behavior, setting + * and getting values from the stub, the _P macros are + * for mocking functions that return pointer values. + */ +#define MOCK_SET(fn, val) \ + ut_ ## fn ## _mocked = true; \ + ut_ ## fn = val + +#define MOCK_GET(fn) \ + ut_ ## fn + +#define MOCK_CLEAR(fn) \ + ut_ ## fn ## _mocked = false; + +#define MOCK_CLEAR_P(fn) \ + ut_ ## fn ## _mocked = false; \ + ut_ ## fn = NULL; + +/* for declaring function protoypes for wrappers */ +#define DECLARE_WRAPPER(fn, ret, args) \ + extern bool ut_ ## fn ## _mocked; \ + extern ret ut_ ## fn; \ + ret __wrap_ ## fn args; ret __real_ ## fn args; + +/* for defining the implmentation of wrappers for syscalls */ +#define DEFINE_WRAPPER(fn, ret, dargs, pargs) \ + DEFINE_RETURN_MOCK(fn, ret); \ + __attribute__((used)) ret __wrap_ ## fn dargs \ + { \ + if (!ut_ ## fn ## _mocked) { \ + return __real_ ## fn pargs; \ + } else { \ + return MOCK_GET(fn); \ + } \ + } + +/* DEFINE_STUB is for defining the implmentation of stubs for SPDK funcs. */ +#define DEFINE_STUB(fn, ret, dargs, val) \ + bool ut_ ## fn ## _mocked = true; \ + ret ut_ ## fn = val; \ + ret fn dargs; \ + ret fn dargs \ + { \ + return MOCK_GET(fn); \ + } + +/* DEFINE_STUB_V macro is for stubs that don't have a return value */ +#define DEFINE_STUB_V(fn, dargs) \ + void fn dargs; \ + void fn dargs \ + { \ + } + +#define HANDLE_RETURN_MOCK(fn) \ + if (ut_ ## fn ## _mocked) { \ + return ut_ ## fn; \ + } + + +/* declare wrapper protos (alphabetically please) here */ +DECLARE_WRAPPER(calloc, void *, (size_t nmemb, size_t size)); + +DECLARE_WRAPPER(pthread_mutex_init, int, + (pthread_mutex_t *mtx, const pthread_mutexattr_t *attr)); + +DECLARE_WRAPPER(pthread_mutexattr_init, int, + (pthread_mutexattr_t *attr)); + +DECLARE_WRAPPER(pthread_self, pthread_t, (void)); + +#endif /* SPDK_INTERNAL_MOCK_H */ diff --git a/src/spdk/include/spdk_internal/sock.h b/src/spdk/include/spdk_internal/sock.h new file mode 100644 index 00000000..d5542bf3 --- /dev/null +++ b/src/spdk/include/spdk_internal/sock.h @@ -0,0 +1,109 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** \file + * TCP network implementation abstraction layer + */ + +#ifndef SPDK_INTERNAL_SOCK_H +#define SPDK_INTERNAL_SOCK_H + +#include "spdk/stdinc.h" +#include "spdk/sock.h" +#include "spdk/queue.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define MAX_EVENTS_PER_POLL 32 + +struct spdk_sock { + struct spdk_net_impl *net_impl; + spdk_sock_cb cb_fn; + void *cb_arg; + TAILQ_ENTRY(spdk_sock) link; +}; + +struct spdk_sock_group { + STAILQ_HEAD(, spdk_sock_group_impl) group_impls; +}; + +struct spdk_sock_group_impl { + struct spdk_net_impl *net_impl; + TAILQ_HEAD(, spdk_sock) socks; + STAILQ_ENTRY(spdk_sock_group_impl) link; +}; + +struct spdk_net_impl { + const char *name; + + int (*getaddr)(struct spdk_sock *sock, char *saddr, int slen, uint16_t *sport, char *caddr, + int clen, uint16_t *cport); + struct spdk_sock *(*connect)(const char *ip, int port); + struct spdk_sock *(*listen)(const char *ip, int port); + struct spdk_sock *(*accept)(struct spdk_sock *sock); + int (*close)(struct spdk_sock *sock); + ssize_t (*recv)(struct spdk_sock *sock, void *buf, size_t len); + ssize_t (*writev)(struct spdk_sock *sock, struct iovec *iov, int iovcnt); + + int (*set_recvlowat)(struct spdk_sock *sock, int nbytes); + int (*set_recvbuf)(struct spdk_sock *sock, int sz); + int (*set_sendbuf)(struct spdk_sock *sock, int sz); + + bool (*is_ipv6)(struct spdk_sock *sock); + bool (*is_ipv4)(struct spdk_sock *sock); + + struct spdk_sock_group_impl *(*group_impl_create)(void); + int (*group_impl_add_sock)(struct spdk_sock_group_impl *group, struct spdk_sock *sock); + int (*group_impl_remove_sock)(struct spdk_sock_group_impl *group, struct spdk_sock *sock); + int (*group_impl_poll)(struct spdk_sock_group_impl *group, int max_events, + struct spdk_sock **socks); + int (*group_impl_close)(struct spdk_sock_group_impl *group); + + STAILQ_ENTRY(spdk_net_impl) link; +}; + +void spdk_net_impl_register(struct spdk_net_impl *impl); + +#define SPDK_NET_IMPL_REGISTER(name, impl) \ +static void __attribute__((constructor)) net_impl_register_##name(void) \ +{ \ + spdk_net_impl_register(impl); \ +} + +#ifdef __cplusplus +} +#endif + +#endif /* SPDK_INTERNAL_SOCK_H */ diff --git a/src/spdk/include/spdk_internal/utf.h b/src/spdk/include/spdk_internal/utf.h new file mode 100644 index 00000000..b2b1c3c4 --- /dev/null +++ b/src/spdk/include/spdk_internal/utf.h @@ -0,0 +1,325 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef SPDK_UTF_H_ +#define SPDK_UTF_H_ + +#include "spdk/stdinc.h" + +#include "spdk/endian.h" +#include "spdk/likely.h" +#include "spdk/string.h" + +static inline bool +utf8_tail(uint8_t c) +{ + /* c >= 0x80 && c <= 0xBF, or binary 01xxxxxx */ + return (c & 0xC0) == 0x80; +} + +/* + * Check for a valid UTF-8 encoding of a single codepoint. + * + * \return Length of valid UTF-8 byte sequence, or negative if invalid. + */ +static inline int +utf8_valid(const uint8_t *start, const uint8_t *end) +{ + const uint8_t *p = start; + uint8_t b0, b1, b2, b3; + + if (p == end) { + return 0; + } + + b0 = *p; + + if (b0 <= 0x7F) { + return 1; + } + + if (b0 <= 0xC1) { + /* Invalid start byte */ + return -1; + } + + if (++p == end) { + /* Not enough bytes left */ + return -1; + } + b1 = *p; + + if (b0 <= 0xDF) { + /* C2..DF 80..BF */ + if (!utf8_tail(b1)) { + return -1; + } + return 2; + } + + if (++p == end) { + /* Not enough bytes left */ + return -1; + } + b2 = *p; + + if (b0 == 0xE0) { + /* E0 A0..BF 80..BF */ + if (b1 < 0xA0 || b1 > 0xBF || !utf8_tail(b2)) { + return -1; + } + return 3; + } else if (b0 == 0xED && b1 >= 0xA0) { + /* + * UTF-16 surrogate pairs use U+D800..U+DFFF, which would be encoded as + * ED A0..BF 80..BF in UTF-8; however, surrogate pairs are not allowed in UTF-8. + */ + return -1; + } else if (b0 <= 0xEF) { + /* E1..EF 80..BF 80..BF */ + if (!utf8_tail(b1) || !utf8_tail(b2)) { + return -1; + } + return 3; + } + + if (++p == end) { + /* Not enough bytes left */ + return -1; + } + b3 = *p; + + if (b0 == 0xF0) { + /* F0 90..BF 80..BF 80..BF */ + if (b1 < 0x90 || b1 > 0xBF || !utf8_tail(b2) || !utf8_tail(b3)) { + return -1; + } + return 4; + } else if (b0 <= 0xF3) { + /* F1..F3 80..BF 80..BF 80..BF */ + if (!utf8_tail(b1) || !utf8_tail(b2) || !utf8_tail(b3)) { + return -1; + } + return 4; + } else if (b0 == 0xF4) { + /* F4 80..8F 80..BF 80..BF */ + if (b1 < 0x80 || b1 > 0x8F || !utf8_tail(b2) || !utf8_tail(b3)) { + return -1; + } + return 4; + } + + return -1; +} + +static inline uint32_t +utf8_decode_unsafe_1(const uint8_t *data) +{ + return data[0]; +} + +static inline uint32_t +utf8_decode_unsafe_2(const uint8_t *data) +{ + uint32_t codepoint; + + codepoint = ((data[0] & 0x1F) << 6); + codepoint |= (data[1] & 0x3F); + + return codepoint; +} + +static inline uint32_t +utf8_decode_unsafe_3(const uint8_t *data) +{ + uint32_t codepoint; + + codepoint = ((data[0] & 0x0F) << 12); + codepoint |= (data[1] & 0x3F) << 6; + codepoint |= (data[2] & 0x3F); + + return codepoint; +} + +static inline uint32_t +utf8_decode_unsafe_4(const uint8_t *data) +{ + uint32_t codepoint; + + codepoint = ((data[0] & 0x07) << 18); + codepoint |= (data[1] & 0x3F) << 12; + codepoint |= (data[2] & 0x3F) << 6; + codepoint |= (data[3] & 0x3F); + + return codepoint; +} + +/* + * Encode a single Unicode codepoint as UTF-8. + * + * buf must have at least 4 bytes of space available (hence unsafe). + * + * \return Number of bytes appended to buf, or negative if encoding failed. + */ +static inline int +utf8_encode_unsafe(uint8_t *buf, uint32_t c) +{ + if (c <= 0x7F) { + buf[0] = c; + return 1; + } else if (c <= 0x7FF) { + buf[0] = 0xC0 | (c >> 6); + buf[1] = 0x80 | (c & 0x3F); + return 2; + } else if (c >= 0xD800 && c <= 0xDFFF) { + /* UTF-16 surrogate pairs - invalid in UTF-8 */ + return -1; + } else if (c <= 0xFFFF) { + buf[0] = 0xE0 | (c >> 12); + buf[1] = 0x80 | ((c >> 6) & 0x3F); + buf[2] = 0x80 | (c & 0x3F); + return 3; + } else if (c <= 0x10FFFF) { + buf[0] = 0xF0 | (c >> 18); + buf[1] = 0x80 | ((c >> 12) & 0x3F); + buf[2] = 0x80 | ((c >> 6) & 0x3F); + buf[3] = 0x80 | (c & 0x3F); + return 4; + } + return -1; +} + +static inline int +utf8_codepoint_len(uint32_t c) +{ + if (c <= 0x7F) { + return 1; + } else if (c <= 0x7FF) { + return 2; + } else if (c >= 0xD800 && c <= 0xDFFF) { + /* UTF-16 surrogate pairs - invalid in UTF-8 */ + return -1; + } else if (c <= 0xFFFF) { + return 3; + } else if (c <= 0x10FFFF) { + return 4; + } + return -1; +} + +static inline bool +utf16_valid_surrogate_high(uint32_t val) +{ + return val >= 0xD800 && val <= 0xDBFF; +} + +static inline bool +utf16_valid_surrogate_low(uint32_t val) +{ + return val >= 0xDC00 && val <= 0xDFFF; +} + +/* + * Check for a valid UTF-16LE encoding of a single codepoint. + * + * \return Length of valid UTF-16LE sequence in 16-bit code units, or negative if invalid. + */ +static inline int +utf16le_valid(const uint16_t *start, const uint16_t *end) +{ + const uint16_t *p = start; + uint16_t high, low; + + if (p == end) { + return 0; + } + + high = from_le16(p); + + if (high <= 0xD7FF || high >= 0xE000) { + /* Single code unit in BMP */ + return 1; + } + + if (high >= 0xDC00) { + /* Low surrogate in first code unit - invalid */ + return -1; + } + + assert(utf16_valid_surrogate_high(high)); + + if (++p == end) { + /* Not enough code units left */ + return -1; + } + low = from_le16(p); + + if (!utf16_valid_surrogate_low(low)) { + return -1; + } + + /* Valid surrogate pair */ + return 2; +} + +static inline uint32_t +utf16_decode_surrogate_pair(uint32_t high, uint32_t low) +{ + uint32_t codepoint; + + assert(utf16_valid_surrogate_high(high)); + assert(utf16_valid_surrogate_low(low)); + + codepoint = low; + codepoint &= 0x3FF; + codepoint |= ((high & 0x3FF) << 10); + codepoint += 0x10000; + + return codepoint; +} + +static inline void +utf16_encode_surrogate_pair(uint32_t codepoint, uint16_t *high, uint16_t *low) +{ + assert(codepoint >= 0x10000); + assert(codepoint <= 0x10FFFF); + + codepoint -= 0x10000; + *high = 0xD800 | (codepoint >> 10); + *low = 0xDC00 | (codepoint & 0x3FF); + + assert(utf16_valid_surrogate_high(*high)); + assert(utf16_valid_surrogate_low(*low)); +} + +#endif diff --git a/src/spdk/include/spdk_internal/virtio.h b/src/spdk/include/spdk_internal/virtio.h new file mode 100644 index 00000000..77744029 --- /dev/null +++ b/src/spdk/include/spdk_internal/virtio.h @@ -0,0 +1,490 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef SPDK_VIRTIO_H +#define SPDK_VIRTIO_H + +#include "spdk/stdinc.h" + +#include <linux/virtio_ring.h> +#include <linux/virtio_pci.h> +#include <linux/virtio_config.h> + +#include "spdk_internal/log.h" +#include "spdk/likely.h" +#include "spdk/queue.h" +#include "spdk/json.h" +#include "spdk/thread.h" +#include "spdk/pci_ids.h" +#include "spdk/env.h" + +#ifndef VHOST_USER_F_PROTOCOL_FEATURES +#define VHOST_USER_F_PROTOCOL_FEATURES 30 +#endif + +/** + * The maximum virtqueue size is 2^15. Use that value as the end of + * descriptor chain terminator since it will never be a valid index + * in the descriptor table. This is used to verify we are correctly + * handling vq_free_cnt. + */ +#define VQ_RING_DESC_CHAIN_END 32768 + +#define SPDK_VIRTIO_MAX_VIRTQUEUES 0x100 + +/* Extra status define for readability */ +#define VIRTIO_CONFIG_S_RESET 0 + +struct virtio_dev_ops; + +struct virtio_dev { + struct virtqueue **vqs; + + /** Name of this virtio dev set by backend */ + char *name; + + /** Fixed number of backend-specific non-I/O virtqueues. */ + uint16_t fixed_queues_num; + + /** Max number of virtqueues the host supports. */ + uint16_t max_queues; + + /** Common device & guest features. */ + uint64_t negotiated_features; + + int is_hw; + + /** Modern/legacy virtio device flag. */ + uint8_t modern; + + /** Mutex for asynchronous virtqueue-changing operations. */ + pthread_mutex_t mutex; + + /** Backend-specific callbacks. */ + const struct virtio_dev_ops *backend_ops; + + /** Context for the backend ops */ + void *ctx; +}; + +struct virtio_dev_ops { + int (*read_dev_cfg)(struct virtio_dev *hw, size_t offset, + void *dst, int len); + int (*write_dev_cfg)(struct virtio_dev *hw, size_t offset, + const void *src, int len); + uint8_t (*get_status)(struct virtio_dev *hw); + void (*set_status)(struct virtio_dev *hw, uint8_t status); + + /** + * Get device features. The features might be already + * negotiated with driver (guest) features. + */ + uint64_t (*get_features)(struct virtio_dev *vdev); + + /** + * Negotiate and set device features. + * The negotiation can fail with return code -1. + * This function should also set vdev->negotiated_features field. + */ + int (*set_features)(struct virtio_dev *vdev, uint64_t features); + + /** Destruct virtio device */ + void (*destruct_dev)(struct virtio_dev *vdev); + + uint16_t (*get_queue_size)(struct virtio_dev *vdev, uint16_t queue_id); + int (*setup_queue)(struct virtio_dev *hw, struct virtqueue *vq); + void (*del_queue)(struct virtio_dev *hw, struct virtqueue *vq); + void (*notify_queue)(struct virtio_dev *hw, struct virtqueue *vq); + + void (*dump_json_info)(struct virtio_dev *hw, struct spdk_json_write_ctx *w); + void (*write_json_config)(struct virtio_dev *hw, struct spdk_json_write_ctx *w); +}; + +struct vq_desc_extra { + void *cookie; + uint16_t ndescs; +}; + +struct virtqueue { + struct virtio_dev *vdev; /**< owner of this virtqueue */ + struct vring vq_ring; /**< vring keeping desc, used and avail */ + /** + * Last consumed descriptor in the used table, + * trails vq_ring.used->idx. + */ + uint16_t vq_used_cons_idx; + uint16_t vq_nentries; /**< vring desc numbers */ + uint16_t vq_free_cnt; /**< num of desc available */ + uint16_t vq_avail_idx; /**< sync until needed */ + + void *vq_ring_virt_mem; /**< virtual address of vring */ + unsigned int vq_ring_size; + + uint64_t vq_ring_mem; /**< physical address of vring */ + + /** + * Head of the free chain in the descriptor table. If + * there are no free descriptors, this will be set to + * VQ_RING_DESC_CHAIN_END. + */ + uint16_t vq_desc_head_idx; + + /** + * Tail of the free chain in desc table. If + * there are no free descriptors, this will be set to + * VQ_RING_DESC_CHAIN_END. + */ + uint16_t vq_desc_tail_idx; + uint16_t vq_queue_index; /**< PCI queue index */ + uint16_t *notify_addr; + + /** Thread that's polling this queue. */ + struct spdk_thread *owner_thread; + + uint16_t req_start; + uint16_t req_end; + uint16_t reqs_finished; + + struct vq_desc_extra vq_descx[0]; +}; + +enum spdk_virtio_desc_type { + SPDK_VIRTIO_DESC_RO = 0, /**< Read only */ + SPDK_VIRTIO_DESC_WR = VRING_DESC_F_WRITE, /**< Write only */ + /* TODO VIRTIO_DESC_INDIRECT */ +}; + +/** Context for creating PCI virtio_devs */ +struct virtio_pci_ctx; + +/** + * Callback for creating virtio_dev from a PCI device. + * \param pci_ctx PCI context to be associated with a virtio_dev + * \param ctx context provided by the user + * \return 0 on success, -1 on error. + */ +typedef int (*virtio_pci_create_cb)(struct virtio_pci_ctx *pci_ctx, void *ctx); + +uint16_t virtio_recv_pkts(struct virtqueue *vq, void **io, uint32_t *len, uint16_t io_cnt); + +/** + * Start a new request on the current vring head position and associate it + * with an opaque cookie object. The previous request in given vq will be + * made visible to the device in hopes it can be processed early, but there's + * no guarantee it will be until the device is notified with \c + * virtqueue_req_flush. This behavior is simply an optimization and virtqueues + * must always be flushed. Empty requests (with no descriptors added) will be + * ignored. The device owning given virtqueue must be started. + * + * \param vq virtio queue + * \param cookie opaque object to associate with this request. Once the request + * is sent, processed and a response is received, the same object will be + * returned to the user after calling the virtio poll API. + * \param iovcnt number of required iovectors for the request. This can be + * higher than than the actual number of iovectors to be added. + * \return 0 on success or negative errno otherwise. If the `iovcnt` is + * greater than virtqueue depth, -EINVAL is returned. If simply not enough + * iovectors are available, -ENOMEM is returned. + */ +int virtqueue_req_start(struct virtqueue *vq, void *cookie, int iovcnt); + +/** + * Flush a virtqueue. This will notify the device if it's required. + * The device owning given virtqueue must be started. + * + * \param vq virtio queue + */ +void virtqueue_req_flush(struct virtqueue *vq); + +/** + * Abort the very last request in a virtqueue. This will restore virtqueue + * state to the point before the last request was created. Note that this + * is only effective if a queue hasn't been flushed yet. The device owning + * given virtqueue must be started. + * + * \param vq virtio queue + */ +void virtqueue_req_abort(struct virtqueue *vq); + +/** + * Add iovec chain to the last created request. This call does not provide any + * error-checking. The caller has to ensure that he doesn't add more iovs than + * what was specified during request creation. The device owning given virtqueue + * must be started. + * + * \param vq virtio queue + * \param iovs iovec array + * \param iovcnt number of iovs in iovec array + * \param desc_type type of all given iovectors + */ +void virtqueue_req_add_iovs(struct virtqueue *vq, struct iovec *iovs, uint16_t iovcnt, + enum spdk_virtio_desc_type desc_type); + +/** + * Construct a virtio device. The device will be in stopped state by default. + * Before doing any I/O, it has to be manually started via \c virtio_dev_restart. + * + * \param vdev memory for virtio device, must be zeroed + * \param name name for the virtio device + * \param ops backend callbacks + * \param ops_ctx argument for the backend callbacks + * \return zero on success, or negative error code otherwise + */ +int virtio_dev_construct(struct virtio_dev *vdev, const char *name, + const struct virtio_dev_ops *ops, void *ops_ctx); + +/** + * Reset the device and prepare it to be `virtio_dev_start`ed. This call + * will also renegotiate feature flags. + * + * \param vdev virtio device + * \param req_features features this driver supports. A VIRTIO_F_VERSION_1 + * flag will be automatically appended, as legacy devices are not supported. + */ +int virtio_dev_reset(struct virtio_dev *vdev, uint64_t req_features); + +/** + * Notify the host to start processing this virtio device. This is + * a blocking call that won't return until the host has started. + * This will also allocate virtqueues. + * + * \param vdev virtio device + * \param max_queues number of queues to allocate. The max number of + * usable I/O queues is also limited by the host device. `vdev` will be + * started successfully even if the host supports less queues than requested. + * \param fixed_queue_num number of queues preceeding the first + * request queue. For Virtio-SCSI this is equal to 2, as there are + * additional event and control queues. + */ +int virtio_dev_start(struct virtio_dev *vdev, uint16_t max_queues, + uint16_t fixed_queues_num); + +/** + * Stop the host from processing the device. This is a blocking call + * that won't return until all outstanding I/O has been processed on + * the host (virtio device) side. In order to re-start the device, it + * has to be `virtio_dev_reset` first. + * + * \param vdev virtio device + */ +void virtio_dev_stop(struct virtio_dev *vdev); + +/** + * Destruct a virtio device. Note that it must be in the stopped state. + * The virtio_dev should be manually freed afterwards. + * + * \param vdev virtio device + */ +void virtio_dev_destruct(struct virtio_dev *vdev); + +/** + * Bind a virtqueue with given index to the current thread; + * + * This function is thread-safe. + * + * \param vdev vhost device + * \param index virtqueue index + * \return 0 on success, -1 in case a virtqueue with given index either + * does not exists or is already acquired. + */ +int virtio_dev_acquire_queue(struct virtio_dev *vdev, uint16_t index); + +/** + * Look for unused queue and bind it to the current thread. This will + * scan the queues in range from *start_index* (inclusive) up to + * vdev->max_queues (exclusive). + * + * This function is thread-safe. + * + * \param vdev vhost device + * \param start_index virtqueue index to start looking from + * \return index of acquired queue or -1 in case no unused queue in given range + * has been found + */ +int32_t virtio_dev_find_and_acquire_queue(struct virtio_dev *vdev, uint16_t start_index); + +/** + * Get thread that acquired given virtqueue. + * + * This function is thread-safe. + * + * \param vdev vhost device + * \param index index of virtqueue + * \return thread that acquired given virtqueue. If the queue is unused + * or doesn't exist a NULL is returned. + */ +struct spdk_thread *virtio_dev_queue_get_thread(struct virtio_dev *vdev, uint16_t index); + +/** + * Check if virtqueue with given index is acquired. + * + * This function is thread-safe. + * + * \param vdev vhost device + * \param index index of virtqueue + * \return virtqueue acquire status. in case of invalid index *false* is returned. + */ +bool virtio_dev_queue_is_acquired(struct virtio_dev *vdev, uint16_t index); + +/** + * Release previously acquired queue. + * + * This function must be called from the thread that acquired the queue. + * + * \param vdev vhost device + * \param index index of virtqueue to release + */ +void virtio_dev_release_queue(struct virtio_dev *vdev, uint16_t index); + +/** + * Get Virtio status flags. + * + * \param vdev virtio device + */ +uint8_t virtio_dev_get_status(struct virtio_dev *vdev); + +/** + * Set Virtio status flag. The flags have to be set in very specific order + * defined the VIRTIO 1.0 spec section 3.1.1. To unset the flags, stop the + * device or set \c VIRTIO_CONFIG_S_RESET status flag. There is no way to + * unset only particular flags. + * + * \param vdev virtio device + * \param flag flag to set + */ +void virtio_dev_set_status(struct virtio_dev *vdev, uint8_t flag); + +/** + * Write raw data into the device config at given offset. This call does not + * provide any error checking. + * + * \param vdev virtio device + * \param offset offset in bytes + * \param src pointer to data to copy from + * \param len length of data to copy in bytes + * \return 0 on success, negative errno otherwise + */ +int virtio_dev_write_dev_config(struct virtio_dev *vdev, size_t offset, const void *src, int len); + +/** + * Read raw data from the device config at given offset. This call does not + * provide any error checking. + * + * \param vdev virtio device + * \param offset offset in bytes + * \param dst pointer to buffer to copy data into + * \param len length of data to copy in bytes + * \return 0 on success, negative errno otherwise + */ +int virtio_dev_read_dev_config(struct virtio_dev *vdev, size_t offset, void *dst, int len); + +/** + * Get backend-specific ops for given device. + * + * \param vdev virtio device + */ +const struct virtio_dev_ops *virtio_dev_backend_ops(struct virtio_dev *vdev); + +/** + * Check if the device has negotiated given feature bit. + * + * \param vdev virtio device + * \param bit feature bit + */ +static inline bool +virtio_dev_has_feature(struct virtio_dev *vdev, uint64_t bit) +{ + return !!(vdev->negotiated_features & (1ULL << bit)); +} + +/** + * Dump all device specific information into given json stream. + * + * \param vdev virtio device + * \param w json stream + */ +void virtio_dev_dump_json_info(struct virtio_dev *vdev, struct spdk_json_write_ctx *w); + +/** + * Enumerate all PCI Virtio devices of given type on the system. + * + * \param enum_cb a function to be called for each valid PCI device. + * If a virtio_dev is has been created, the callback should return 0. + * Returning any other value will cause the PCI context to be freed, + * making it unusable. + * \param enum_ctx additional opaque context to be passed into `enum_cb` + * \param pci_device_id PCI Device ID of devices to iterate through + */ +int virtio_pci_dev_enumerate(virtio_pci_create_cb enum_cb, void *enum_ctx, + uint16_t pci_device_id); + +/** + * Attach a PCI Virtio device of given type. + * + * \param create_cb callback to create a virtio_dev. + * If virtio_dev is has been created, the callback should return 0. + * Returning any other value will cause the PCI context to be freed, + * making it unusable. + * \param enum_ctx additional opaque context to be passed into `enum_cb` + * \param pci_device_id PCI Device ID of devices to iterate through + * \param pci_addr PCI address of the device to attach + */ +int virtio_pci_dev_attach(virtio_pci_create_cb create_cb, void *enum_ctx, + uint16_t pci_device_id, struct spdk_pci_addr *pci_addr); + +/** + * Connect to a vhost-user device and init corresponding virtio_dev struct. + * The virtio_dev will have to be freed with \c virtio_dev_free. + * + * \param vdev preallocated vhost device struct to operate on + * \param name name of this virtio device + * \param path path to the Unix domain socket of the vhost-user device + * \param queue_size size of each of the queues + * \return virtio device + */ +int virtio_user_dev_init(struct virtio_dev *vdev, const char *name, const char *path, + uint32_t queue_size); + +/** + * Initialize virtio_dev for a given PCI device. + * The virtio_dev has to be freed with \c virtio_dev_destruct. + * + * \param vdev preallocated vhost device struct to operate on + * \param name name of this virtio device + * \param pci_ctx context of the PCI device + * \return 0 on success, -1 on error. + */ +int virtio_pci_dev_init(struct virtio_dev *vdev, const char *name, + struct virtio_pci_ctx *pci_ctx); + +#endif /* SPDK_VIRTIO_H */ |