diff options
Diffstat (limited to '')
| -rw-r--r-- | src/spdk/lib/copy/Makefile | 42 | ||||
| -rw-r--r-- | src/spdk/lib/copy/copy_engine.c | 318 | ||||
| -rw-r--r-- | src/spdk/lib/copy/ioat/Makefile | 40 | ||||
| -rw-r--r-- | src/spdk/lib/copy/ioat/copy_engine_ioat.c | 421 | ||||
| -rw-r--r-- | src/spdk/lib/copy/ioat/copy_engine_ioat.h | 44 | ||||
| -rw-r--r-- | src/spdk/lib/copy/ioat/copy_engine_ioat_rpc.c | 118 |
6 files changed, 983 insertions, 0 deletions
diff --git a/src/spdk/lib/copy/Makefile b/src/spdk/lib/copy/Makefile new file mode 100644 index 00000000..31f983b5 --- /dev/null +++ b/src/spdk/lib/copy/Makefile @@ -0,0 +1,42 @@ +# +# BSD LICENSE +# +# Copyright (c) Intel Corporation. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..) +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk + +LIBNAME = copy +C_SRCS = copy_engine.c + +DIRS-y = ioat + +include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk diff --git a/src/spdk/lib/copy/copy_engine.c b/src/spdk/lib/copy/copy_engine.c new file mode 100644 index 00000000..921e17fa --- /dev/null +++ b/src/spdk/lib/copy/copy_engine.c @@ -0,0 +1,318 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "spdk/stdinc.h" + +#include "spdk_internal/copy_engine.h" + +#include "spdk/env.h" +#include "spdk/event.h" +#include "spdk/log.h" +#include "spdk/thread.h" + +static size_t g_max_copy_module_size = 0; + +static struct spdk_copy_engine *hw_copy_engine = NULL; +/* Memcpy engine always exist */ +static struct spdk_copy_engine *mem_copy_engine = NULL; + +TAILQ_HEAD(, spdk_copy_module_if) spdk_copy_module_list = + TAILQ_HEAD_INITIALIZER(spdk_copy_module_list); + +struct copy_io_channel { + struct spdk_copy_engine *engine; + struct spdk_io_channel *ch; +}; + +struct spdk_copy_module_if *g_copy_engine_module = NULL; +spdk_copy_fini_cb g_fini_cb_fn = NULL; +void *g_fini_cb_arg = NULL; + +void +spdk_copy_engine_register(struct spdk_copy_engine *copy_engine) +{ + assert(hw_copy_engine == NULL); + hw_copy_engine = copy_engine; +} + +static void +spdk_memcpy_register(struct spdk_copy_engine *copy_engine) +{ + assert(mem_copy_engine == NULL); + mem_copy_engine = copy_engine; +} + +static void +spdk_memcpy_unregister(void) +{ + mem_copy_engine = NULL; +} + +static void +copy_engine_done(void *ref, int status) +{ + struct spdk_copy_task *req = (struct spdk_copy_task *)ref; + + req->cb(req, status); +} + +int +spdk_copy_submit(struct spdk_copy_task *copy_req, struct spdk_io_channel *ch, + void *dst, void *src, uint64_t nbytes, spdk_copy_completion_cb cb) +{ + struct spdk_copy_task *req = copy_req; + struct copy_io_channel *copy_ch = spdk_io_channel_get_ctx(ch); + + req->cb = cb; + return copy_ch->engine->copy(req->offload_ctx, copy_ch->ch, dst, src, nbytes, + copy_engine_done); +} + +int +spdk_copy_submit_fill(struct spdk_copy_task *copy_req, struct spdk_io_channel *ch, + void *dst, uint8_t fill, uint64_t nbytes, spdk_copy_completion_cb cb) +{ + struct spdk_copy_task *req = copy_req; + struct copy_io_channel *copy_ch = spdk_io_channel_get_ctx(ch); + + req->cb = cb; + return copy_ch->engine->fill(req->offload_ctx, copy_ch->ch, dst, fill, nbytes, + copy_engine_done); +} + +/* memcpy default copy engine */ +static int +mem_copy_submit(void *cb_arg, struct spdk_io_channel *ch, void *dst, void *src, uint64_t nbytes, + spdk_copy_completion_cb cb) +{ + struct spdk_copy_task *copy_req; + + memcpy(dst, src, (size_t)nbytes); + + copy_req = (struct spdk_copy_task *)((uintptr_t)cb_arg - + offsetof(struct spdk_copy_task, offload_ctx)); + cb(copy_req, 0); + return 0; +} + +static int +mem_copy_fill(void *cb_arg, struct spdk_io_channel *ch, void *dst, uint8_t fill, uint64_t nbytes, + spdk_copy_completion_cb cb) +{ + struct spdk_copy_task *copy_req; + + memset(dst, fill, nbytes); + copy_req = (struct spdk_copy_task *)((uintptr_t)cb_arg - + offsetof(struct spdk_copy_task, offload_ctx)); + cb(copy_req, 0); + + return 0; +} + +static struct spdk_io_channel *mem_get_io_channel(void); + +static struct spdk_copy_engine memcpy_copy_engine = { + .copy = mem_copy_submit, + .fill = mem_copy_fill, + .get_io_channel = mem_get_io_channel, +}; + +static int +memcpy_create_cb(void *io_device, void *ctx_buf) +{ + return 0; +} + +static void +memcpy_destroy_cb(void *io_device, void *ctx_buf) +{ +} + +static struct spdk_io_channel *mem_get_io_channel(void) +{ + return spdk_get_io_channel(&memcpy_copy_engine); +} + +static size_t +copy_engine_mem_get_ctx_size(void) +{ + return sizeof(struct spdk_copy_task); +} + +size_t +spdk_copy_task_size(void) +{ + return g_max_copy_module_size; +} + +void spdk_copy_module_list_add(struct spdk_copy_module_if *copy_module) +{ + TAILQ_INSERT_TAIL(&spdk_copy_module_list, copy_module, tailq); + if (copy_module->get_ctx_size && copy_module->get_ctx_size() > g_max_copy_module_size) { + g_max_copy_module_size = copy_module->get_ctx_size(); + } +} + +static int +copy_create_cb(void *io_device, void *ctx_buf) +{ + struct copy_io_channel *copy_ch = ctx_buf; + + if (hw_copy_engine != NULL) { + copy_ch->ch = hw_copy_engine->get_io_channel(); + if (copy_ch->ch != NULL) { + copy_ch->engine = hw_copy_engine; + return 0; + } + } + + copy_ch->ch = mem_copy_engine->get_io_channel(); + assert(copy_ch->ch != NULL); + copy_ch->engine = mem_copy_engine; + return 0; +} + +static void +copy_destroy_cb(void *io_device, void *ctx_buf) +{ + struct copy_io_channel *copy_ch = ctx_buf; + + spdk_put_io_channel(copy_ch->ch); +} + +struct spdk_io_channel * +spdk_copy_engine_get_io_channel(void) +{ + return spdk_get_io_channel(&spdk_copy_module_list); +} + +static int +copy_engine_mem_init(void) +{ + spdk_memcpy_register(&memcpy_copy_engine); + spdk_io_device_register(&memcpy_copy_engine, memcpy_create_cb, memcpy_destroy_cb, 0, + "memcpy_engine"); + + return 0; +} + +static void +copy_engine_mem_fini(void *ctxt) +{ + spdk_io_device_unregister(&memcpy_copy_engine, NULL); + spdk_memcpy_unregister(); + + spdk_copy_engine_module_finish(); +} + +static void +spdk_copy_engine_module_initialize(void) +{ + struct spdk_copy_module_if *copy_engine_module; + + TAILQ_FOREACH(copy_engine_module, &spdk_copy_module_list, tailq) { + copy_engine_module->module_init(); + } +} + +int +spdk_copy_engine_initialize(void) +{ + spdk_copy_engine_module_initialize(); + /* + * We need a unique identifier for the copy engine framework, so use the + * spdk_copy_module_list address for this purpose. + */ + spdk_io_device_register(&spdk_copy_module_list, copy_create_cb, copy_destroy_cb, + sizeof(struct copy_io_channel), "copy_module"); + + return 0; +} + +static void +spdk_copy_engine_module_finish_cb(void) +{ + spdk_copy_fini_cb cb_fn = g_fini_cb_fn; + + cb_fn(g_fini_cb_arg); + g_fini_cb_fn = NULL; + g_fini_cb_arg = NULL; +} + +void +spdk_copy_engine_module_finish(void) +{ + if (!g_copy_engine_module) { + g_copy_engine_module = TAILQ_FIRST(&spdk_copy_module_list); + } else { + g_copy_engine_module = TAILQ_NEXT(g_copy_engine_module, tailq); + } + + if (!g_copy_engine_module) { + spdk_copy_engine_module_finish_cb(); + return; + } + + if (g_copy_engine_module->module_fini) { + spdk_thread_send_msg(spdk_get_thread(), g_copy_engine_module->module_fini, NULL); + } else { + spdk_copy_engine_module_finish(); + } +} + +void +spdk_copy_engine_finish(spdk_copy_fini_cb cb_fn, void *cb_arg) +{ + assert(cb_fn != NULL); + + g_fini_cb_fn = cb_fn; + g_fini_cb_arg = cb_arg; + + spdk_io_device_unregister(&spdk_copy_module_list, NULL); + spdk_copy_engine_module_finish(); +} + +void +spdk_copy_engine_config_text(FILE *fp) +{ + struct spdk_copy_module_if *copy_engine_module; + + TAILQ_FOREACH(copy_engine_module, &spdk_copy_module_list, tailq) { + if (copy_engine_module->config_text) { + copy_engine_module->config_text(fp); + } + } +} + +SPDK_COPY_MODULE_REGISTER(copy_engine_mem_init, copy_engine_mem_fini, + NULL, copy_engine_mem_get_ctx_size) diff --git a/src/spdk/lib/copy/ioat/Makefile b/src/spdk/lib/copy/ioat/Makefile new file mode 100644 index 00000000..3d19e38f --- /dev/null +++ b/src/spdk/lib/copy/ioat/Makefile @@ -0,0 +1,40 @@ +# +# BSD LICENSE +# +# Copyright (c) Intel Corporation. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..) +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk + +LIBNAME = copy_ioat +C_SRCS = copy_engine_ioat.c copy_engine_ioat_rpc.c + +include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk diff --git a/src/spdk/lib/copy/ioat/copy_engine_ioat.c b/src/spdk/lib/copy/ioat/copy_engine_ioat.c new file mode 100644 index 00000000..40bc6cf5 --- /dev/null +++ b/src/spdk/lib/copy/ioat/copy_engine_ioat.c @@ -0,0 +1,421 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "copy_engine_ioat.h" + +#include "spdk/stdinc.h" + +#include "spdk_internal/copy_engine.h" +#include "spdk_internal/log.h" + +#include "spdk/env.h" +#include "spdk/conf.h" +#include "spdk/event.h" +#include "spdk/thread.h" +#include "spdk/ioat.h" + +static bool g_ioat_enable = false; + +struct ioat_probe_ctx { + int num_whitelist_devices; + struct spdk_pci_addr whitelist[IOAT_MAX_CHANNELS]; +}; + +static struct ioat_probe_ctx g_probe_ctx; + +struct ioat_device { + struct spdk_ioat_chan *ioat; + bool is_allocated; + /** linked list pointer for device list */ + TAILQ_ENTRY(ioat_device) tailq; +}; + +static TAILQ_HEAD(, ioat_device) g_devices = TAILQ_HEAD_INITIALIZER(g_devices); +static pthread_mutex_t g_ioat_mutex = PTHREAD_MUTEX_INITIALIZER; + +struct ioat_io_channel { + struct spdk_ioat_chan *ioat_ch; + struct ioat_device *ioat_dev; + struct spdk_poller *poller; +}; + +static int +ioat_find_dev_by_whitelist_bdf(const struct spdk_pci_addr *pci_addr, + const struct spdk_pci_addr *whitelist, + int num_whitelist_devices) +{ + int i; + + for (i = 0; i < num_whitelist_devices; i++) { + if (spdk_pci_addr_compare(pci_addr, &whitelist[i]) == 0) { + return 1; + } + } + return 0; +} + +static struct ioat_device * +ioat_allocate_device(void) +{ + struct ioat_device *dev; + + pthread_mutex_lock(&g_ioat_mutex); + TAILQ_FOREACH(dev, &g_devices, tailq) { + if (!dev->is_allocated) { + dev->is_allocated = true; + pthread_mutex_unlock(&g_ioat_mutex); + return dev; + } + } + pthread_mutex_unlock(&g_ioat_mutex); + + return NULL; +} + +static void +ioat_free_device(struct ioat_device *dev) +{ + pthread_mutex_lock(&g_ioat_mutex); + dev->is_allocated = false; + pthread_mutex_unlock(&g_ioat_mutex); +} + +struct ioat_task { + spdk_copy_completion_cb cb; +}; + +static int copy_engine_ioat_init(void); +static void copy_engine_ioat_exit(void *ctx); +static void copy_engine_ioat_config_text(FILE *fp); + +static size_t +copy_engine_ioat_get_ctx_size(void) +{ + return sizeof(struct ioat_task) + sizeof(struct spdk_copy_task); +} + +SPDK_COPY_MODULE_REGISTER(copy_engine_ioat_init, copy_engine_ioat_exit, + copy_engine_ioat_config_text, + copy_engine_ioat_get_ctx_size) + +static void +copy_engine_ioat_exit(void *ctx) +{ + struct ioat_device *dev; + + while (!TAILQ_EMPTY(&g_devices)) { + dev = TAILQ_FIRST(&g_devices); + TAILQ_REMOVE(&g_devices, dev, tailq); + spdk_ioat_detach(dev->ioat); + ioat_free_device(dev); + spdk_dma_free(dev); + } + spdk_copy_engine_module_finish(); +} + +static void +ioat_done(void *cb_arg) +{ + struct spdk_copy_task *copy_req; + struct ioat_task *ioat_task = cb_arg; + + copy_req = (struct spdk_copy_task *) + ((uintptr_t)ioat_task - + offsetof(struct spdk_copy_task, offload_ctx)); + + ioat_task->cb(copy_req, 0); +} + +static int +ioat_copy_submit(void *cb_arg, struct spdk_io_channel *ch, void *dst, void *src, uint64_t nbytes, + spdk_copy_completion_cb cb) +{ + struct ioat_task *ioat_task = (struct ioat_task *)cb_arg; + struct ioat_io_channel *ioat_ch = spdk_io_channel_get_ctx(ch); + + assert(ioat_ch->ioat_ch != NULL); + + ioat_task->cb = cb; + + return spdk_ioat_submit_copy(ioat_ch->ioat_ch, ioat_task, ioat_done, dst, src, nbytes); +} + +static int +ioat_copy_submit_fill(void *cb_arg, struct spdk_io_channel *ch, void *dst, uint8_t fill, + uint64_t nbytes, spdk_copy_completion_cb cb) +{ + struct ioat_task *ioat_task = (struct ioat_task *)cb_arg; + struct ioat_io_channel *ioat_ch = spdk_io_channel_get_ctx(ch); + uint64_t fill64 = 0x0101010101010101ULL * fill; + + assert(ioat_ch->ioat_ch != NULL); + + ioat_task->cb = cb; + + return spdk_ioat_submit_fill(ioat_ch->ioat_ch, ioat_task, ioat_done, dst, fill64, nbytes); +} + +static int +ioat_poll(void *arg) +{ + struct spdk_ioat_chan *chan = arg; + + spdk_ioat_process_events(chan); + + return -1; +} + +static struct spdk_io_channel *ioat_get_io_channel(void); + +static struct spdk_copy_engine ioat_copy_engine = { + .copy = ioat_copy_submit, + .fill = ioat_copy_submit_fill, + .get_io_channel = ioat_get_io_channel, +}; + +static int +ioat_create_cb(void *io_device, void *ctx_buf) +{ + struct ioat_io_channel *ch = ctx_buf; + struct ioat_device *ioat_dev; + + ioat_dev = ioat_allocate_device(); + if (ioat_dev == NULL) { + return -1; + } + + ch->ioat_dev = ioat_dev; + ch->ioat_ch = ioat_dev->ioat; + ch->poller = spdk_poller_register(ioat_poll, ch->ioat_ch, 0); + return 0; +} + +static void +ioat_destroy_cb(void *io_device, void *ctx_buf) +{ + struct ioat_io_channel *ch = ctx_buf; + + ioat_free_device(ch->ioat_dev); + spdk_poller_unregister(&ch->poller); +} + +static struct spdk_io_channel * +ioat_get_io_channel(void) +{ + return spdk_get_io_channel(&ioat_copy_engine); +} + +static bool +probe_cb(void *cb_ctx, struct spdk_pci_device *pci_dev) +{ + struct ioat_probe_ctx *ctx = cb_ctx; + struct spdk_pci_addr pci_addr = spdk_pci_device_get_addr(pci_dev); + + SPDK_INFOLOG(SPDK_LOG_COPY_IOAT, + " Found matching device at %04x:%02x:%02x.%x vendor:0x%04x device:0x%04x\n", + pci_addr.domain, + pci_addr.bus, + pci_addr.dev, + pci_addr.func, + spdk_pci_device_get_vendor_id(pci_dev), + spdk_pci_device_get_device_id(pci_dev)); + + if (ctx->num_whitelist_devices > 0 && + !ioat_find_dev_by_whitelist_bdf(&pci_addr, ctx->whitelist, ctx->num_whitelist_devices)) { + return false; + } + + /* Claim the device in case conflict with other process */ + if (spdk_pci_device_claim(&pci_addr) < 0) { + return false; + } + + return true; +} + +static void +attach_cb(void *cb_ctx, struct spdk_pci_device *pci_dev, struct spdk_ioat_chan *ioat) +{ + struct ioat_device *dev; + + dev = spdk_dma_zmalloc(sizeof(*dev), 0, NULL); + if (dev == NULL) { + SPDK_ERRLOG("Failed to allocate device struct\n"); + return; + } + + dev->ioat = ioat; + TAILQ_INSERT_TAIL(&g_devices, dev, tailq); +} + +void +copy_engine_ioat_enable_probe(void) +{ + g_ioat_enable = true; +} + +static int +copy_engine_ioat_add_whitelist_device(const char *pci_bdf) +{ + if (pci_bdf == NULL) { + return -1; + } + + if (g_probe_ctx.num_whitelist_devices >= IOAT_MAX_CHANNELS) { + SPDK_ERRLOG("Ioat whitelist is full (max size is %d)\n", + IOAT_MAX_CHANNELS); + return -1; + } + + if (spdk_pci_addr_parse(&g_probe_ctx.whitelist[g_probe_ctx.num_whitelist_devices], + pci_bdf) < 0) { + SPDK_ERRLOG("Invalid address %s\n", pci_bdf); + return -1; + } + + g_probe_ctx.num_whitelist_devices++; + + return 0; +} + +int +copy_engine_ioat_add_whitelist_devices(const char *pci_bdfs[], size_t num_pci_bdfs) +{ + size_t i; + + for (i = 0; i < num_pci_bdfs; i++) { + if (copy_engine_ioat_add_whitelist_device(pci_bdfs[i]) < 0) { + return -1; + } + } + + return 0; +} + +static int +copy_engine_ioat_read_config_file_params(struct spdk_conf_section *sp) +{ + int i; + char *val, *pci_bdf; + + if (spdk_conf_section_get_boolval(sp, "Enable", false)) { + g_ioat_enable = true; + /* Enable Ioat */ + } + + val = spdk_conf_section_get_val(sp, "Disable"); + if (val != NULL) { + SPDK_WARNLOG("\"Disable\" option is deprecated and will be removed in a future release.\n"); + SPDK_WARNLOG("IOAT is now disabled by default. It may be enabled by \"Enable Yes\"\n"); + + if (g_ioat_enable && (strcasecmp(val, "Yes") == 0)) { + SPDK_ERRLOG("\"Enable Yes\" and \"Disable Yes\" cannot be set at the same time\n"); + return -1; + } + } + + /* Init the whitelist */ + for (i = 0; ; i++) { + pci_bdf = spdk_conf_section_get_nmval(sp, "Whitelist", i, 0); + if (!pci_bdf) { + break; + } + + if (copy_engine_ioat_add_whitelist_device(pci_bdf) < 0) { + return -1; + } + } + + return 0; +} + +static int +copy_engine_ioat_init(void) +{ + struct spdk_conf_section *sp; + int rc; + + sp = spdk_conf_find_section(NULL, "Ioat"); + if (sp != NULL) { + rc = copy_engine_ioat_read_config_file_params(sp); + if (rc != 0) { + SPDK_ERRLOG("copy_engine_ioat_read_config_file_params() failed\n"); + return rc; + } + } + + if (!g_ioat_enable) { + return 0; + } + + if (spdk_ioat_probe(&g_probe_ctx, probe_cb, attach_cb) != 0) { + SPDK_ERRLOG("spdk_ioat_probe() failed\n"); + return -1; + } + + SPDK_INFOLOG(SPDK_LOG_COPY_IOAT, "Ioat Copy Engine Offload Enabled\n"); + spdk_copy_engine_register(&ioat_copy_engine); + spdk_io_device_register(&ioat_copy_engine, ioat_create_cb, ioat_destroy_cb, + sizeof(struct ioat_io_channel), "ioat_copy_engine"); + return 0; +} + +#define COPY_ENGINE_IOAT_HEADER_TMPL \ +"[Ioat]\n" \ +" # Users may not want to use offload even it is available.\n" \ +" # Users may use the whitelist to initialize specified devices, IDS\n" \ +" # uses BUS:DEVICE.FUNCTION to identify each Ioat channel.\n" + +#define COPY_ENGINE_IOAT_ENABLE_TMPL \ +" Enable %s\n" + +#define COPY_ENGINE_IOAT_WHITELIST_TMPL \ +" Whitelist %.4" PRIx16 ":%.2" PRIx8 ":%.2" PRIx8 ".%" PRIx8 "\n" + +static void +copy_engine_ioat_config_text(FILE *fp) +{ + int i; + struct spdk_pci_addr *dev; + + fprintf(fp, COPY_ENGINE_IOAT_HEADER_TMPL); + fprintf(fp, COPY_ENGINE_IOAT_ENABLE_TMPL, g_ioat_enable ? "Yes" : "No"); + + for (i = 0; i < g_probe_ctx.num_whitelist_devices; i++) { + dev = &g_probe_ctx.whitelist[i]; + fprintf(fp, COPY_ENGINE_IOAT_WHITELIST_TMPL, + dev->domain, dev->bus, dev->dev, dev->func); + } +} + +SPDK_LOG_REGISTER_COMPONENT("copy_ioat", SPDK_LOG_COPY_IOAT) diff --git a/src/spdk/lib/copy/ioat/copy_engine_ioat.h b/src/spdk/lib/copy/ioat/copy_engine_ioat.h new file mode 100644 index 00000000..ae69fb2d --- /dev/null +++ b/src/spdk/lib/copy/ioat/copy_engine_ioat.h @@ -0,0 +1,44 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef SPDK_COPY_ENGINE_IOAT_H +#define SPDK_COPY_ENGINE_IOAT_H + +#include "spdk/stdinc.h" + +#define IOAT_MAX_CHANNELS 64 + +int copy_engine_ioat_add_whitelist_devices(const char *pci_bdfs[], size_t num_pci_bdfs); +void copy_engine_ioat_enable_probe(void); + +#endif /* SPDK_COPY_ENGINE_IOAT_H */ diff --git a/src/spdk/lib/copy/ioat/copy_engine_ioat_rpc.c b/src/spdk/lib/copy/ioat/copy_engine_ioat_rpc.c new file mode 100644 index 00000000..ae03fdb1 --- /dev/null +++ b/src/spdk/lib/copy/ioat/copy_engine_ioat_rpc.c @@ -0,0 +1,118 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "copy_engine_ioat.h" + +#include "spdk/rpc.h" +#include "spdk/util.h" +#include "spdk/event.h" + +struct rpc_pci_whitelist { + size_t num_bdfs; + char *bdfs[IOAT_MAX_CHANNELS]; +}; + +static int +decode_rpc_pci_whitelist(const struct spdk_json_val *val, void *out) +{ + struct rpc_pci_whitelist *pci_whitelist = out; + + return spdk_json_decode_array(val, spdk_json_decode_string, pci_whitelist->bdfs, + IOAT_MAX_CHANNELS, &pci_whitelist->num_bdfs, sizeof(char *)); +} + +static void +free_rpc_pci_whitelist(struct rpc_pci_whitelist *list) +{ + size_t i; + + for (i = 0; i < list->num_bdfs; i++) { + free(list->bdfs[i]); + } +} + +struct rpc_copy_engine_ioat { + struct rpc_pci_whitelist pci_whitelist; +}; + +static void +free_rpc_copy_engine_ioat(struct rpc_copy_engine_ioat *p) +{ + free_rpc_pci_whitelist(&p->pci_whitelist); +} + +static const struct spdk_json_object_decoder rpc_copy_engine_ioat_decoder[] = { + {"pci_whitelist", offsetof(struct rpc_copy_engine_ioat, pci_whitelist), decode_rpc_pci_whitelist}, +}; + +static void +spdk_rpc_scan_copy_engine_ioat(struct spdk_jsonrpc_request *request, + const struct spdk_json_val *params) +{ + struct rpc_copy_engine_ioat req = {}; + struct spdk_json_write_ctx *w; + int rc; + + if (params != NULL) { + if (spdk_json_decode_object(params, rpc_copy_engine_ioat_decoder, + SPDK_COUNTOF(rpc_copy_engine_ioat_decoder), + &req)) { + free_rpc_copy_engine_ioat(&req); + SPDK_ERRLOG("spdk_json_decode_object() failed\n"); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "Invalid parameters"); + return; + } + + rc = copy_engine_ioat_add_whitelist_devices((const char **)req.pci_whitelist.bdfs, + req.pci_whitelist.num_bdfs); + free_rpc_copy_engine_ioat(&req); + if (rc < 0) { + SPDK_ERRLOG("copy_engine_ioat_add_whitelist_devices() failed\n"); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "Invalid parameters"); + return; + } + } + + copy_engine_ioat_enable_probe(); + + w = spdk_jsonrpc_begin_result(request); + if (w == NULL) { + return; + } + + spdk_json_write_bool(w, true); + spdk_jsonrpc_end_result(request, w); +} +SPDK_RPC_REGISTER("scan_ioat_copy_engine", spdk_rpc_scan_copy_engine_ioat, SPDK_RPC_STARTUP) |