summaryrefslogtreecommitdiffstats
path: root/src/spdk/lib/env_dpdk/env.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/spdk/lib/env_dpdk/env.c')
-rw-r--r--src/spdk/lib/env_dpdk/env.c419
1 files changed, 419 insertions, 0 deletions
diff --git a/src/spdk/lib/env_dpdk/env.c b/src/spdk/lib/env_dpdk/env.c
new file mode 100644
index 00000000..a5238e54
--- /dev/null
+++ b/src/spdk/lib/env_dpdk/env.c
@@ -0,0 +1,419 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+
+#include "spdk/env.h"
+
+#include <rte_config.h>
+#include <rte_cycles.h>
+#include <rte_malloc.h>
+#include <rte_mempool.h>
+#include <rte_memzone.h>
+#include <rte_version.h>
+
+static uint64_t
+virt_to_phys(void *vaddr)
+{
+ uint64_t ret;
+
+#if RTE_VERSION >= RTE_VERSION_NUM(17, 11, 0, 3)
+ ret = rte_malloc_virt2iova(vaddr);
+ if (ret != RTE_BAD_IOVA) {
+ return ret;
+ }
+#else
+ ret = rte_malloc_virt2phy(vaddr);
+ if (ret != RTE_BAD_PHYS_ADDR) {
+ return ret;
+ }
+#endif
+
+ return spdk_vtophys(vaddr);
+}
+
+void *
+spdk_malloc(size_t size, size_t align, uint64_t *phys_addr, int socket_id, uint32_t flags)
+{
+ if (flags == 0) {
+ return NULL;
+ }
+
+ void *buf = rte_malloc_socket(NULL, size, align, socket_id);
+ if (buf && phys_addr) {
+ *phys_addr = virt_to_phys(buf);
+ }
+ return buf;
+}
+
+void *
+spdk_zmalloc(size_t size, size_t align, uint64_t *phys_addr, int socket_id, uint32_t flags)
+{
+ void *buf = spdk_malloc(size, align, phys_addr, socket_id, flags);
+ if (buf) {
+ memset(buf, 0, size);
+ }
+ return buf;
+}
+
+void
+spdk_free(void *buf)
+{
+ rte_free(buf);
+}
+
+void *
+spdk_dma_malloc_socket(size_t size, size_t align, uint64_t *phys_addr, int socket_id)
+{
+ return spdk_malloc(size, align, phys_addr, socket_id, (SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE));
+}
+
+void *
+spdk_dma_zmalloc_socket(size_t size, size_t align, uint64_t *phys_addr, int socket_id)
+{
+ return spdk_zmalloc(size, align, phys_addr, socket_id, (SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE));
+}
+
+void *
+spdk_dma_malloc(size_t size, size_t align, uint64_t *phys_addr)
+{
+ return spdk_dma_malloc_socket(size, align, phys_addr, SPDK_ENV_SOCKET_ID_ANY);
+}
+
+void *
+spdk_dma_zmalloc(size_t size, size_t align, uint64_t *phys_addr)
+{
+ return spdk_dma_zmalloc_socket(size, align, phys_addr, SPDK_ENV_SOCKET_ID_ANY);
+}
+
+void *
+spdk_dma_realloc(void *buf, size_t size, size_t align, uint64_t *phys_addr)
+{
+ void *new_buf = rte_realloc(buf, size, align);
+ if (new_buf && phys_addr) {
+ *phys_addr = virt_to_phys(new_buf);
+ }
+ return new_buf;
+}
+
+void
+spdk_dma_free(void *buf)
+{
+ spdk_free(buf);
+}
+
+void *
+spdk_memzone_reserve_aligned(const char *name, size_t len, int socket_id,
+ unsigned flags, unsigned align)
+{
+ const struct rte_memzone *mz;
+ unsigned dpdk_flags = 0;
+
+#if RTE_VERSION >= RTE_VERSION_NUM(18, 05, 0, 0)
+ /* Older DPDKs do not offer such flag since their
+ * memzones are iova-contiguous by default.
+ */
+ if ((flags & SPDK_MEMZONE_NO_IOVA_CONTIG) == 0) {
+ dpdk_flags |= RTE_MEMZONE_IOVA_CONTIG;
+ }
+#endif
+
+ if (socket_id == SPDK_ENV_SOCKET_ID_ANY) {
+ socket_id = SOCKET_ID_ANY;
+ }
+
+ mz = rte_memzone_reserve_aligned(name, len, socket_id, dpdk_flags, align);
+
+ if (mz != NULL) {
+ memset(mz->addr, 0, len);
+ return mz->addr;
+ } else {
+ return NULL;
+ }
+}
+
+void *
+spdk_memzone_reserve(const char *name, size_t len, int socket_id, unsigned flags)
+{
+ return spdk_memzone_reserve_aligned(name, len, socket_id, flags,
+ RTE_CACHE_LINE_SIZE);
+}
+
+void *
+spdk_memzone_lookup(const char *name)
+{
+ const struct rte_memzone *mz = rte_memzone_lookup(name);
+
+ if (mz != NULL) {
+ return mz->addr;
+ } else {
+ return NULL;
+ }
+}
+
+int
+spdk_memzone_free(const char *name)
+{
+ const struct rte_memzone *mz = rte_memzone_lookup(name);
+
+ if (mz != NULL) {
+ return rte_memzone_free(mz);
+ }
+
+ return -1;
+}
+
+void
+spdk_memzone_dump(FILE *f)
+{
+ rte_memzone_dump(f);
+}
+
+struct spdk_mempool *
+spdk_mempool_create_ctor(const char *name, size_t count,
+ size_t ele_size, size_t cache_size, int socket_id,
+ spdk_mempool_obj_cb_t *obj_init, void *obj_init_arg)
+{
+ struct rte_mempool *mp;
+ size_t tmp;
+
+ if (socket_id == SPDK_ENV_SOCKET_ID_ANY) {
+ socket_id = SOCKET_ID_ANY;
+ }
+
+ /* No more than half of all elements can be in cache */
+ tmp = (count / 2) / rte_lcore_count();
+ if (cache_size > tmp) {
+ cache_size = tmp;
+ }
+
+ if (cache_size > RTE_MEMPOOL_CACHE_MAX_SIZE) {
+ cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE;
+ }
+
+ mp = rte_mempool_create(name, count, ele_size, cache_size,
+ 0, NULL, NULL, (rte_mempool_obj_cb_t *)obj_init, obj_init_arg,
+ socket_id, MEMPOOL_F_NO_PHYS_CONTIG);
+
+ return (struct spdk_mempool *)mp;
+}
+
+
+struct spdk_mempool *
+spdk_mempool_create(const char *name, size_t count,
+ size_t ele_size, size_t cache_size, int socket_id)
+{
+ return spdk_mempool_create_ctor(name, count, ele_size, cache_size, socket_id,
+ NULL, NULL);
+}
+
+char *
+spdk_mempool_get_name(struct spdk_mempool *mp)
+{
+ return ((struct rte_mempool *)mp)->name;
+}
+
+void
+spdk_mempool_free(struct spdk_mempool *mp)
+{
+#if RTE_VERSION >= RTE_VERSION_NUM(16, 7, 0, 1)
+ rte_mempool_free((struct rte_mempool *)mp);
+#endif
+}
+
+void *
+spdk_mempool_get(struct spdk_mempool *mp)
+{
+ void *ele = NULL;
+ int rc;
+
+ rc = rte_mempool_get((struct rte_mempool *)mp, &ele);
+ if (rc != 0) {
+ return NULL;
+ }
+ return ele;
+}
+
+int
+spdk_mempool_get_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count)
+{
+ return rte_mempool_get_bulk((struct rte_mempool *)mp, ele_arr, count);
+}
+
+void
+spdk_mempool_put(struct spdk_mempool *mp, void *ele)
+{
+ rte_mempool_put((struct rte_mempool *)mp, ele);
+}
+
+void
+spdk_mempool_put_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count)
+{
+ rte_mempool_put_bulk((struct rte_mempool *)mp, ele_arr, count);
+}
+
+size_t
+spdk_mempool_count(const struct spdk_mempool *pool)
+{
+#if RTE_VERSION < RTE_VERSION_NUM(16, 7, 0, 1)
+ return rte_mempool_count((struct rte_mempool *)pool);
+#else
+ return rte_mempool_avail_count((struct rte_mempool *)pool);
+#endif
+}
+
+bool
+spdk_process_is_primary(void)
+{
+ return (rte_eal_process_type() == RTE_PROC_PRIMARY);
+}
+
+uint64_t spdk_get_ticks(void)
+{
+ return rte_get_timer_cycles();
+}
+
+uint64_t spdk_get_ticks_hz(void)
+{
+ return rte_get_timer_hz();
+}
+
+void spdk_delay_us(unsigned int us)
+{
+ rte_delay_us(us);
+}
+
+void
+spdk_unaffinitize_thread(void)
+{
+ rte_cpuset_t new_cpuset;
+ long num_cores, i;
+
+ CPU_ZERO(&new_cpuset);
+
+ num_cores = sysconf(_SC_NPROCESSORS_CONF);
+
+ /* Create a mask containing all CPUs */
+ for (i = 0; i < num_cores; i++) {
+ CPU_SET(i, &new_cpuset);
+ }
+
+ rte_thread_set_affinity(&new_cpuset);
+}
+
+void *
+spdk_call_unaffinitized(void *cb(void *arg), void *arg)
+{
+ rte_cpuset_t orig_cpuset;
+ void *ret;
+
+ if (cb == NULL) {
+ return NULL;
+ }
+
+ rte_thread_get_affinity(&orig_cpuset);
+
+ spdk_unaffinitize_thread();
+
+ ret = cb(arg);
+
+ rte_thread_set_affinity(&orig_cpuset);
+
+ return ret;
+}
+
+struct spdk_ring *
+spdk_ring_create(enum spdk_ring_type type, size_t count, int socket_id)
+{
+ char ring_name[64];
+ static uint32_t ring_num = 0;
+ unsigned flags = 0;
+
+ switch (type) {
+ case SPDK_RING_TYPE_SP_SC:
+ flags = RING_F_SP_ENQ | RING_F_SC_DEQ;
+ break;
+ case SPDK_RING_TYPE_MP_SC:
+ flags = RING_F_SC_DEQ;
+ break;
+ case SPDK_RING_TYPE_MP_MC:
+ flags = 0;
+ break;
+ default:
+ return NULL;
+ }
+
+ snprintf(ring_name, sizeof(ring_name), "ring_%u_%d",
+ __sync_fetch_and_add(&ring_num, 1), getpid());
+
+ return (struct spdk_ring *)rte_ring_create(ring_name, count, socket_id, flags);
+}
+
+void
+spdk_ring_free(struct spdk_ring *ring)
+{
+ rte_ring_free((struct rte_ring *)ring);
+}
+
+size_t
+spdk_ring_count(struct spdk_ring *ring)
+{
+ return rte_ring_count((struct rte_ring *)ring);
+}
+
+size_t
+spdk_ring_enqueue(struct spdk_ring *ring, void **objs, size_t count)
+{
+ int rc;
+#if RTE_VERSION < RTE_VERSION_NUM(17, 5, 0, 0)
+ rc = rte_ring_enqueue_bulk((struct rte_ring *)ring, objs, count);
+ if (rc == 0) {
+ return count;
+ }
+
+ return 0;
+#else
+ rc = rte_ring_enqueue_bulk((struct rte_ring *)ring, objs, count, NULL);
+ return rc;
+#endif
+}
+
+size_t
+spdk_ring_dequeue(struct spdk_ring *ring, void **objs, size_t count)
+{
+#if RTE_VERSION < RTE_VERSION_NUM(17, 5, 0, 0)
+ return rte_ring_dequeue_burst((struct rte_ring *)ring, objs, count);
+#else
+ return rte_ring_dequeue_burst((struct rte_ring *)ring, objs, count, NULL);
+#endif
+}