diff options
Diffstat (limited to 'src/seastar/dpdk/drivers/mempool')
11 files changed, 997 insertions, 0 deletions
diff --git a/src/seastar/dpdk/drivers/mempool/Makefile b/src/seastar/dpdk/drivers/mempool/Makefile new file mode 100644 index 00000000..8fd40e1b --- /dev/null +++ b/src/seastar/dpdk/drivers/mempool/Makefile @@ -0,0 +1,43 @@ +# BSD LICENSE +# +# Copyright(c) 2017 NXP. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of NXP nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +core-libs := librte_eal librte_mempool librte_ring + +DIRS-$(CONFIG_RTE_LIBRTE_DPAA2_MEMPOOL) += dpaa2 +DEPDIRS-dpaa2 = $(core-libs) +DIRS-$(CONFIG_RTE_DRIVER_MEMPOOL_RING) += ring +DEPDIRS-ring = $(core-libs) +DIRS-$(CONFIG_RTE_DRIVER_MEMPOOL_STACK) += stack +DEPDIRS-stack = $(core-libs) + +include $(RTE_SDK)/mk/rte.subdir.mk diff --git a/src/seastar/dpdk/drivers/mempool/dpaa2/Makefile b/src/seastar/dpdk/drivers/mempool/dpaa2/Makefile new file mode 100644 index 00000000..26ccebde --- /dev/null +++ b/src/seastar/dpdk/drivers/mempool/dpaa2/Makefile @@ -0,0 +1,63 @@ +# BSD LICENSE +# +# Copyright(c) 2016 NXP. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of NXP nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +# +# library name +# +LIB = librte_mempool_dpaa2.a + +ifeq ($(CONFIG_RTE_LIBRTE_DPAA2_DEBUG_INIT),y) +CFLAGS += -O0 -g +CFLAGS += "-Wno-error" +else +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) +endif + +CFLAGS += -I$(RTE_SDK)/drivers/bus/fslmc +CFLAGS += -I$(RTE_SDK)/drivers/bus/fslmc/qbman/include +CFLAGS += -I$(RTE_SDK)/lib/librte_eal/linuxapp/eal + +# versioning export map +EXPORT_MAP := rte_mempool_dpaa2_version.map + +# Lbrary version +LIBABIVER := 1 + +# all source are stored in SRCS-y +# +SRCS-$(CONFIG_RTE_LIBRTE_DPAA2_MEMPOOL) += dpaa2_hw_mempool.c + +LDLIBS += -lrte_bus_fslmc + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/src/seastar/dpdk/drivers/mempool/dpaa2/dpaa2_hw_mempool.c b/src/seastar/dpdk/drivers/mempool/dpaa2/dpaa2_hw_mempool.c new file mode 100644 index 00000000..5a5d6aaa --- /dev/null +++ b/src/seastar/dpdk/drivers/mempool/dpaa2/dpaa2_hw_mempool.c @@ -0,0 +1,373 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. + * Copyright (c) 2016 NXP. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Freescale Semiconductor, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <unistd.h> +#include <stdio.h> +#include <sys/types.h> +#include <string.h> +#include <stdlib.h> +#include <fcntl.h> +#include <errno.h> + +#include <rte_mbuf.h> +#include <rte_ethdev.h> +#include <rte_malloc.h> +#include <rte_memcpy.h> +#include <rte_string_fns.h> +#include <rte_cycles.h> +#include <rte_kvargs.h> +#include <rte_dev.h> +#include <rte_ethdev.h> + +#include <fslmc_logs.h> +#include <mc/fsl_dpbp.h> +#include <portal/dpaa2_hw_pvt.h> +#include <portal/dpaa2_hw_dpio.h> +#include "dpaa2_hw_mempool.h" + +struct dpaa2_bp_info rte_dpaa2_bpid_info[MAX_BPID]; +static struct dpaa2_bp_list *h_bp_list; + +static int +rte_hw_mbuf_create_pool(struct rte_mempool *mp) +{ + struct dpaa2_bp_list *bp_list; + struct dpaa2_dpbp_dev *avail_dpbp; + struct dpbp_attr dpbp_attr; + uint32_t bpid; + int ret, p_ret; + + avail_dpbp = dpaa2_alloc_dpbp_dev(); + + if (!avail_dpbp) { + PMD_DRV_LOG(ERR, "DPAA2 resources not available"); + return -ENOENT; + } + + if (unlikely(!DPAA2_PER_LCORE_DPIO)) { + ret = dpaa2_affine_qbman_swp(); + if (ret) { + RTE_LOG(ERR, PMD, "Failure in affining portal\n"); + return ret; + } + } + + ret = dpbp_enable(&avail_dpbp->dpbp, CMD_PRI_LOW, avail_dpbp->token); + if (ret != 0) { + PMD_INIT_LOG(ERR, "Resource enable failure with" + " err code: %d\n", ret); + return ret; + } + + ret = dpbp_get_attributes(&avail_dpbp->dpbp, CMD_PRI_LOW, + avail_dpbp->token, &dpbp_attr); + if (ret != 0) { + PMD_INIT_LOG(ERR, "Resource read failure with" + " err code: %d\n", ret); + p_ret = ret; + ret = dpbp_disable(&avail_dpbp->dpbp, CMD_PRI_LOW, + avail_dpbp->token); + return p_ret; + } + + /* Allocate the bp_list which will be added into global_bp_list */ + bp_list = rte_malloc(NULL, sizeof(struct dpaa2_bp_list), + RTE_CACHE_LINE_SIZE); + if (!bp_list) { + PMD_INIT_LOG(ERR, "No heap memory available"); + return -ENOMEM; + } + + /* Set parameters of buffer pool list */ + bp_list->buf_pool.num_bufs = mp->size; + bp_list->buf_pool.size = mp->elt_size + - sizeof(struct rte_mbuf) - rte_pktmbuf_priv_size(mp); + bp_list->buf_pool.bpid = dpbp_attr.bpid; + bp_list->buf_pool.h_bpool_mem = NULL; + bp_list->buf_pool.dpbp_node = avail_dpbp; + /* Identification for our offloaded pool_data structure */ + bp_list->dpaa2_ops_index = mp->ops_index; + bp_list->next = h_bp_list; + bp_list->mp = mp; + + bpid = dpbp_attr.bpid; + + rte_dpaa2_bpid_info[bpid].meta_data_size = sizeof(struct rte_mbuf) + + rte_pktmbuf_priv_size(mp); + rte_dpaa2_bpid_info[bpid].bp_list = bp_list; + rte_dpaa2_bpid_info[bpid].bpid = bpid; + + mp->pool_data = (void *)&rte_dpaa2_bpid_info[bpid]; + + PMD_INIT_LOG(DEBUG, "BP List created for bpid =%d", dpbp_attr.bpid); + + h_bp_list = bp_list; + return 0; +} + +static void +rte_hw_mbuf_free_pool(struct rte_mempool *mp) +{ + struct dpaa2_bp_info *bpinfo; + struct dpaa2_bp_list *bp; + struct dpaa2_dpbp_dev *dpbp_node; + + if (!mp->pool_data) { + PMD_DRV_LOG(ERR, "Not a valid dpaa22 pool"); + return; + } + + bpinfo = (struct dpaa2_bp_info *)mp->pool_data; + bp = bpinfo->bp_list; + dpbp_node = bp->buf_pool.dpbp_node; + + dpbp_disable(&(dpbp_node->dpbp), CMD_PRI_LOW, dpbp_node->token); + + if (h_bp_list == bp) { + h_bp_list = h_bp_list->next; + } else { /* if it is not the first node */ + struct dpaa2_bp_list *prev = h_bp_list, *temp; + temp = h_bp_list->next; + while (temp) { + if (temp == bp) { + prev->next = temp->next; + free(bp); + break; + } + prev = temp; + temp = temp->next; + } + } + + dpaa2_free_dpbp_dev(dpbp_node); +} + +static void +rte_dpaa2_mbuf_release(struct rte_mempool *pool __rte_unused, + void * const *obj_table, + uint32_t bpid, + uint32_t meta_data_size, + int count) +{ + struct qbman_release_desc releasedesc; + struct qbman_swp *swp; + int ret; + int i, n; + uint64_t bufs[DPAA2_MBUF_MAX_ACQ_REL]; + + if (unlikely(!DPAA2_PER_LCORE_DPIO)) { + ret = dpaa2_affine_qbman_swp(); + if (ret != 0) { + RTE_LOG(ERR, PMD, "Failed to allocate IO portal"); + return; + } + } + swp = DPAA2_PER_LCORE_PORTAL; + + /* Create a release descriptor required for releasing + * buffers into QBMAN + */ + qbman_release_desc_clear(&releasedesc); + qbman_release_desc_set_bpid(&releasedesc, bpid); + + n = count % DPAA2_MBUF_MAX_ACQ_REL; + if (unlikely(!n)) + goto aligned; + + /* convert mbuf to buffers for the remainder */ + for (i = 0; i < n ; i++) { +#ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA + bufs[i] = (uint64_t)rte_mempool_virt2phy(pool, obj_table[i]) + + meta_data_size; +#else + bufs[i] = (uint64_t)obj_table[i] + meta_data_size; +#endif + } + + /* feed them to bman */ + do { + ret = qbman_swp_release(swp, &releasedesc, bufs, n); + } while (ret == -EBUSY); + +aligned: + /* if there are more buffers to free */ + while (n < count) { + /* convert mbuf to buffers */ + for (i = 0; i < DPAA2_MBUF_MAX_ACQ_REL; i++) { +#ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA + bufs[i] = (uint64_t) + rte_mempool_virt2phy(pool, obj_table[n + i]) + + meta_data_size; +#else + bufs[i] = (uint64_t)obj_table[n + i] + meta_data_size; +#endif + } + + do { + ret = qbman_swp_release(swp, &releasedesc, bufs, + DPAA2_MBUF_MAX_ACQ_REL); + } while (ret == -EBUSY); + n += DPAA2_MBUF_MAX_ACQ_REL; + } +} + +int +rte_dpaa2_mbuf_alloc_bulk(struct rte_mempool *pool, + void **obj_table, unsigned int count) +{ +#ifdef RTE_LIBRTE_DPAA2_DEBUG_DRIVER + static int alloc; +#endif + struct qbman_swp *swp; + uint16_t bpid; + uint64_t bufs[DPAA2_MBUF_MAX_ACQ_REL]; + int i, ret; + unsigned int n = 0; + struct dpaa2_bp_info *bp_info; + + bp_info = mempool_to_bpinfo(pool); + + if (!(bp_info->bp_list)) { + RTE_LOG(ERR, PMD, "DPAA2 buffer pool not configured\n"); + return -ENOENT; + } + + bpid = bp_info->bpid; + + if (unlikely(!DPAA2_PER_LCORE_DPIO)) { + ret = dpaa2_affine_qbman_swp(); + if (ret != 0) { + RTE_LOG(ERR, PMD, "Failed to allocate IO portal"); + return ret; + } + } + swp = DPAA2_PER_LCORE_PORTAL; + + while (n < count) { + /* Acquire is all-or-nothing, so we drain in 7s, + * then the remainder. + */ + if ((count - n) > DPAA2_MBUF_MAX_ACQ_REL) { + ret = qbman_swp_acquire(swp, bpid, bufs, + DPAA2_MBUF_MAX_ACQ_REL); + } else { + ret = qbman_swp_acquire(swp, bpid, bufs, + count - n); + } + /* In case of less than requested number of buffers available + * in pool, qbman_swp_acquire returns 0 + */ + if (ret <= 0) { + PMD_TX_LOG(ERR, "Buffer acquire failed with" + " err code: %d", ret); + /* The API expect the exact number of requested bufs */ + /* Releasing all buffers allocated */ + rte_dpaa2_mbuf_release(pool, obj_table, bpid, + bp_info->meta_data_size, n); + return ret; + } + /* assigning mbuf from the acquired objects */ + for (i = 0; (i < ret) && bufs[i]; i++) { + DPAA2_MODIFY_IOVA_TO_VADDR(bufs[i], uint64_t); + obj_table[n] = (struct rte_mbuf *) + (bufs[i] - bp_info->meta_data_size); + PMD_TX_LOG(DEBUG, "Acquired %p address %p from BMAN", + (void *)bufs[i], (void *)obj_table[n]); + n++; + } + } + +#ifdef RTE_LIBRTE_DPAA2_DEBUG_DRIVER + alloc += n; + PMD_TX_LOG(DEBUG, "Total = %d , req = %d done = %d", + alloc, count, n); +#endif + return 0; +} + +static int +rte_hw_mbuf_free_bulk(struct rte_mempool *pool, + void * const *obj_table, unsigned int n) +{ + struct dpaa2_bp_info *bp_info; + + bp_info = mempool_to_bpinfo(pool); + if (!(bp_info->bp_list)) { + RTE_LOG(ERR, PMD, "DPAA2 buffer pool not configured"); + return -ENOENT; + } + rte_dpaa2_mbuf_release(pool, obj_table, bp_info->bpid, + bp_info->meta_data_size, n); + + return 0; +} + +static unsigned int +rte_hw_mbuf_get_count(const struct rte_mempool *mp) +{ + int ret; + unsigned int num_of_bufs = 0; + struct dpaa2_bp_info *bp_info; + struct dpaa2_dpbp_dev *dpbp_node; + + if (!mp || !mp->pool_data) { + RTE_LOG(ERR, PMD, "Invalid mempool provided"); + return 0; + } + + bp_info = (struct dpaa2_bp_info *)mp->pool_data; + dpbp_node = bp_info->bp_list->buf_pool.dpbp_node; + + ret = dpbp_get_num_free_bufs(&dpbp_node->dpbp, CMD_PRI_LOW, + dpbp_node->token, &num_of_bufs); + if (ret) { + RTE_LOG(ERR, PMD, "Unable to obtain free buf count (err=%d)", + ret); + return 0; + } + + RTE_LOG(DEBUG, PMD, "Free bufs = %u", num_of_bufs); + + return num_of_bufs; +} + +struct rte_mempool_ops dpaa2_mpool_ops = { + .name = "dpaa2", + .alloc = rte_hw_mbuf_create_pool, + .free = rte_hw_mbuf_free_pool, + .enqueue = rte_hw_mbuf_free_bulk, + .dequeue = rte_dpaa2_mbuf_alloc_bulk, + .get_count = rte_hw_mbuf_get_count, +}; + +MEMPOOL_REGISTER_OPS(dpaa2_mpool_ops); diff --git a/src/seastar/dpdk/drivers/mempool/dpaa2/dpaa2_hw_mempool.h b/src/seastar/dpdk/drivers/mempool/dpaa2/dpaa2_hw_mempool.h new file mode 100644 index 00000000..c4d7fc02 --- /dev/null +++ b/src/seastar/dpdk/drivers/mempool/dpaa2/dpaa2_hw_mempool.h @@ -0,0 +1,91 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. + * Copyright (c) 2016 NXP. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Freescale Semiconductor, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _DPAA2_HW_DPBP_H_ +#define _DPAA2_HW_DPBP_H_ + +#define DPAA2_MAX_BUF_POOLS 8 + +struct buf_pool_cfg { + void *addr; + /**< The address from where DPAA2 will carve out the buffers */ + phys_addr_t phys_addr; + /**< Physical address of the memory provided in addr */ + uint32_t num; + /**< Number of buffers */ + uint32_t size; + /**< Size including headroom for each buffer */ + uint16_t align; + /**< Buffer alignment (in bytes) */ + uint16_t bpid; + /**< Autogenerated buffer pool ID for internal use */ +}; + +struct buf_pool { + uint32_t size; /**< Size of the Pool */ + uint32_t num_bufs; /**< Number of buffers in Pool */ + uint16_t bpid; /**< Pool ID, from pool configuration */ + uint8_t *h_bpool_mem; /**< Internal context data */ + struct dpaa2_dpbp_dev *dpbp_node; /**< Hardware context */ +}; + +/*! + * Buffer pool list configuration structure. User need to give DPAA2 the + * valid number of 'num_buf_pools'. + */ +struct dpaa2_bp_list_cfg { + struct buf_pool_cfg buf_pool; /* Configuration of each buffer pool*/ +}; + +struct dpaa2_bp_list { + struct dpaa2_bp_list *next; + struct rte_mempool *mp; /**< DPDK RTE EAL pool reference */ + int32_t dpaa2_ops_index; /**< Index into DPDK Mempool ops table */ + struct buf_pool buf_pool; +}; + +struct dpaa2_bp_info { + uint32_t meta_data_size; + uint32_t bpid; + struct dpaa2_bp_list *bp_list; +}; + +#define mempool_to_bpinfo(mp) ((struct dpaa2_bp_info *)(mp)->pool_data) +#define mempool_to_bpid(mp) ((mempool_to_bpinfo(mp))->bpid) + +extern struct dpaa2_bp_info rte_dpaa2_bpid_info[MAX_BPID]; + +int rte_dpaa2_mbuf_alloc_bulk(struct rte_mempool *pool, + void **obj_table, unsigned int count); + +#endif /* _DPAA2_HW_DPBP_H_ */ diff --git a/src/seastar/dpdk/drivers/mempool/dpaa2/rte_mempool_dpaa2_version.map b/src/seastar/dpdk/drivers/mempool/dpaa2/rte_mempool_dpaa2_version.map new file mode 100644 index 00000000..a8aa685c --- /dev/null +++ b/src/seastar/dpdk/drivers/mempool/dpaa2/rte_mempool_dpaa2_version.map @@ -0,0 +1,8 @@ +DPDK_17.05 { + global: + + rte_dpaa2_bpid_info; + rte_dpaa2_mbuf_alloc_bulk; + + local: *; +}; diff --git a/src/seastar/dpdk/drivers/mempool/ring/Makefile b/src/seastar/dpdk/drivers/mempool/ring/Makefile new file mode 100644 index 00000000..54630d99 --- /dev/null +++ b/src/seastar/dpdk/drivers/mempool/ring/Makefile @@ -0,0 +1,48 @@ +# BSD LICENSE +# +# Copyright(c) 2017 NXP. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of NXP nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +# +# library name +# +LIB = librte_mempool_ring.a + +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) + +EXPORT_MAP := rte_mempool_ring_version.map + +LIBABIVER := 1 + +SRCS-$(CONFIG_RTE_DRIVER_MEMPOOL_RING) += rte_mempool_ring.c + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/src/seastar/dpdk/drivers/mempool/ring/rte_mempool_ring.c b/src/seastar/dpdk/drivers/mempool/ring/rte_mempool_ring.c new file mode 100644 index 00000000..5c132bf6 --- /dev/null +++ b/src/seastar/dpdk/drivers/mempool/ring/rte_mempool_ring.c @@ -0,0 +1,165 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <string.h> + +#include <rte_errno.h> +#include <rte_ring.h> +#include <rte_mempool.h> + +static int +common_ring_mp_enqueue(struct rte_mempool *mp, void * const *obj_table, + unsigned n) +{ + return rte_ring_mp_enqueue_bulk(mp->pool_data, + obj_table, n, NULL) == 0 ? -ENOBUFS : 0; +} + +static int +common_ring_sp_enqueue(struct rte_mempool *mp, void * const *obj_table, + unsigned n) +{ + return rte_ring_sp_enqueue_bulk(mp->pool_data, + obj_table, n, NULL) == 0 ? -ENOBUFS : 0; +} + +static int +common_ring_mc_dequeue(struct rte_mempool *mp, void **obj_table, unsigned n) +{ + return rte_ring_mc_dequeue_bulk(mp->pool_data, + obj_table, n, NULL) == 0 ? -ENOBUFS : 0; +} + +static int +common_ring_sc_dequeue(struct rte_mempool *mp, void **obj_table, unsigned n) +{ + return rte_ring_sc_dequeue_bulk(mp->pool_data, + obj_table, n, NULL) == 0 ? -ENOBUFS : 0; +} + +static unsigned +common_ring_get_count(const struct rte_mempool *mp) +{ + return rte_ring_count(mp->pool_data); +} + + +static int +common_ring_alloc(struct rte_mempool *mp) +{ + int rg_flags = 0, ret; + char rg_name[RTE_RING_NAMESIZE]; + struct rte_ring *r; + + ret = snprintf(rg_name, sizeof(rg_name), + RTE_MEMPOOL_MZ_FORMAT, mp->name); + if (ret < 0 || ret >= (int)sizeof(rg_name)) { + rte_errno = ENAMETOOLONG; + return -rte_errno; + } + + /* ring flags */ + if (mp->flags & MEMPOOL_F_SP_PUT) + rg_flags |= RING_F_SP_ENQ; + if (mp->flags & MEMPOOL_F_SC_GET) + rg_flags |= RING_F_SC_DEQ; + + /* + * Allocate the ring that will be used to store objects. + * Ring functions will return appropriate errors if we are + * running as a secondary process etc., so no checks made + * in this function for that condition. + */ + r = rte_ring_create(rg_name, rte_align32pow2(mp->size + 1), + mp->socket_id, rg_flags); + if (r == NULL) + return -rte_errno; + + mp->pool_data = r; + + return 0; +} + +static void +common_ring_free(struct rte_mempool *mp) +{ + rte_ring_free(mp->pool_data); +} + +/* + * The following 4 declarations of mempool ops structs address + * the need for the backward compatible mempool handlers for + * single/multi producers and single/multi consumers as dictated by the + * flags provided to the rte_mempool_create function + */ +static const struct rte_mempool_ops ops_mp_mc = { + .name = "ring_mp_mc", + .alloc = common_ring_alloc, + .free = common_ring_free, + .enqueue = common_ring_mp_enqueue, + .dequeue = common_ring_mc_dequeue, + .get_count = common_ring_get_count, +}; + +static const struct rte_mempool_ops ops_sp_sc = { + .name = "ring_sp_sc", + .alloc = common_ring_alloc, + .free = common_ring_free, + .enqueue = common_ring_sp_enqueue, + .dequeue = common_ring_sc_dequeue, + .get_count = common_ring_get_count, +}; + +static const struct rte_mempool_ops ops_mp_sc = { + .name = "ring_mp_sc", + .alloc = common_ring_alloc, + .free = common_ring_free, + .enqueue = common_ring_mp_enqueue, + .dequeue = common_ring_sc_dequeue, + .get_count = common_ring_get_count, +}; + +static const struct rte_mempool_ops ops_sp_mc = { + .name = "ring_sp_mc", + .alloc = common_ring_alloc, + .free = common_ring_free, + .enqueue = common_ring_sp_enqueue, + .dequeue = common_ring_mc_dequeue, + .get_count = common_ring_get_count, +}; + +MEMPOOL_REGISTER_OPS(ops_mp_mc); +MEMPOOL_REGISTER_OPS(ops_sp_sc); +MEMPOOL_REGISTER_OPS(ops_mp_sc); +MEMPOOL_REGISTER_OPS(ops_sp_mc); diff --git a/src/seastar/dpdk/drivers/mempool/ring/rte_mempool_ring_version.map b/src/seastar/dpdk/drivers/mempool/ring/rte_mempool_ring_version.map new file mode 100644 index 00000000..8591cc0b --- /dev/null +++ b/src/seastar/dpdk/drivers/mempool/ring/rte_mempool_ring_version.map @@ -0,0 +1,4 @@ +DPDK_17.05 { + + local: *; +}; diff --git a/src/seastar/dpdk/drivers/mempool/stack/Makefile b/src/seastar/dpdk/drivers/mempool/stack/Makefile new file mode 100644 index 00000000..8f3125c7 --- /dev/null +++ b/src/seastar/dpdk/drivers/mempool/stack/Makefile @@ -0,0 +1,51 @@ +# BSD LICENSE +# +# Copyright(c) 2017 NXP. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of NXP nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +# +# library name +# +LIB = librte_mempool_stack.a + +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) + +# Headers +CFLAGS += -I$(RTE_SDK)/lib/librte_mempool + +EXPORT_MAP := rte_mempool_stack_version.map + +LIBABIVER := 1 + +SRCS-$(CONFIG_RTE_DRIVER_MEMPOOL_STACK) += rte_mempool_stack.c + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/src/seastar/dpdk/drivers/mempool/stack/rte_mempool_stack.c b/src/seastar/dpdk/drivers/mempool/stack/rte_mempool_stack.c new file mode 100644 index 00000000..817f77e6 --- /dev/null +++ b/src/seastar/dpdk/drivers/mempool/stack/rte_mempool_stack.c @@ -0,0 +1,147 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <rte_mempool.h> +#include <rte_malloc.h> + +struct rte_mempool_stack { + rte_spinlock_t sl; + + uint32_t size; + uint32_t len; + void *objs[]; +}; + +static int +stack_alloc(struct rte_mempool *mp) +{ + struct rte_mempool_stack *s; + unsigned n = mp->size; + int size = sizeof(*s) + (n+16)*sizeof(void *); + + /* Allocate our local memory structure */ + s = rte_zmalloc_socket("mempool-stack", + size, + RTE_CACHE_LINE_SIZE, + mp->socket_id); + if (s == NULL) { + RTE_LOG(ERR, MEMPOOL, "Cannot allocate stack!\n"); + return -ENOMEM; + } + + rte_spinlock_init(&s->sl); + + s->size = n; + mp->pool_data = s; + + return 0; +} + +static int +stack_enqueue(struct rte_mempool *mp, void * const *obj_table, + unsigned n) +{ + struct rte_mempool_stack *s = mp->pool_data; + void **cache_objs; + unsigned index; + + rte_spinlock_lock(&s->sl); + cache_objs = &s->objs[s->len]; + + /* Is there sufficient space in the stack ? */ + if ((s->len + n) > s->size) { + rte_spinlock_unlock(&s->sl); + return -ENOBUFS; + } + + /* Add elements back into the cache */ + for (index = 0; index < n; ++index, obj_table++) + cache_objs[index] = *obj_table; + + s->len += n; + + rte_spinlock_unlock(&s->sl); + return 0; +} + +static int +stack_dequeue(struct rte_mempool *mp, void **obj_table, + unsigned n) +{ + struct rte_mempool_stack *s = mp->pool_data; + void **cache_objs; + unsigned index, len; + + rte_spinlock_lock(&s->sl); + + if (unlikely(n > s->len)) { + rte_spinlock_unlock(&s->sl); + return -ENOENT; + } + + cache_objs = s->objs; + + for (index = 0, len = s->len - 1; index < n; + ++index, len--, obj_table++) + *obj_table = cache_objs[len]; + + s->len -= n; + rte_spinlock_unlock(&s->sl); + return 0; +} + +static unsigned +stack_get_count(const struct rte_mempool *mp) +{ + struct rte_mempool_stack *s = mp->pool_data; + + return s->len; +} + +static void +stack_free(struct rte_mempool *mp) +{ + rte_free((void *)(mp->pool_data)); +} + +static struct rte_mempool_ops ops_stack = { + .name = "stack", + .alloc = stack_alloc, + .free = stack_free, + .enqueue = stack_enqueue, + .dequeue = stack_dequeue, + .get_count = stack_get_count +}; + +MEMPOOL_REGISTER_OPS(ops_stack); diff --git a/src/seastar/dpdk/drivers/mempool/stack/rte_mempool_stack_version.map b/src/seastar/dpdk/drivers/mempool/stack/rte_mempool_stack_version.map new file mode 100644 index 00000000..8591cc0b --- /dev/null +++ b/src/seastar/dpdk/drivers/mempool/stack/rte_mempool_stack_version.map @@ -0,0 +1,4 @@ +DPDK_17.05 { + + local: *; +}; |