From e6918187568dbd01842d8d1d2c808ce16a894239 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 21 Apr 2024 13:54:28 +0200 Subject: Adding upstream version 18.2.2. Signed-off-by: Daniel Baumann --- src/spdk/dpdk/drivers/net/iavf/Makefile | 54 + src/spdk/dpdk/drivers/net/iavf/iavf.h | 275 ++ src/spdk/dpdk/drivers/net/iavf/iavf_ethdev.c | 1586 +++++++++++ src/spdk/dpdk/drivers/net/iavf/iavf_fdir.c | 971 +++++++ src/spdk/dpdk/drivers/net/iavf/iavf_generic_flow.c | 1044 +++++++ src/spdk/dpdk/drivers/net/iavf/iavf_generic_flow.h | 320 +++ src/spdk/dpdk/drivers/net/iavf/iavf_hash.c | 1236 +++++++++ src/spdk/dpdk/drivers/net/iavf/iavf_log.h | 51 + src/spdk/dpdk/drivers/net/iavf/iavf_rxtx.c | 2869 ++++++++++++++++++++ src/spdk/dpdk/drivers/net/iavf/iavf_rxtx.h | 534 ++++ .../dpdk/drivers/net/iavf/iavf_rxtx_vec_avx2.c | 1541 +++++++++++ .../dpdk/drivers/net/iavf/iavf_rxtx_vec_common.h | 276 ++ src/spdk/dpdk/drivers/net/iavf/iavf_rxtx_vec_sse.c | 1191 ++++++++ src/spdk/dpdk/drivers/net/iavf/iavf_vchnl.c | 1077 ++++++++ src/spdk/dpdk/drivers/net/iavf/meson.build | 37 + .../dpdk/drivers/net/iavf/rte_pmd_iavf_version.map | 3 + 16 files changed, 13065 insertions(+) create mode 100644 src/spdk/dpdk/drivers/net/iavf/Makefile create mode 100644 src/spdk/dpdk/drivers/net/iavf/iavf.h create mode 100644 src/spdk/dpdk/drivers/net/iavf/iavf_ethdev.c create mode 100644 src/spdk/dpdk/drivers/net/iavf/iavf_fdir.c create mode 100644 src/spdk/dpdk/drivers/net/iavf/iavf_generic_flow.c create mode 100644 src/spdk/dpdk/drivers/net/iavf/iavf_generic_flow.h create mode 100644 src/spdk/dpdk/drivers/net/iavf/iavf_hash.c create mode 100644 src/spdk/dpdk/drivers/net/iavf/iavf_log.h create mode 100644 src/spdk/dpdk/drivers/net/iavf/iavf_rxtx.c create mode 100644 src/spdk/dpdk/drivers/net/iavf/iavf_rxtx.h create mode 100644 src/spdk/dpdk/drivers/net/iavf/iavf_rxtx_vec_avx2.c create mode 100644 src/spdk/dpdk/drivers/net/iavf/iavf_rxtx_vec_common.h create mode 100644 src/spdk/dpdk/drivers/net/iavf/iavf_rxtx_vec_sse.c create mode 100644 src/spdk/dpdk/drivers/net/iavf/iavf_vchnl.c create mode 100644 src/spdk/dpdk/drivers/net/iavf/meson.build create mode 100644 src/spdk/dpdk/drivers/net/iavf/rte_pmd_iavf_version.map (limited to 'src/spdk/dpdk/drivers/net/iavf') diff --git a/src/spdk/dpdk/drivers/net/iavf/Makefile b/src/spdk/dpdk/drivers/net/iavf/Makefile new file mode 100644 index 000000000..792cbb7f7 --- /dev/null +++ b/src/spdk/dpdk/drivers/net/iavf/Makefile @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2017 Intel Corporation + +include $(RTE_SDK)/mk/rte.vars.mk + +# +# library name +# +LIB = librte_pmd_iavf.a + +CFLAGS += -I$(RTE_SDK)/drivers/common/iavf +CFLAGS += -O3 $(WERROR_FLAGS) -Wno-strict-aliasing +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs -lrte_hash +LDLIBS += -lrte_bus_pci +LDLIBS += -lrte_common_iavf + +EXPORT_MAP := rte_pmd_iavf_version.map + +# +# all source are stored in SRCS-y +# +SRCS-$(CONFIG_RTE_LIBRTE_IAVF_PMD) += iavf_ethdev.c +SRCS-$(CONFIG_RTE_LIBRTE_IAVF_PMD) += iavf_vchnl.c +SRCS-$(CONFIG_RTE_LIBRTE_IAVF_PMD) += iavf_rxtx.c +SRCS-$(CONFIG_RTE_LIBRTE_IAVF_PMD) += iavf_generic_flow.c +SRCS-$(CONFIG_RTE_LIBRTE_IAVF_PMD) += iavf_fdir.c +SRCS-$(CONFIG_RTE_LIBRTE_IAVF_PMD) += iavf_hash.c +ifeq ($(CONFIG_RTE_ARCH_X86), y) +SRCS-$(CONFIG_RTE_LIBRTE_IAVF_PMD) += iavf_rxtx_vec_sse.c +endif + +ifeq ($(CONFIG_RTE_LIBRTE_IAVF_PMD), y) + ifeq ($(findstring RTE_MACHINE_CPUFLAG_AVX2,$(CFLAGS)),RTE_MACHINE_CPUFLAG_AVX2) + CC_AVX2_SUPPORT=1 + else + CC_AVX2_SUPPORT=\ + $(shell $(CC) -march=core-avx2 -dM -E - &1 | \ + grep -q AVX2 && echo 1) + ifeq ($(CC_AVX2_SUPPORT), 1) + ifeq ($(CONFIG_RTE_TOOLCHAIN_ICC),y) + CFLAGS_iavf_rxtx_vec_avx2.o += -march=core-avx2 + else + CFLAGS_iavf_rxtx_vec_avx2.o += -mavx2 + endif + endif + endif +endif + +ifeq ($(CC_AVX2_SUPPORT), 1) + SRCS-$(CONFIG_RTE_LIBRTE_IAVF_PMD) += iavf_rxtx_vec_avx2.c +endif + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/src/spdk/dpdk/drivers/net/iavf/iavf.h b/src/spdk/dpdk/drivers/net/iavf/iavf.h new file mode 100644 index 000000000..9be8a2381 --- /dev/null +++ b/src/spdk/dpdk/drivers/net/iavf/iavf.h @@ -0,0 +1,275 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017 Intel Corporation + */ + +#ifndef _IAVF_ETHDEV_H_ +#define _IAVF_ETHDEV_H_ + +#include +#include +#include +#include + +#include "iavf_log.h" + +#define IAVF_AQ_LEN 32 +#define IAVF_AQ_BUF_SZ 4096 +#define IAVF_RESET_WAIT_CNT 50 +#define IAVF_BUF_SIZE_MIN 1024 +#define IAVF_FRAME_SIZE_MAX 9728 +#define IAVF_QUEUE_BASE_ADDR_UNIT 128 + +#define IAVF_MAX_NUM_QUEUES 16 + +#define IAVF_NUM_MACADDR_MAX 64 + +#define IAVF_DEFAULT_RX_PTHRESH 8 +#define IAVF_DEFAULT_RX_HTHRESH 8 +#define IAVF_DEFAULT_RX_WTHRESH 0 + +#define IAVF_DEFAULT_RX_FREE_THRESH 32 + +#define IAVF_DEFAULT_TX_PTHRESH 32 +#define IAVF_DEFAULT_TX_HTHRESH 0 +#define IAVF_DEFAULT_TX_WTHRESH 0 + +#define IAVF_DEFAULT_TX_FREE_THRESH 32 +#define IAVF_DEFAULT_TX_RS_THRESH 32 + +#define IAVF_BASIC_OFFLOAD_CAPS ( \ + VF_BASE_MODE_OFFLOADS | \ + VIRTCHNL_VF_OFFLOAD_WB_ON_ITR | \ + VIRTCHNL_VF_OFFLOAD_RX_POLLING) + +#define IAVF_RSS_OFFLOAD_ALL ( \ + ETH_RSS_FRAG_IPV4 | \ + ETH_RSS_NONFRAG_IPV4_TCP | \ + ETH_RSS_NONFRAG_IPV4_UDP | \ + ETH_RSS_NONFRAG_IPV4_SCTP | \ + ETH_RSS_NONFRAG_IPV4_OTHER) + +#define IAVF_MISC_VEC_ID RTE_INTR_VEC_ZERO_OFFSET +#define IAVF_RX_VEC_START RTE_INTR_VEC_RXTX_OFFSET + +/* Default queue interrupt throttling time in microseconds */ +#define IAVF_ITR_INDEX_DEFAULT 0 +#define IAVF_QUEUE_ITR_INTERVAL_DEFAULT 32 /* 32 us */ +#define IAVF_QUEUE_ITR_INTERVAL_MAX 8160 /* 8160 us */ + +/* The overhead from MTU to max frame size. + * Considering QinQ packet, the VLAN tag needs to be counted twice. + */ +#define IAVF_VLAN_TAG_SIZE 4 +#define IAVF_ETH_OVERHEAD \ + (RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN + IAVF_VLAN_TAG_SIZE * 2) + +#define IAVF_32_BIT_WIDTH (CHAR_BIT * 4) +#define IAVF_48_BIT_WIDTH (CHAR_BIT * 6) +#define IAVF_48_BIT_MASK RTE_LEN2MASK(IAVF_48_BIT_WIDTH, uint64_t) + +#define IAVF_RX_DESC_EXT_STATUS_FLEXBH_MASK 0x03 +#define IAVF_RX_DESC_EXT_STATUS_FLEXBH_FD_ID 0x01 + +struct iavf_adapter; +struct iavf_rx_queue; +struct iavf_tx_queue; + +/* Structure that defines a VSI, associated with a adapter. */ +struct iavf_vsi { + struct iavf_adapter *adapter; /* Backreference to associated adapter */ + uint16_t vsi_id; + uint16_t nb_qps; /* Number of queue pairs VSI can occupy */ + uint16_t nb_used_qps; /* Number of queue pairs VSI uses */ + uint16_t max_macaddrs; /* Maximum number of MAC addresses */ + uint16_t base_vector; + uint16_t msix_intr; /* The MSIX interrupt binds to VSI */ + struct virtchnl_eth_stats eth_stats_offset; +}; + +struct rte_flow; +TAILQ_HEAD(iavf_flow_list, rte_flow); + +struct iavf_flow_parser_node; +TAILQ_HEAD(iavf_parser_list, iavf_flow_parser_node); + +struct iavf_fdir_conf { + struct virtchnl_fdir_add add_fltr; + struct virtchnl_fdir_del del_fltr; + uint64_t input_set; + uint32_t flow_id; + uint32_t mark_flag; +}; + +struct iavf_fdir_info { + struct iavf_fdir_conf conf; +}; + +/* TODO: is that correct to assume the max number to be 16 ?*/ +#define IAVF_MAX_MSIX_VECTORS 16 + +/* Structure to store private data specific for VF instance. */ +struct iavf_info { + uint16_t num_queue_pairs; + uint16_t max_pkt_len; /* Maximum packet length */ + uint16_t mac_num; /* Number of MAC addresses */ + bool promisc_unicast_enabled; + bool promisc_multicast_enabled; + + struct virtchnl_version_info virtchnl_version; + struct virtchnl_vf_resource *vf_res; /* VF resource */ + struct virtchnl_vsi_resource *vsi_res; /* LAN VSI */ + uint64_t supported_rxdid; + + volatile enum virtchnl_ops pend_cmd; /* pending command not finished */ + uint32_t cmd_retval; /* return value of the cmd response from PF */ + uint8_t *aq_resp; /* buffer to store the adminq response from PF */ + + /* Event from pf */ + bool dev_closed; + bool link_up; + uint32_t link_speed; + + struct iavf_vsi vsi; + bool vf_reset; + uint64_t flags; + + uint8_t *rss_lut; + uint8_t *rss_key; + uint16_t nb_msix; /* number of MSI-X interrupts on Rx */ + uint16_t msix_base; /* msix vector base from */ + /* queue bitmask for each vector */ + uint16_t rxq_map[IAVF_MAX_MSIX_VECTORS]; + struct iavf_flow_list flow_list; + rte_spinlock_t flow_ops_lock; + struct iavf_parser_list rss_parser_list; + struct iavf_parser_list dist_parser_list; + + struct iavf_fdir_info fdir; /* flow director info */ +}; + +#define IAVF_MAX_PKT_TYPE 1024 + +/* Structure to store private data for each VF instance. */ +struct iavf_adapter { + struct iavf_hw hw; + struct rte_eth_dev *eth_dev; + struct iavf_info vf; + + bool rx_bulk_alloc_allowed; + /* For vector PMD */ + bool rx_vec_allowed; + bool tx_vec_allowed; + const uint32_t *ptype_tbl; + bool stopped; + uint16_t fdir_ref_cnt; +}; + +/* IAVF_DEV_PRIVATE_TO */ +#define IAVF_DEV_PRIVATE_TO_ADAPTER(adapter) \ + ((struct iavf_adapter *)adapter) +#define IAVF_DEV_PRIVATE_TO_VF(adapter) \ + (&((struct iavf_adapter *)adapter)->vf) +#define IAVF_DEV_PRIVATE_TO_HW(adapter) \ + (&((struct iavf_adapter *)adapter)->hw) + +/* IAVF_VSI_TO */ +#define IAVF_VSI_TO_HW(vsi) \ + (&(((struct iavf_vsi *)vsi)->adapter->hw)) +#define IAVF_VSI_TO_VF(vsi) \ + (&(((struct iavf_vsi *)vsi)->adapter->vf)) +#define IAVF_VSI_TO_ETH_DEV(vsi) \ + (((struct iavf_vsi *)vsi)->adapter->eth_dev) + +static inline void +iavf_init_adminq_parameter(struct iavf_hw *hw) +{ + hw->aq.num_arq_entries = IAVF_AQ_LEN; + hw->aq.num_asq_entries = IAVF_AQ_LEN; + hw->aq.arq_buf_size = IAVF_AQ_BUF_SZ; + hw->aq.asq_buf_size = IAVF_AQ_BUF_SZ; +} + +static inline uint16_t +iavf_calc_itr_interval(int16_t interval) +{ + if (interval < 0 || interval > IAVF_QUEUE_ITR_INTERVAL_MAX) + interval = IAVF_QUEUE_ITR_INTERVAL_DEFAULT; + + /* Convert to hardware count, as writing each 1 represents 2 us */ + return interval / 2; +} + +/* structure used for sending and checking response of virtchnl ops */ +struct iavf_cmd_info { + enum virtchnl_ops ops; + uint8_t *in_args; /* buffer for sending */ + uint32_t in_args_size; /* buffer size for sending */ + uint8_t *out_buffer; /* buffer for response */ + uint32_t out_size; /* buffer size for response */ +}; + +/* notify current command done. Only call in case execute + * _atomic_set_cmd successfully. + */ +static inline void +_notify_cmd(struct iavf_info *vf, uint32_t msg_ret) +{ + vf->cmd_retval = msg_ret; + rte_wmb(); + vf->pend_cmd = VIRTCHNL_OP_UNKNOWN; +} + +/* clear current command. Only call in case execute + * _atomic_set_cmd successfully. + */ +static inline void +_clear_cmd(struct iavf_info *vf) +{ + rte_wmb(); + vf->pend_cmd = VIRTCHNL_OP_UNKNOWN; + vf->cmd_retval = VIRTCHNL_STATUS_SUCCESS; +} + +/* Check there is pending cmd in execution. If none, set new command. */ +static inline int +_atomic_set_cmd(struct iavf_info *vf, enum virtchnl_ops ops) +{ + int ret = rte_atomic32_cmpset(&vf->pend_cmd, VIRTCHNL_OP_UNKNOWN, ops); + + if (!ret) + PMD_DRV_LOG(ERR, "There is incomplete cmd %d", vf->pend_cmd); + + return !ret; +} + +int iavf_check_api_version(struct iavf_adapter *adapter); +int iavf_get_vf_resource(struct iavf_adapter *adapter); +void iavf_handle_virtchnl_msg(struct rte_eth_dev *dev); +int iavf_enable_vlan_strip(struct iavf_adapter *adapter); +int iavf_disable_vlan_strip(struct iavf_adapter *adapter); +int iavf_switch_queue(struct iavf_adapter *adapter, uint16_t qid, + bool rx, bool on); +int iavf_enable_queues(struct iavf_adapter *adapter); +int iavf_disable_queues(struct iavf_adapter *adapter); +int iavf_configure_rss_lut(struct iavf_adapter *adapter); +int iavf_configure_rss_key(struct iavf_adapter *adapter); +int iavf_configure_queues(struct iavf_adapter *adapter); +int iavf_get_supported_rxdid(struct iavf_adapter *adapter); +int iavf_config_irq_map(struct iavf_adapter *adapter); +void iavf_add_del_all_mac_addr(struct iavf_adapter *adapter, bool add); +int iavf_dev_link_update(struct rte_eth_dev *dev, + __rte_unused int wait_to_complete); +int iavf_query_stats(struct iavf_adapter *adapter, + struct virtchnl_eth_stats **pstats); +int iavf_config_promisc(struct iavf_adapter *adapter, bool enable_unicast, + bool enable_multicast); +int iavf_add_del_eth_addr(struct iavf_adapter *adapter, + struct rte_ether_addr *addr, bool add); +int iavf_add_del_vlan(struct iavf_adapter *adapter, uint16_t vlanid, bool add); +int iavf_fdir_add(struct iavf_adapter *adapter, struct iavf_fdir_conf *filter); +int iavf_fdir_del(struct iavf_adapter *adapter, struct iavf_fdir_conf *filter); +int iavf_fdir_check(struct iavf_adapter *adapter, + struct iavf_fdir_conf *filter); +int iavf_add_del_rss_cfg(struct iavf_adapter *adapter, + struct virtchnl_rss_cfg *rss_cfg, bool add); +#endif /* _IAVF_ETHDEV_H_ */ diff --git a/src/spdk/dpdk/drivers/net/iavf/iavf_ethdev.c b/src/spdk/dpdk/drivers/net/iavf/iavf_ethdev.c new file mode 100644 index 000000000..e09efffd1 --- /dev/null +++ b/src/spdk/dpdk/drivers/net/iavf/iavf_ethdev.c @@ -0,0 +1,1586 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017 Intel Corporation + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "iavf.h" +#include "iavf_rxtx.h" +#include "iavf_generic_flow.h" + +static int iavf_dev_configure(struct rte_eth_dev *dev); +static int iavf_dev_start(struct rte_eth_dev *dev); +static void iavf_dev_stop(struct rte_eth_dev *dev); +static void iavf_dev_close(struct rte_eth_dev *dev); +static int iavf_dev_reset(struct rte_eth_dev *dev); +static int iavf_dev_info_get(struct rte_eth_dev *dev, + struct rte_eth_dev_info *dev_info); +static const uint32_t *iavf_dev_supported_ptypes_get(struct rte_eth_dev *dev); +static int iavf_dev_stats_get(struct rte_eth_dev *dev, + struct rte_eth_stats *stats); +static int iavf_dev_stats_reset(struct rte_eth_dev *dev); +static int iavf_dev_promiscuous_enable(struct rte_eth_dev *dev); +static int iavf_dev_promiscuous_disable(struct rte_eth_dev *dev); +static int iavf_dev_allmulticast_enable(struct rte_eth_dev *dev); +static int iavf_dev_allmulticast_disable(struct rte_eth_dev *dev); +static int iavf_dev_add_mac_addr(struct rte_eth_dev *dev, + struct rte_ether_addr *addr, + uint32_t index, + uint32_t pool); +static void iavf_dev_del_mac_addr(struct rte_eth_dev *dev, uint32_t index); +static int iavf_dev_vlan_filter_set(struct rte_eth_dev *dev, + uint16_t vlan_id, int on); +static int iavf_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask); +static int iavf_dev_rss_reta_update(struct rte_eth_dev *dev, + struct rte_eth_rss_reta_entry64 *reta_conf, + uint16_t reta_size); +static int iavf_dev_rss_reta_query(struct rte_eth_dev *dev, + struct rte_eth_rss_reta_entry64 *reta_conf, + uint16_t reta_size); +static int iavf_dev_rss_hash_update(struct rte_eth_dev *dev, + struct rte_eth_rss_conf *rss_conf); +static int iavf_dev_rss_hash_conf_get(struct rte_eth_dev *dev, + struct rte_eth_rss_conf *rss_conf); +static int iavf_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); +static int iavf_dev_set_default_mac_addr(struct rte_eth_dev *dev, + struct rte_ether_addr *mac_addr); +static int iavf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, + uint16_t queue_id); +static int iavf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, + uint16_t queue_id); +static int iavf_dev_filter_ctrl(struct rte_eth_dev *dev, + enum rte_filter_type filter_type, + enum rte_filter_op filter_op, + void *arg); + + +int iavf_logtype_init; +int iavf_logtype_driver; + +#ifdef RTE_LIBRTE_IAVF_DEBUG_RX +int iavf_logtype_rx; +#endif +#ifdef RTE_LIBRTE_IAVF_DEBUG_TX +int iavf_logtype_tx; +#endif +#ifdef RTE_LIBRTE_IAVF_DEBUG_TX_FREE +int iavf_logtype_tx_free; +#endif + +static const struct rte_pci_id pci_id_iavf_map[] = { + { RTE_PCI_DEVICE(IAVF_INTEL_VENDOR_ID, IAVF_DEV_ID_ADAPTIVE_VF) }, + { .vendor_id = 0, /* sentinel */ }, +}; + +static const struct eth_dev_ops iavf_eth_dev_ops = { + .dev_configure = iavf_dev_configure, + .dev_start = iavf_dev_start, + .dev_stop = iavf_dev_stop, + .dev_close = iavf_dev_close, + .dev_reset = iavf_dev_reset, + .dev_infos_get = iavf_dev_info_get, + .dev_supported_ptypes_get = iavf_dev_supported_ptypes_get, + .link_update = iavf_dev_link_update, + .stats_get = iavf_dev_stats_get, + .stats_reset = iavf_dev_stats_reset, + .promiscuous_enable = iavf_dev_promiscuous_enable, + .promiscuous_disable = iavf_dev_promiscuous_disable, + .allmulticast_enable = iavf_dev_allmulticast_enable, + .allmulticast_disable = iavf_dev_allmulticast_disable, + .mac_addr_add = iavf_dev_add_mac_addr, + .mac_addr_remove = iavf_dev_del_mac_addr, + .vlan_filter_set = iavf_dev_vlan_filter_set, + .vlan_offload_set = iavf_dev_vlan_offload_set, + .rx_queue_start = iavf_dev_rx_queue_start, + .rx_queue_stop = iavf_dev_rx_queue_stop, + .tx_queue_start = iavf_dev_tx_queue_start, + .tx_queue_stop = iavf_dev_tx_queue_stop, + .rx_queue_setup = iavf_dev_rx_queue_setup, + .rx_queue_release = iavf_dev_rx_queue_release, + .tx_queue_setup = iavf_dev_tx_queue_setup, + .tx_queue_release = iavf_dev_tx_queue_release, + .mac_addr_set = iavf_dev_set_default_mac_addr, + .reta_update = iavf_dev_rss_reta_update, + .reta_query = iavf_dev_rss_reta_query, + .rss_hash_update = iavf_dev_rss_hash_update, + .rss_hash_conf_get = iavf_dev_rss_hash_conf_get, + .rxq_info_get = iavf_dev_rxq_info_get, + .txq_info_get = iavf_dev_txq_info_get, + .rx_queue_count = iavf_dev_rxq_count, + .rx_descriptor_status = iavf_dev_rx_desc_status, + .tx_descriptor_status = iavf_dev_tx_desc_status, + .mtu_set = iavf_dev_mtu_set, + .rx_queue_intr_enable = iavf_dev_rx_queue_intr_enable, + .rx_queue_intr_disable = iavf_dev_rx_queue_intr_disable, + .filter_ctrl = iavf_dev_filter_ctrl, +}; + +static int +iavf_dev_configure(struct rte_eth_dev *dev) +{ + struct iavf_adapter *ad = + IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad); + struct rte_eth_conf *dev_conf = &dev->data->dev_conf; + + ad->rx_bulk_alloc_allowed = true; + /* Initialize to TRUE. If any of Rx queues doesn't meet the + * vector Rx/Tx preconditions, it will be reset. + */ + ad->rx_vec_allowed = true; + ad->tx_vec_allowed = true; + + if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) + dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_RSS_HASH; + + /* Vlan stripping setting */ + if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN) { + if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_VLAN_STRIP) + iavf_enable_vlan_strip(ad); + else + iavf_disable_vlan_strip(ad); + } + return 0; +} + +static int +iavf_init_rss(struct iavf_adapter *adapter) +{ + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + struct rte_eth_rss_conf *rss_conf; + uint8_t i, j, nb_q; + int ret; + + rss_conf = &adapter->eth_dev->data->dev_conf.rx_adv_conf.rss_conf; + nb_q = RTE_MIN(adapter->eth_dev->data->nb_rx_queues, + IAVF_MAX_NUM_QUEUES); + + if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) { + PMD_DRV_LOG(DEBUG, "RSS is not supported"); + return -ENOTSUP; + } + if (adapter->eth_dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) { + PMD_DRV_LOG(WARNING, "RSS is enabled by PF by default"); + /* set all lut items to default queue */ + for (i = 0; i < vf->vf_res->rss_lut_size; i++) + vf->rss_lut[i] = 0; + ret = iavf_configure_rss_lut(adapter); + return ret; + } + + /* In IAVF, RSS enablement is set by PF driver. It is not supported + * to set based on rss_conf->rss_hf. + */ + + /* configure RSS key */ + if (!rss_conf->rss_key) { + /* Calculate the default hash key */ + for (i = 0; i <= vf->vf_res->rss_key_size; i++) + vf->rss_key[i] = (uint8_t)rte_rand(); + } else + rte_memcpy(vf->rss_key, rss_conf->rss_key, + RTE_MIN(rss_conf->rss_key_len, + vf->vf_res->rss_key_size)); + + /* init RSS LUT table */ + for (i = 0, j = 0; i < vf->vf_res->rss_lut_size; i++, j++) { + if (j >= nb_q) + j = 0; + vf->rss_lut[i] = j; + } + /* send virtchnnl ops to configure rss*/ + ret = iavf_configure_rss_lut(adapter); + if (ret) + return ret; + ret = iavf_configure_rss_key(adapter); + if (ret) + return ret; + + return 0; +} + +static int +iavf_init_rxq(struct rte_eth_dev *dev, struct iavf_rx_queue *rxq) +{ + struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct rte_eth_dev_data *dev_data = dev->data; + uint16_t buf_size, max_pkt_len, len; + + buf_size = rte_pktmbuf_data_room_size(rxq->mp) - RTE_PKTMBUF_HEADROOM; + + /* Calculate the maximum packet length allowed */ + len = rxq->rx_buf_len * IAVF_MAX_CHAINED_RX_BUFFERS; + max_pkt_len = RTE_MIN(len, dev->data->dev_conf.rxmode.max_rx_pkt_len); + + /* Check if the jumbo frame and maximum packet length are set + * correctly. + */ + if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) { + if (max_pkt_len <= RTE_ETHER_MAX_LEN || + max_pkt_len > IAVF_FRAME_SIZE_MAX) { + PMD_DRV_LOG(ERR, "maximum packet length must be " + "larger than %u and smaller than %u, " + "as jumbo frame is enabled", + (uint32_t)RTE_ETHER_MAX_LEN, + (uint32_t)IAVF_FRAME_SIZE_MAX); + return -EINVAL; + } + } else { + if (max_pkt_len < RTE_ETHER_MIN_LEN || + max_pkt_len > RTE_ETHER_MAX_LEN) { + PMD_DRV_LOG(ERR, "maximum packet length must be " + "larger than %u and smaller than %u, " + "as jumbo frame is disabled", + (uint32_t)RTE_ETHER_MIN_LEN, + (uint32_t)RTE_ETHER_MAX_LEN); + return -EINVAL; + } + } + + rxq->max_pkt_len = max_pkt_len; + if ((dev_data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_SCATTER) || + (rxq->max_pkt_len + 2 * IAVF_VLAN_TAG_SIZE) > buf_size) { + dev_data->scattered_rx = 1; + } + IAVF_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1); + IAVF_WRITE_FLUSH(hw); + + return 0; +} + +static int +iavf_init_queues(struct rte_eth_dev *dev) +{ + struct iavf_rx_queue **rxq = + (struct iavf_rx_queue **)dev->data->rx_queues; + int i, ret = IAVF_SUCCESS; + + for (i = 0; i < dev->data->nb_rx_queues; i++) { + if (!rxq[i] || !rxq[i]->q_set) + continue; + ret = iavf_init_rxq(dev, rxq[i]); + if (ret != IAVF_SUCCESS) + break; + } + /* set rx/tx function to vector/scatter/single-segment + * according to parameters + */ + iavf_set_rx_function(dev); + iavf_set_tx_function(dev); + + return ret; +} + +static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev, + struct rte_intr_handle *intr_handle) +{ + struct iavf_adapter *adapter = + IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter); + uint16_t interval, i; + int vec; + + if (rte_intr_cap_multiple(intr_handle) && + dev->data->dev_conf.intr_conf.rxq) { + if (rte_intr_efd_enable(intr_handle, dev->data->nb_rx_queues)) + return -1; + } + + if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { + intr_handle->intr_vec = + rte_zmalloc("intr_vec", + dev->data->nb_rx_queues * sizeof(int), 0); + if (!intr_handle->intr_vec) { + PMD_DRV_LOG(ERR, "Failed to allocate %d rx intr_vec", + dev->data->nb_rx_queues); + return -1; + } + } + + if (!dev->data->dev_conf.intr_conf.rxq || + !rte_intr_dp_is_en(intr_handle)) { + /* Rx interrupt disabled, Map interrupt only for writeback */ + vf->nb_msix = 1; + if (vf->vf_res->vf_cap_flags & + VIRTCHNL_VF_OFFLOAD_WB_ON_ITR) { + /* If WB_ON_ITR supports, enable it */ + vf->msix_base = IAVF_RX_VEC_START; + IAVF_WRITE_REG(hw, + IAVF_VFINT_DYN_CTLN1(vf->msix_base - 1), + IAVF_VFINT_DYN_CTLN1_ITR_INDX_MASK | + IAVF_VFINT_DYN_CTLN1_WB_ON_ITR_MASK); + } else { + /* If no WB_ON_ITR offload flags, need to set + * interrupt for descriptor write back. + */ + vf->msix_base = IAVF_MISC_VEC_ID; + + /* set ITR to max */ + interval = iavf_calc_itr_interval( + IAVF_QUEUE_ITR_INTERVAL_MAX); + IAVF_WRITE_REG(hw, IAVF_VFINT_DYN_CTL01, + IAVF_VFINT_DYN_CTL01_INTENA_MASK | + (IAVF_ITR_INDEX_DEFAULT << + IAVF_VFINT_DYN_CTL01_ITR_INDX_SHIFT) | + (interval << + IAVF_VFINT_DYN_CTL01_INTERVAL_SHIFT)); + } + IAVF_WRITE_FLUSH(hw); + /* map all queues to the same interrupt */ + for (i = 0; i < dev->data->nb_rx_queues; i++) + vf->rxq_map[vf->msix_base] |= 1 << i; + } else { + if (!rte_intr_allow_others(intr_handle)) { + vf->nb_msix = 1; + vf->msix_base = IAVF_MISC_VEC_ID; + for (i = 0; i < dev->data->nb_rx_queues; i++) { + vf->rxq_map[vf->msix_base] |= 1 << i; + intr_handle->intr_vec[i] = IAVF_MISC_VEC_ID; + } + PMD_DRV_LOG(DEBUG, + "vector %u are mapping to all Rx queues", + vf->msix_base); + } else { + /* If Rx interrupt is reuquired, and we can use + * multi interrupts, then the vec is from 1 + */ + vf->nb_msix = RTE_MIN(vf->vf_res->max_vectors, + intr_handle->nb_efd); + vf->msix_base = IAVF_RX_VEC_START; + vec = IAVF_RX_VEC_START; + for (i = 0; i < dev->data->nb_rx_queues; i++) { + vf->rxq_map[vec] |= 1 << i; + intr_handle->intr_vec[i] = vec++; + if (vec >= vf->nb_msix) + vec = IAVF_RX_VEC_START; + } + PMD_DRV_LOG(DEBUG, + "%u vectors are mapping to %u Rx queues", + vf->nb_msix, dev->data->nb_rx_queues); + } + } + + if (iavf_config_irq_map(adapter)) { + PMD_DRV_LOG(ERR, "config interrupt mapping failed"); + return -1; + } + return 0; +} + +static int +iavf_start_queues(struct rte_eth_dev *dev) +{ + struct iavf_rx_queue *rxq; + struct iavf_tx_queue *txq; + int i; + + for (i = 0; i < dev->data->nb_tx_queues; i++) { + txq = dev->data->tx_queues[i]; + if (txq->tx_deferred_start) + continue; + if (iavf_dev_tx_queue_start(dev, i) != 0) { + PMD_DRV_LOG(ERR, "Fail to start queue %u", i); + return -1; + } + } + + for (i = 0; i < dev->data->nb_rx_queues; i++) { + rxq = dev->data->rx_queues[i]; + if (rxq->rx_deferred_start) + continue; + if (iavf_dev_rx_queue_start(dev, i) != 0) { + PMD_DRV_LOG(ERR, "Fail to start queue %u", i); + return -1; + } + } + + return 0; +} + +static int +iavf_dev_start(struct rte_eth_dev *dev) +{ + struct iavf_adapter *adapter = + IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); + struct rte_intr_handle *intr_handle = dev->intr_handle; + + PMD_INIT_FUNC_TRACE(); + + adapter->stopped = 0; + + vf->max_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len; + vf->num_queue_pairs = RTE_MAX(dev->data->nb_rx_queues, + dev->data->nb_tx_queues); + + if (iavf_init_queues(dev) != 0) { + PMD_DRV_LOG(ERR, "failed to do Queue init"); + return -1; + } + + if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) { + if (iavf_init_rss(adapter) != 0) { + PMD_DRV_LOG(ERR, "configure rss failed"); + goto err_rss; + } + } + + if (iavf_configure_queues(adapter) != 0) { + PMD_DRV_LOG(ERR, "configure queues failed"); + goto err_queue; + } + + if (iavf_config_rx_queues_irqs(dev, intr_handle) != 0) { + PMD_DRV_LOG(ERR, "configure irq failed"); + goto err_queue; + } + /* re-enable intr again, because efd assign may change */ + if (dev->data->dev_conf.intr_conf.rxq != 0) { + rte_intr_disable(intr_handle); + rte_intr_enable(intr_handle); + } + + /* Set all mac addrs */ + iavf_add_del_all_mac_addr(adapter, true); + + if (iavf_start_queues(dev) != 0) { + PMD_DRV_LOG(ERR, "enable queues failed"); + goto err_mac; + } + + return 0; + +err_mac: + iavf_add_del_all_mac_addr(adapter, false); +err_queue: +err_rss: + return -1; +} + +static void +iavf_dev_stop(struct rte_eth_dev *dev) +{ + struct iavf_adapter *adapter = + IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct rte_intr_handle *intr_handle = dev->intr_handle; + + PMD_INIT_FUNC_TRACE(); + + if (adapter->stopped == 1) + return; + + iavf_stop_queues(dev); + + /* Disable the interrupt for Rx */ + rte_intr_efd_disable(intr_handle); + /* Rx interrupt vector mapping free */ + if (intr_handle->intr_vec) { + rte_free(intr_handle->intr_vec); + intr_handle->intr_vec = NULL; + } + + /* remove all mac addrs */ + iavf_add_del_all_mac_addr(adapter, false); + adapter->stopped = 1; +} + +static int +iavf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) +{ + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); + + dev_info->max_rx_queues = vf->vsi_res->num_queue_pairs; + dev_info->max_tx_queues = vf->vsi_res->num_queue_pairs; + dev_info->min_rx_bufsize = IAVF_BUF_SIZE_MIN; + dev_info->max_rx_pktlen = IAVF_FRAME_SIZE_MAX; + dev_info->hash_key_size = vf->vf_res->rss_key_size; + dev_info->reta_size = vf->vf_res->rss_lut_size; + dev_info->flow_type_rss_offloads = IAVF_RSS_OFFLOAD_ALL; + dev_info->max_mac_addrs = IAVF_NUM_MACADDR_MAX; + dev_info->rx_offload_capa = + DEV_RX_OFFLOAD_VLAN_STRIP | + DEV_RX_OFFLOAD_QINQ_STRIP | + DEV_RX_OFFLOAD_IPV4_CKSUM | + DEV_RX_OFFLOAD_UDP_CKSUM | + DEV_RX_OFFLOAD_TCP_CKSUM | + DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM | + DEV_RX_OFFLOAD_SCATTER | + DEV_RX_OFFLOAD_JUMBO_FRAME | + DEV_RX_OFFLOAD_VLAN_FILTER | + DEV_RX_OFFLOAD_RSS_HASH; + dev_info->tx_offload_capa = + DEV_TX_OFFLOAD_VLAN_INSERT | + DEV_TX_OFFLOAD_QINQ_INSERT | + DEV_TX_OFFLOAD_IPV4_CKSUM | + DEV_TX_OFFLOAD_UDP_CKSUM | + DEV_TX_OFFLOAD_TCP_CKSUM | + DEV_TX_OFFLOAD_SCTP_CKSUM | + DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM | + DEV_TX_OFFLOAD_TCP_TSO | + DEV_TX_OFFLOAD_VXLAN_TNL_TSO | + DEV_TX_OFFLOAD_GRE_TNL_TSO | + DEV_TX_OFFLOAD_IPIP_TNL_TSO | + DEV_TX_OFFLOAD_GENEVE_TNL_TSO | + DEV_TX_OFFLOAD_MULTI_SEGS; + + dev_info->default_rxconf = (struct rte_eth_rxconf) { + .rx_free_thresh = IAVF_DEFAULT_RX_FREE_THRESH, + .rx_drop_en = 0, + .offloads = 0, + }; + + dev_info->default_txconf = (struct rte_eth_txconf) { + .tx_free_thresh = IAVF_DEFAULT_TX_FREE_THRESH, + .tx_rs_thresh = IAVF_DEFAULT_TX_RS_THRESH, + .offloads = 0, + }; + + dev_info->rx_desc_lim = (struct rte_eth_desc_lim) { + .nb_max = IAVF_MAX_RING_DESC, + .nb_min = IAVF_MIN_RING_DESC, + .nb_align = IAVF_ALIGN_RING_DESC, + }; + + dev_info->tx_desc_lim = (struct rte_eth_desc_lim) { + .nb_max = IAVF_MAX_RING_DESC, + .nb_min = IAVF_MIN_RING_DESC, + .nb_align = IAVF_ALIGN_RING_DESC, + }; + + return 0; +} + +static const uint32_t * +iavf_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused) +{ + static const uint32_t ptypes[] = { + RTE_PTYPE_L2_ETHER, + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, + RTE_PTYPE_L4_FRAG, + RTE_PTYPE_L4_ICMP, + RTE_PTYPE_L4_NONFRAG, + RTE_PTYPE_L4_SCTP, + RTE_PTYPE_L4_TCP, + RTE_PTYPE_L4_UDP, + RTE_PTYPE_UNKNOWN + }; + return ptypes; +} + +int +iavf_dev_link_update(struct rte_eth_dev *dev, + __rte_unused int wait_to_complete) +{ + struct rte_eth_link new_link; + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); + + /* Only read status info stored in VF, and the info is updated + * when receive LINK_CHANGE evnet from PF by Virtchnnl. + */ + switch (vf->link_speed) { + case 10: + new_link.link_speed = ETH_SPEED_NUM_10M; + break; + case 100: + new_link.link_speed = ETH_SPEED_NUM_100M; + break; + case 1000: + new_link.link_speed = ETH_SPEED_NUM_1G; + break; + case 10000: + new_link.link_speed = ETH_SPEED_NUM_10G; + break; + case 20000: + new_link.link_speed = ETH_SPEED_NUM_20G; + break; + case 25000: + new_link.link_speed = ETH_SPEED_NUM_25G; + break; + case 40000: + new_link.link_speed = ETH_SPEED_NUM_40G; + break; + case 50000: + new_link.link_speed = ETH_SPEED_NUM_50G; + break; + case 100000: + new_link.link_speed = ETH_SPEED_NUM_100G; + break; + default: + new_link.link_speed = ETH_SPEED_NUM_NONE; + break; + } + + new_link.link_duplex = ETH_LINK_FULL_DUPLEX; + new_link.link_status = vf->link_up ? ETH_LINK_UP : + ETH_LINK_DOWN; + new_link.link_autoneg = !(dev->data->dev_conf.link_speeds & + ETH_LINK_SPEED_FIXED); + + if (rte_atomic64_cmpset((uint64_t *)&dev->data->dev_link, + *(uint64_t *)&dev->data->dev_link, + *(uint64_t *)&new_link) == 0) + return -1; + + return 0; +} + +static int +iavf_dev_promiscuous_enable(struct rte_eth_dev *dev) +{ + struct iavf_adapter *adapter = + IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + int ret; + + if (vf->promisc_unicast_enabled) + return 0; + + ret = iavf_config_promisc(adapter, true, vf->promisc_multicast_enabled); + if (!ret) + vf->promisc_unicast_enabled = true; + else + ret = -EAGAIN; + + return ret; +} + +static int +iavf_dev_promiscuous_disable(struct rte_eth_dev *dev) +{ + struct iavf_adapter *adapter = + IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + int ret; + + if (!vf->promisc_unicast_enabled) + return 0; + + ret = iavf_config_promisc(adapter, false, + vf->promisc_multicast_enabled); + if (!ret) + vf->promisc_unicast_enabled = false; + else + ret = -EAGAIN; + + return ret; +} + +static int +iavf_dev_allmulticast_enable(struct rte_eth_dev *dev) +{ + struct iavf_adapter *adapter = + IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + int ret; + + if (vf->promisc_multicast_enabled) + return 0; + + ret = iavf_config_promisc(adapter, vf->promisc_unicast_enabled, true); + if (!ret) + vf->promisc_multicast_enabled = true; + else + ret = -EAGAIN; + + return ret; +} + +static int +iavf_dev_allmulticast_disable(struct rte_eth_dev *dev) +{ + struct iavf_adapter *adapter = + IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + int ret; + + if (!vf->promisc_multicast_enabled) + return 0; + + ret = iavf_config_promisc(adapter, vf->promisc_unicast_enabled, false); + if (!ret) + vf->promisc_multicast_enabled = false; + else + ret = -EAGAIN; + + return ret; +} + +static int +iavf_dev_add_mac_addr(struct rte_eth_dev *dev, struct rte_ether_addr *addr, + __rte_unused uint32_t index, + __rte_unused uint32_t pool) +{ + struct iavf_adapter *adapter = + IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + int err; + + if (rte_is_zero_ether_addr(addr)) { + PMD_DRV_LOG(ERR, "Invalid Ethernet Address"); + return -EINVAL; + } + + err = iavf_add_del_eth_addr(adapter, addr, true); + if (err) { + PMD_DRV_LOG(ERR, "fail to add MAC address"); + return -EIO; + } + + vf->mac_num++; + + return 0; +} + +static void +iavf_dev_del_mac_addr(struct rte_eth_dev *dev, uint32_t index) +{ + struct iavf_adapter *adapter = + IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + struct rte_ether_addr *addr; + int err; + + addr = &dev->data->mac_addrs[index]; + + err = iavf_add_del_eth_addr(adapter, addr, false); + if (err) + PMD_DRV_LOG(ERR, "fail to delete MAC address"); + + vf->mac_num--; +} + +static int +iavf_dev_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) +{ + struct iavf_adapter *adapter = + IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + int err; + + if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN)) + return -ENOTSUP; + + err = iavf_add_del_vlan(adapter, vlan_id, on); + if (err) + return -EIO; + return 0; +} + +static int +iavf_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask) +{ + struct iavf_adapter *adapter = + IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + struct rte_eth_conf *dev_conf = &dev->data->dev_conf; + int err; + + if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN)) + return -ENOTSUP; + + /* Vlan stripping setting */ + if (mask & ETH_VLAN_STRIP_MASK) { + /* Enable or disable VLAN stripping */ + if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_VLAN_STRIP) + err = iavf_enable_vlan_strip(adapter); + else + err = iavf_disable_vlan_strip(adapter); + + if (err) + return -EIO; + } + return 0; +} + +static int +iavf_dev_rss_reta_update(struct rte_eth_dev *dev, + struct rte_eth_rss_reta_entry64 *reta_conf, + uint16_t reta_size) +{ + struct iavf_adapter *adapter = + IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + uint8_t *lut; + uint16_t i, idx, shift; + int ret; + + if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) + return -ENOTSUP; + + if (reta_size != vf->vf_res->rss_lut_size) { + PMD_DRV_LOG(ERR, "The size of hash lookup table configured " + "(%d) doesn't match the number of hardware can " + "support (%d)", reta_size, vf->vf_res->rss_lut_size); + return -EINVAL; + } + + lut = rte_zmalloc("rss_lut", reta_size, 0); + if (!lut) { + PMD_DRV_LOG(ERR, "No memory can be allocated"); + return -ENOMEM; + } + /* store the old lut table temporarily */ + rte_memcpy(lut, vf->rss_lut, reta_size); + + for (i = 0; i < reta_size; i++) { + idx = i / RTE_RETA_GROUP_SIZE; + shift = i % RTE_RETA_GROUP_SIZE; + if (reta_conf[idx].mask & (1ULL << shift)) + lut[i] = reta_conf[idx].reta[shift]; + } + + rte_memcpy(vf->rss_lut, lut, reta_size); + /* send virtchnnl ops to configure rss*/ + ret = iavf_configure_rss_lut(adapter); + if (ret) /* revert back */ + rte_memcpy(vf->rss_lut, lut, reta_size); + rte_free(lut); + + return ret; +} + +static int +iavf_dev_rss_reta_query(struct rte_eth_dev *dev, + struct rte_eth_rss_reta_entry64 *reta_conf, + uint16_t reta_size) +{ + struct iavf_adapter *adapter = + IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + uint16_t i, idx, shift; + + if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) + return -ENOTSUP; + + if (reta_size != vf->vf_res->rss_lut_size) { + PMD_DRV_LOG(ERR, "The size of hash lookup table configured " + "(%d) doesn't match the number of hardware can " + "support (%d)", reta_size, vf->vf_res->rss_lut_size); + return -EINVAL; + } + + for (i = 0; i < reta_size; i++) { + idx = i / RTE_RETA_GROUP_SIZE; + shift = i % RTE_RETA_GROUP_SIZE; + if (reta_conf[idx].mask & (1ULL << shift)) + reta_conf[idx].reta[shift] = vf->rss_lut[i]; + } + + return 0; +} + +static int +iavf_dev_rss_hash_update(struct rte_eth_dev *dev, + struct rte_eth_rss_conf *rss_conf) +{ + struct iavf_adapter *adapter = + IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + + if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) + return -ENOTSUP; + + /* HENA setting, it is enabled by default, no change */ + if (!rss_conf->rss_key || rss_conf->rss_key_len == 0) { + PMD_DRV_LOG(DEBUG, "No key to be configured"); + return 0; + } else if (rss_conf->rss_key_len != vf->vf_res->rss_key_size) { + PMD_DRV_LOG(ERR, "The size of hash key configured " + "(%d) doesn't match the size of hardware can " + "support (%d)", rss_conf->rss_key_len, + vf->vf_res->rss_key_size); + return -EINVAL; + } + + rte_memcpy(vf->rss_key, rss_conf->rss_key, rss_conf->rss_key_len); + + return iavf_configure_rss_key(adapter); +} + +static int +iavf_dev_rss_hash_conf_get(struct rte_eth_dev *dev, + struct rte_eth_rss_conf *rss_conf) +{ + struct iavf_adapter *adapter = + IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + + if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) + return -ENOTSUP; + + /* Just set it to default value now. */ + rss_conf->rss_hf = IAVF_RSS_OFFLOAD_ALL; + + if (!rss_conf->rss_key) + return 0; + + rss_conf->rss_key_len = vf->vf_res->rss_key_size; + rte_memcpy(rss_conf->rss_key, vf->rss_key, rss_conf->rss_key_len); + + return 0; +} + +static int +iavf_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) +{ + uint32_t frame_size = mtu + IAVF_ETH_OVERHEAD; + int ret = 0; + + if (mtu < RTE_ETHER_MIN_MTU || frame_size > IAVF_FRAME_SIZE_MAX) + return -EINVAL; + + /* mtu setting is forbidden if port is start */ + if (dev->data->dev_started) { + PMD_DRV_LOG(ERR, "port must be stopped before configuration"); + return -EBUSY; + } + + if (frame_size > RTE_ETHER_MAX_LEN) + dev->data->dev_conf.rxmode.offloads |= + DEV_RX_OFFLOAD_JUMBO_FRAME; + else + dev->data->dev_conf.rxmode.offloads &= + ~DEV_RX_OFFLOAD_JUMBO_FRAME; + + dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size; + + return ret; +} + +static int +iavf_dev_set_default_mac_addr(struct rte_eth_dev *dev, + struct rte_ether_addr *mac_addr) +{ + struct iavf_adapter *adapter = + IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter); + struct rte_ether_addr *perm_addr, *old_addr; + int ret; + + old_addr = (struct rte_ether_addr *)hw->mac.addr; + perm_addr = (struct rte_ether_addr *)hw->mac.perm_addr; + + if (rte_is_same_ether_addr(mac_addr, old_addr)) + return 0; + + /* If the MAC address is configured by host, skip the setting */ + if (rte_is_valid_assigned_ether_addr(perm_addr)) + return -EPERM; + + ret = iavf_add_del_eth_addr(adapter, old_addr, false); + if (ret) + PMD_DRV_LOG(ERR, "Fail to delete old MAC:" + " %02X:%02X:%02X:%02X:%02X:%02X", + old_addr->addr_bytes[0], + old_addr->addr_bytes[1], + old_addr->addr_bytes[2], + old_addr->addr_bytes[3], + old_addr->addr_bytes[4], + old_addr->addr_bytes[5]); + + ret = iavf_add_del_eth_addr(adapter, mac_addr, true); + if (ret) + PMD_DRV_LOG(ERR, "Fail to add new MAC:" + " %02X:%02X:%02X:%02X:%02X:%02X", + mac_addr->addr_bytes[0], + mac_addr->addr_bytes[1], + mac_addr->addr_bytes[2], + mac_addr->addr_bytes[3], + mac_addr->addr_bytes[4], + mac_addr->addr_bytes[5]); + + if (ret) + return -EIO; + + rte_ether_addr_copy(mac_addr, (struct rte_ether_addr *)hw->mac.addr); + return 0; +} + +static void +iavf_stat_update_48(uint64_t *offset, uint64_t *stat) +{ + if (*stat >= *offset) + *stat = *stat - *offset; + else + *stat = (uint64_t)((*stat + + ((uint64_t)1 << IAVF_48_BIT_WIDTH)) - *offset); + + *stat &= IAVF_48_BIT_MASK; +} + +static void +iavf_stat_update_32(uint64_t *offset, uint64_t *stat) +{ + if (*stat >= *offset) + *stat = (uint64_t)(*stat - *offset); + else + *stat = (uint64_t)((*stat + + ((uint64_t)1 << IAVF_32_BIT_WIDTH)) - *offset); +} + +static void +iavf_update_stats(struct iavf_vsi *vsi, struct virtchnl_eth_stats *nes) +{ + struct virtchnl_eth_stats *oes = &vsi->eth_stats_offset; + + iavf_stat_update_48(&oes->rx_bytes, &nes->rx_bytes); + iavf_stat_update_48(&oes->rx_unicast, &nes->rx_unicast); + iavf_stat_update_48(&oes->rx_multicast, &nes->rx_multicast); + iavf_stat_update_48(&oes->rx_broadcast, &nes->rx_broadcast); + iavf_stat_update_32(&oes->rx_discards, &nes->rx_discards); + iavf_stat_update_48(&oes->tx_bytes, &nes->tx_bytes); + iavf_stat_update_48(&oes->tx_unicast, &nes->tx_unicast); + iavf_stat_update_48(&oes->tx_multicast, &nes->tx_multicast); + iavf_stat_update_48(&oes->tx_broadcast, &nes->tx_broadcast); + iavf_stat_update_32(&oes->tx_errors, &nes->tx_errors); + iavf_stat_update_32(&oes->tx_discards, &nes->tx_discards); +} + +static int +iavf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) +{ + struct iavf_adapter *adapter = + IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); + struct iavf_vsi *vsi = &vf->vsi; + struct virtchnl_eth_stats *pstats = NULL; + int ret; + + ret = iavf_query_stats(adapter, &pstats); + if (ret == 0) { + iavf_update_stats(vsi, pstats); + stats->ipackets = pstats->rx_unicast + pstats->rx_multicast + + pstats->rx_broadcast - pstats->rx_discards; + stats->opackets = pstats->tx_broadcast + pstats->tx_multicast + + pstats->tx_unicast; + stats->imissed = pstats->rx_discards; + stats->oerrors = pstats->tx_errors + pstats->tx_discards; + stats->ibytes = pstats->rx_bytes; + stats->ibytes -= stats->ipackets * RTE_ETHER_CRC_LEN; + stats->obytes = pstats->tx_bytes; + } else { + PMD_DRV_LOG(ERR, "Get statistics failed"); + } + return ret; +} + +static int +iavf_dev_stats_reset(struct rte_eth_dev *dev) +{ + int ret; + struct iavf_adapter *adapter = + IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); + struct iavf_vsi *vsi = &vf->vsi; + struct virtchnl_eth_stats *pstats = NULL; + + /* read stat values to clear hardware registers */ + ret = iavf_query_stats(adapter, &pstats); + if (ret != 0) + return ret; + + /* set stats offset base on current values */ + vsi->eth_stats_offset = *pstats; + + return 0; +} + +static int +iavf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) +{ + struct iavf_adapter *adapter = + IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter); + uint16_t msix_intr; + + msix_intr = pci_dev->intr_handle.intr_vec[queue_id]; + if (msix_intr == IAVF_MISC_VEC_ID) { + PMD_DRV_LOG(INFO, "MISC is also enabled for control"); + IAVF_WRITE_REG(hw, IAVF_VFINT_DYN_CTL01, + IAVF_VFINT_DYN_CTL01_INTENA_MASK | + IAVF_VFINT_DYN_CTL01_CLEARPBA_MASK | + IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK); + } else { + IAVF_WRITE_REG(hw, + IAVF_VFINT_DYN_CTLN1 + (msix_intr - IAVF_RX_VEC_START), + IAVF_VFINT_DYN_CTLN1_INTENA_MASK | + IAVF_VFINT_DYN_CTL01_CLEARPBA_MASK | + IAVF_VFINT_DYN_CTLN1_ITR_INDX_MASK); + } + + IAVF_WRITE_FLUSH(hw); + + rte_intr_ack(&pci_dev->intr_handle); + + return 0; +} + +static int +iavf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) +{ + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); + uint16_t msix_intr; + + msix_intr = pci_dev->intr_handle.intr_vec[queue_id]; + if (msix_intr == IAVF_MISC_VEC_ID) { + PMD_DRV_LOG(ERR, "MISC is used for control, cannot disable it"); + return -EIO; + } + + IAVF_WRITE_REG(hw, + IAVF_VFINT_DYN_CTLN1(msix_intr - IAVF_RX_VEC_START), + 0); + + IAVF_WRITE_FLUSH(hw); + return 0; +} + +static int +iavf_check_vf_reset_done(struct iavf_hw *hw) +{ + int i, reset; + + for (i = 0; i < IAVF_RESET_WAIT_CNT; i++) { + reset = IAVF_READ_REG(hw, IAVF_VFGEN_RSTAT) & + IAVF_VFGEN_RSTAT_VFR_STATE_MASK; + reset = reset >> IAVF_VFGEN_RSTAT_VFR_STATE_SHIFT; + if (reset == VIRTCHNL_VFR_VFACTIVE || + reset == VIRTCHNL_VFR_COMPLETED) + break; + rte_delay_ms(20); + } + + if (i >= IAVF_RESET_WAIT_CNT) + return -1; + + return 0; +} + +static int +iavf_init_vf(struct rte_eth_dev *dev) +{ + int err, bufsz; + struct iavf_adapter *adapter = + IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); + + err = iavf_set_mac_type(hw); + if (err) { + PMD_INIT_LOG(ERR, "set_mac_type failed: %d", err); + goto err; + } + + err = iavf_check_vf_reset_done(hw); + if (err) { + PMD_INIT_LOG(ERR, "VF is still resetting"); + goto err; + } + + iavf_init_adminq_parameter(hw); + err = iavf_init_adminq(hw); + if (err) { + PMD_INIT_LOG(ERR, "init_adminq failed: %d", err); + goto err; + } + + vf->aq_resp = rte_zmalloc("vf_aq_resp", IAVF_AQ_BUF_SZ, 0); + if (!vf->aq_resp) { + PMD_INIT_LOG(ERR, "unable to allocate vf_aq_resp memory"); + goto err_aq; + } + if (iavf_check_api_version(adapter) != 0) { + PMD_INIT_LOG(ERR, "check_api version failed"); + goto err_api; + } + + bufsz = sizeof(struct virtchnl_vf_resource) + + (IAVF_MAX_VF_VSI * sizeof(struct virtchnl_vsi_resource)); + vf->vf_res = rte_zmalloc("vf_res", bufsz, 0); + if (!vf->vf_res) { + PMD_INIT_LOG(ERR, "unable to allocate vf_res memory"); + goto err_api; + } + if (iavf_get_vf_resource(adapter) != 0) { + PMD_INIT_LOG(ERR, "iavf_get_vf_config failed"); + goto err_alloc; + } + /* Allocate memort for RSS info */ + if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) { + vf->rss_key = rte_zmalloc("rss_key", + vf->vf_res->rss_key_size, 0); + if (!vf->rss_key) { + PMD_INIT_LOG(ERR, "unable to allocate rss_key memory"); + goto err_rss; + } + vf->rss_lut = rte_zmalloc("rss_lut", + vf->vf_res->rss_lut_size, 0); + if (!vf->rss_lut) { + PMD_INIT_LOG(ERR, "unable to allocate rss_lut memory"); + goto err_rss; + } + } + + if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) { + if (iavf_get_supported_rxdid(adapter) != 0) { + PMD_INIT_LOG(ERR, "failed to do get supported rxdid"); + goto err_rss; + } + } + + return 0; +err_rss: + rte_free(vf->rss_key); + rte_free(vf->rss_lut); +err_alloc: + rte_free(vf->vf_res); + vf->vsi_res = NULL; +err_api: + rte_free(vf->aq_resp); +err_aq: + iavf_shutdown_adminq(hw); +err: + return -1; +} + +/* Enable default admin queue interrupt setting */ +static inline void +iavf_enable_irq0(struct iavf_hw *hw) +{ + /* Enable admin queue interrupt trigger */ + IAVF_WRITE_REG(hw, IAVF_VFINT_ICR0_ENA1, + IAVF_VFINT_ICR0_ENA1_ADMINQ_MASK); + + IAVF_WRITE_REG(hw, IAVF_VFINT_DYN_CTL01, + IAVF_VFINT_DYN_CTL01_INTENA_MASK | + IAVF_VFINT_DYN_CTL01_CLEARPBA_MASK | + IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK); + + IAVF_WRITE_FLUSH(hw); +} + +static inline void +iavf_disable_irq0(struct iavf_hw *hw) +{ + /* Disable all interrupt types */ + IAVF_WRITE_REG(hw, IAVF_VFINT_ICR0_ENA1, 0); + IAVF_WRITE_REG(hw, IAVF_VFINT_DYN_CTL01, + IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK); + IAVF_WRITE_FLUSH(hw); +} + +static void +iavf_dev_interrupt_handler(void *param) +{ + struct rte_eth_dev *dev = (struct rte_eth_dev *)param; + struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); + + iavf_disable_irq0(hw); + + iavf_handle_virtchnl_msg(dev); + + iavf_enable_irq0(hw); +} + +static int +iavf_dev_filter_ctrl(struct rte_eth_dev *dev, + enum rte_filter_type filter_type, + enum rte_filter_op filter_op, + void *arg) +{ + int ret = 0; + + if (!dev) + return -EINVAL; + + switch (filter_type) { + case RTE_ETH_FILTER_GENERIC: + if (filter_op != RTE_ETH_FILTER_GET) + return -EINVAL; + *(const void **)arg = &iavf_flow_ops; + break; + default: + PMD_DRV_LOG(WARNING, "Filter type (%d) not supported", + filter_type); + ret = -EINVAL; + break; + } + + return ret; +} + + +static int +iavf_dev_init(struct rte_eth_dev *eth_dev) +{ + struct iavf_adapter *adapter = + IAVF_DEV_PRIVATE_TO_ADAPTER(eth_dev->data->dev_private); + struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + int ret = 0; + + PMD_INIT_FUNC_TRACE(); + + /* assign ops func pointer */ + eth_dev->dev_ops = &iavf_eth_dev_ops; + eth_dev->rx_pkt_burst = &iavf_recv_pkts; + eth_dev->tx_pkt_burst = &iavf_xmit_pkts; + eth_dev->tx_pkt_prepare = &iavf_prep_pkts; + + /* For secondary processes, we don't initialise any further as primary + * has already done this work. Only check if we need a different RX + * and TX function. + */ + if (rte_eal_process_type() != RTE_PROC_PRIMARY) { + iavf_set_rx_function(eth_dev); + iavf_set_tx_function(eth_dev); + return 0; + } + rte_eth_copy_pci_info(eth_dev, pci_dev); + + hw->vendor_id = pci_dev->id.vendor_id; + hw->device_id = pci_dev->id.device_id; + hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id; + hw->subsystem_device_id = pci_dev->id.subsystem_device_id; + hw->bus.bus_id = pci_dev->addr.bus; + hw->bus.device = pci_dev->addr.devid; + hw->bus.func = pci_dev->addr.function; + hw->hw_addr = (void *)pci_dev->mem_resource[0].addr; + hw->back = IAVF_DEV_PRIVATE_TO_ADAPTER(eth_dev->data->dev_private); + adapter->eth_dev = eth_dev; + adapter->stopped = 1; + + if (iavf_init_vf(eth_dev) != 0) { + PMD_INIT_LOG(ERR, "Init vf failed"); + return -1; + } + + /* set default ptype table */ + adapter->ptype_tbl = iavf_get_default_ptype_table(); + + /* copy mac addr */ + eth_dev->data->mac_addrs = rte_zmalloc( + "iavf_mac", RTE_ETHER_ADDR_LEN * IAVF_NUM_MACADDR_MAX, 0); + if (!eth_dev->data->mac_addrs) { + PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to" + " store MAC addresses", + RTE_ETHER_ADDR_LEN * IAVF_NUM_MACADDR_MAX); + return -ENOMEM; + } + /* If the MAC address is not configured by host, + * generate a random one. + */ + if (!rte_is_valid_assigned_ether_addr( + (struct rte_ether_addr *)hw->mac.addr)) + rte_eth_random_addr(hw->mac.addr); + rte_ether_addr_copy((struct rte_ether_addr *)hw->mac.addr, + ð_dev->data->mac_addrs[0]); + + /* register callback func to eal lib */ + rte_intr_callback_register(&pci_dev->intr_handle, + iavf_dev_interrupt_handler, + (void *)eth_dev); + + /* enable uio intr after callback register */ + rte_intr_enable(&pci_dev->intr_handle); + + /* configure and enable device interrupt */ + iavf_enable_irq0(hw); + + ret = iavf_flow_init(adapter); + if (ret) { + PMD_INIT_LOG(ERR, "Failed to initialize flow"); + return ret; + } + + return 0; +} + +static void +iavf_dev_close(struct rte_eth_dev *dev) +{ + struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; + struct iavf_adapter *adapter = + IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + + iavf_dev_stop(dev); + iavf_flow_flush(dev, NULL); + iavf_shutdown_adminq(hw); + /* disable uio intr before callback unregister */ + rte_intr_disable(intr_handle); + + /* unregister callback func from eal lib */ + rte_intr_callback_unregister(intr_handle, + iavf_dev_interrupt_handler, dev); + iavf_disable_irq0(hw); + + iavf_flow_uninit(adapter); +} + +static int +iavf_dev_uninit(struct rte_eth_dev *dev) +{ + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); + + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return -EPERM; + + dev->dev_ops = NULL; + dev->rx_pkt_burst = NULL; + dev->tx_pkt_burst = NULL; + iavf_dev_close(dev); + + rte_free(vf->vf_res); + vf->vsi_res = NULL; + vf->vf_res = NULL; + + rte_free(vf->aq_resp); + vf->aq_resp = NULL; + + if (vf->rss_lut) { + rte_free(vf->rss_lut); + vf->rss_lut = NULL; + } + if (vf->rss_key) { + rte_free(vf->rss_key); + vf->rss_key = NULL; + } + + return 0; +} + +/* + * Reset VF device only to re-initialize resources in PMD layer + */ +static int +iavf_dev_reset(struct rte_eth_dev *dev) +{ + int ret; + + ret = iavf_dev_uninit(dev); + if (ret) + return ret; + + return iavf_dev_init(dev); +} + +static int +iavf_dcf_cap_check_handler(__rte_unused const char *key, + const char *value, __rte_unused void *opaque) +{ + if (strcmp(value, "dcf")) + return -1; + + return 0; +} + +static int +iavf_dcf_cap_selected(struct rte_devargs *devargs) +{ + struct rte_kvargs *kvlist; + const char *key = "cap"; + int ret = 0; + + if (devargs == NULL) + return 0; + + kvlist = rte_kvargs_parse(devargs->args, NULL); + if (kvlist == NULL) + return 0; + + if (!rte_kvargs_count(kvlist, key)) + goto exit; + + /* dcf capability selected when there's a key-value pair: cap=dcf */ + if (rte_kvargs_process(kvlist, key, + iavf_dcf_cap_check_handler, NULL) < 0) + goto exit; + + ret = 1; + +exit: + rte_kvargs_free(kvlist); + return ret; +} + +static int eth_iavf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, + struct rte_pci_device *pci_dev) +{ + if (iavf_dcf_cap_selected(pci_dev->device.devargs)) + return 1; + + return rte_eth_dev_pci_generic_probe(pci_dev, + sizeof(struct iavf_adapter), iavf_dev_init); +} + +static int eth_iavf_pci_remove(struct rte_pci_device *pci_dev) +{ + return rte_eth_dev_pci_generic_remove(pci_dev, iavf_dev_uninit); +} + +/* Adaptive virtual function driver struct */ +static struct rte_pci_driver rte_iavf_pmd = { + .id_table = pci_id_iavf_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, + .probe = eth_iavf_pci_probe, + .remove = eth_iavf_pci_remove, +}; + +RTE_PMD_REGISTER_PCI(net_iavf, rte_iavf_pmd); +RTE_PMD_REGISTER_PCI_TABLE(net_iavf, pci_id_iavf_map); +RTE_PMD_REGISTER_KMOD_DEP(net_iavf, "* igb_uio | vfio-pci"); +RTE_PMD_REGISTER_PARAM_STRING(net_iavf, "cap=dcf"); +RTE_INIT(iavf_init_log) +{ + iavf_logtype_init = rte_log_register("pmd.net.iavf.init"); + if (iavf_logtype_init >= 0) + rte_log_set_level(iavf_logtype_init, RTE_LOG_NOTICE); + iavf_logtype_driver = rte_log_register("pmd.net.iavf.driver"); + if (iavf_logtype_driver >= 0) + rte_log_set_level(iavf_logtype_driver, RTE_LOG_NOTICE); + +#ifdef RTE_LIBRTE_IAVF_DEBUG_RX + iavf_logtype_rx = rte_log_register("pmd.net.iavf.rx"); + if (iavf_logtype_rx >= 0) + rte_log_set_level(iavf_logtype_rx, RTE_LOG_DEBUG); +#endif + +#ifdef RTE_LIBRTE_IAVF_DEBUG_TX + iavf_logtype_tx = rte_log_register("pmd.net.iavf.tx"); + if (iavf_logtype_tx >= 0) + rte_log_set_level(iavf_logtype_tx, RTE_LOG_DEBUG); +#endif + +#ifdef RTE_LIBRTE_IAVF_DEBUG_TX_FREE + iavf_logtype_tx_free = rte_log_register("pmd.net.iavf.tx_free"); + if (iavf_logtype_tx_free >= 0) + rte_log_set_level(iavf_logtype_tx_free, RTE_LOG_DEBUG); +#endif +} diff --git a/src/spdk/dpdk/drivers/net/iavf/iavf_fdir.c b/src/spdk/dpdk/drivers/net/iavf/iavf_fdir.c new file mode 100644 index 000000000..264c47d83 --- /dev/null +++ b/src/spdk/dpdk/drivers/net/iavf/iavf_fdir.c @@ -0,0 +1,971 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2020 Intel Corporation + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "iavf.h" +#include "iavf_generic_flow.h" +#include "virtchnl.h" +#include "iavf_rxtx.h" + +#define IAVF_FDIR_MAX_QREGION_SIZE 128 + +#define IAVF_FDIR_IPV6_TC_OFFSET 20 +#define IAVF_IPV6_TC_MASK (0xFF << IAVF_FDIR_IPV6_TC_OFFSET) + +#define IAVF_FDIR_INSET_ETH (\ + IAVF_INSET_ETHERTYPE) + +#define IAVF_FDIR_INSET_ETH_IPV4 (\ + IAVF_INSET_IPV4_SRC | IAVF_INSET_IPV4_DST | \ + IAVF_INSET_IPV4_PROTO | IAVF_INSET_IPV4_TOS | \ + IAVF_INSET_IPV4_TTL) + +#define IAVF_FDIR_INSET_ETH_IPV4_UDP (\ + IAVF_INSET_IPV4_SRC | IAVF_INSET_IPV4_DST | \ + IAVF_INSET_IPV4_TOS | IAVF_INSET_IPV4_TTL | \ + IAVF_INSET_UDP_SRC_PORT | IAVF_INSET_UDP_DST_PORT) + +#define IAVF_FDIR_INSET_ETH_IPV4_TCP (\ + IAVF_INSET_IPV4_SRC | IAVF_INSET_IPV4_DST | \ + IAVF_INSET_IPV4_TOS | IAVF_INSET_IPV4_TTL | \ + IAVF_INSET_TCP_SRC_PORT | IAVF_INSET_TCP_DST_PORT) + +#define IAVF_FDIR_INSET_ETH_IPV4_SCTP (\ + IAVF_INSET_IPV4_SRC | IAVF_INSET_IPV4_DST | \ + IAVF_INSET_IPV4_TOS | IAVF_INSET_IPV4_TTL | \ + IAVF_INSET_SCTP_SRC_PORT | IAVF_INSET_SCTP_DST_PORT) + +#define IAVF_FDIR_INSET_ETH_IPV6 (\ + IAVF_INSET_IPV6_SRC | IAVF_INSET_IPV6_DST | \ + IAVF_INSET_IPV6_NEXT_HDR | IAVF_INSET_IPV6_TC | \ + IAVF_INSET_IPV6_HOP_LIMIT) + +#define IAVF_FDIR_INSET_ETH_IPV6_UDP (\ + IAVF_INSET_IPV6_SRC | IAVF_INSET_IPV6_DST | \ + IAVF_INSET_IPV6_TC | IAVF_INSET_IPV6_HOP_LIMIT | \ + IAVF_INSET_UDP_SRC_PORT | IAVF_INSET_UDP_DST_PORT) + +#define IAVF_FDIR_INSET_ETH_IPV6_TCP (\ + IAVF_INSET_IPV6_SRC | IAVF_INSET_IPV6_DST | \ + IAVF_INSET_IPV6_TC | IAVF_INSET_IPV6_HOP_LIMIT | \ + IAVF_INSET_TCP_SRC_PORT | IAVF_INSET_TCP_DST_PORT) + +#define IAVF_FDIR_INSET_ETH_IPV6_SCTP (\ + IAVF_INSET_IPV6_SRC | IAVF_INSET_IPV6_DST | \ + IAVF_INSET_IPV6_TC | IAVF_INSET_IPV6_HOP_LIMIT | \ + IAVF_INSET_SCTP_SRC_PORT | IAVF_INSET_SCTP_DST_PORT) + +#define IAVF_FDIR_INSET_GTPU (\ + IAVF_INSET_IPV4_SRC | IAVF_INSET_IPV4_DST | \ + IAVF_INSET_GTPU_TEID) + +#define IAVF_FDIR_INSET_GTPU_EH (\ + IAVF_INSET_IPV4_SRC | IAVF_INSET_IPV4_DST | \ + IAVF_INSET_GTPU_TEID | IAVF_INSET_GTPU_QFI) + +#define IAVF_FDIR_INSET_L2TPV3OIP (\ + IAVF_L2TPV3OIP_SESSION_ID) + +#define IAVF_FDIR_INSET_ESP (\ + IAVF_INSET_ESP_SPI) + +#define IAVF_FDIR_INSET_AH (\ + IAVF_INSET_AH_SPI) + +#define IAVF_FDIR_INSET_IPV4_NATT_ESP (\ + IAVF_INSET_IPV4_SRC | IAVF_INSET_IPV4_DST | \ + IAVF_INSET_ESP_SPI) + +#define IAVF_FDIR_INSET_IPV6_NATT_ESP (\ + IAVF_INSET_IPV6_SRC | IAVF_INSET_IPV6_DST | \ + IAVF_INSET_ESP_SPI) + +#define IAVF_FDIR_INSET_PFCP (\ + IAVF_INSET_PFCP_S_FIELD) + +static struct iavf_pattern_match_item iavf_fdir_pattern[] = { + {iavf_pattern_ethertype, IAVF_FDIR_INSET_ETH, IAVF_INSET_NONE}, + {iavf_pattern_eth_ipv4, IAVF_FDIR_INSET_ETH_IPV4, IAVF_INSET_NONE}, + {iavf_pattern_eth_ipv4_udp, IAVF_FDIR_INSET_ETH_IPV4_UDP, IAVF_INSET_NONE}, + {iavf_pattern_eth_ipv4_tcp, IAVF_FDIR_INSET_ETH_IPV4_TCP, IAVF_INSET_NONE}, + {iavf_pattern_eth_ipv4_sctp, IAVF_FDIR_INSET_ETH_IPV4_SCTP, IAVF_INSET_NONE}, + {iavf_pattern_eth_ipv6, IAVF_FDIR_INSET_ETH_IPV6, IAVF_INSET_NONE}, + {iavf_pattern_eth_ipv6_udp, IAVF_FDIR_INSET_ETH_IPV6_UDP, IAVF_INSET_NONE}, + {iavf_pattern_eth_ipv6_tcp, IAVF_FDIR_INSET_ETH_IPV6_TCP, IAVF_INSET_NONE}, + {iavf_pattern_eth_ipv6_sctp, IAVF_FDIR_INSET_ETH_IPV6_SCTP, IAVF_INSET_NONE}, + {iavf_pattern_eth_ipv4_gtpu, IAVF_FDIR_INSET_GTPU, IAVF_INSET_NONE}, + {iavf_pattern_eth_ipv4_gtpu_eh, IAVF_FDIR_INSET_GTPU_EH, IAVF_INSET_NONE}, + {iavf_pattern_eth_ipv4_l2tpv3, IAVF_FDIR_INSET_L2TPV3OIP, IAVF_INSET_NONE}, + {iavf_pattern_eth_ipv6_l2tpv3, IAVF_FDIR_INSET_L2TPV3OIP, IAVF_INSET_NONE}, + {iavf_pattern_eth_ipv4_esp, IAVF_FDIR_INSET_ESP, IAVF_INSET_NONE}, + {iavf_pattern_eth_ipv6_esp, IAVF_FDIR_INSET_ESP, IAVF_INSET_NONE}, + {iavf_pattern_eth_ipv4_ah, IAVF_FDIR_INSET_AH, IAVF_INSET_NONE}, + {iavf_pattern_eth_ipv6_ah, IAVF_FDIR_INSET_AH, IAVF_INSET_NONE}, + {iavf_pattern_eth_ipv4_udp_esp, IAVF_FDIR_INSET_IPV4_NATT_ESP, IAVF_INSET_NONE}, + {iavf_pattern_eth_ipv6_udp_esp, IAVF_FDIR_INSET_IPV6_NATT_ESP, IAVF_INSET_NONE}, + {iavf_pattern_eth_ipv4_pfcp, IAVF_FDIR_INSET_PFCP, IAVF_INSET_NONE}, + {iavf_pattern_eth_ipv6_pfcp, IAVF_FDIR_INSET_PFCP, IAVF_INSET_NONE}, +}; + +static struct iavf_flow_parser iavf_fdir_parser; + +static int +iavf_fdir_init(struct iavf_adapter *ad) +{ + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad); + struct iavf_flow_parser *parser; + + if (!vf->vf_res) + return -EINVAL; + + if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_FDIR_PF) + parser = &iavf_fdir_parser; + else + return -ENOTSUP; + + return iavf_register_parser(parser, ad); +} + +static void +iavf_fdir_uninit(struct iavf_adapter *ad) +{ + iavf_unregister_parser(&iavf_fdir_parser, ad); +} + +static int +iavf_fdir_create(struct iavf_adapter *ad, + struct rte_flow *flow, + void *meta, + struct rte_flow_error *error) +{ + struct iavf_fdir_conf *filter = meta; + struct iavf_fdir_conf *rule; + int ret; + + rule = rte_zmalloc("fdir_entry", sizeof(*rule), 0); + if (!rule) { + rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "Failed to allocate memory for fdir rule"); + return -rte_errno; + } + + ret = iavf_fdir_add(ad, filter); + if (ret) { + rte_flow_error_set(error, -ret, + RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "Failed to add filter rule."); + goto free_entry; + } + + if (filter->mark_flag == 1) + iavf_fdir_rx_proc_enable(ad, 1); + + rte_memcpy(rule, filter, sizeof(*rule)); + flow->rule = rule; + + return 0; + +free_entry: + rte_free(rule); + return -rte_errno; +} + +static int +iavf_fdir_destroy(struct iavf_adapter *ad, + struct rte_flow *flow, + struct rte_flow_error *error) +{ + struct iavf_fdir_conf *filter; + int ret; + + filter = (struct iavf_fdir_conf *)flow->rule; + + ret = iavf_fdir_del(ad, filter); + if (ret) { + rte_flow_error_set(error, -ret, + RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "Failed to delete filter rule."); + return -rte_errno; + } + + if (filter->mark_flag == 1) + iavf_fdir_rx_proc_enable(ad, 0); + + flow->rule = NULL; + rte_free(filter); + + return 0; +} + +static int +iavf_fdir_validation(struct iavf_adapter *ad, + __rte_unused struct rte_flow *flow, + void *meta, + struct rte_flow_error *error) +{ + struct iavf_fdir_conf *filter = meta; + int ret; + + ret = iavf_fdir_check(ad, filter); + if (ret) { + rte_flow_error_set(error, -ret, + RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "Failed to validate filter rule."); + return -rte_errno; + } + + return 0; +}; + +static struct iavf_flow_engine iavf_fdir_engine = { + .init = iavf_fdir_init, + .uninit = iavf_fdir_uninit, + .create = iavf_fdir_create, + .destroy = iavf_fdir_destroy, + .validation = iavf_fdir_validation, + .type = IAVF_FLOW_ENGINE_FDIR, +}; + +static int +iavf_fdir_parse_action_qregion(struct iavf_adapter *ad, + struct rte_flow_error *error, + const struct rte_flow_action *act, + struct virtchnl_filter_action *filter_action) +{ + const struct rte_flow_action_rss *rss = act->conf; + uint32_t i; + + if (act->type != RTE_FLOW_ACTION_TYPE_RSS) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, act, + "Invalid action."); + return -rte_errno; + } + + if (rss->queue_num <= 1) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, act, + "Queue region size can't be 0 or 1."); + return -rte_errno; + } + + /* check if queue index for queue region is continuous */ + for (i = 0; i < rss->queue_num - 1; i++) { + if (rss->queue[i + 1] != rss->queue[i] + 1) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, act, + "Discontinuous queue region"); + return -rte_errno; + } + } + + if (rss->queue[rss->queue_num - 1] >= ad->eth_dev->data->nb_rx_queues) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, act, + "Invalid queue region indexes."); + return -rte_errno; + } + + if (!(rte_is_power_of_2(rss->queue_num) && + rss->queue_num <= IAVF_FDIR_MAX_QREGION_SIZE)) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, act, + "The region size should be any of the following values:" + "1, 2, 4, 8, 16, 32, 64, 128 as long as the total number " + "of queues do not exceed the VSI allocation."); + return -rte_errno; + } + + filter_action->act_conf.queue.index = rss->queue[0]; + filter_action->act_conf.queue.region = rte_fls_u32(rss->queue_num) - 1; + + return 0; +} + +static int +iavf_fdir_parse_action(struct iavf_adapter *ad, + const struct rte_flow_action actions[], + struct rte_flow_error *error, + struct iavf_fdir_conf *filter) +{ + const struct rte_flow_action_queue *act_q; + const struct rte_flow_action_mark *mark_spec = NULL; + uint32_t dest_num = 0; + uint32_t mark_num = 0; + int ret; + + int number = 0; + struct virtchnl_filter_action *filter_action; + + for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { + switch (actions->type) { + case RTE_FLOW_ACTION_TYPE_VOID: + break; + + case RTE_FLOW_ACTION_TYPE_PASSTHRU: + dest_num++; + + filter_action = &filter->add_fltr.rule_cfg.action_set.actions[number]; + + filter_action->type = VIRTCHNL_ACTION_PASSTHRU; + + filter->add_fltr.rule_cfg.action_set.count = ++number; + break; + + case RTE_FLOW_ACTION_TYPE_DROP: + dest_num++; + + filter_action = &filter->add_fltr.rule_cfg.action_set.actions[number]; + + filter_action->type = VIRTCHNL_ACTION_DROP; + + filter->add_fltr.rule_cfg.action_set.count = ++number; + break; + + case RTE_FLOW_ACTION_TYPE_QUEUE: + dest_num++; + + act_q = actions->conf; + filter_action = &filter->add_fltr.rule_cfg.action_set.actions[number]; + + filter_action->type = VIRTCHNL_ACTION_QUEUE; + filter_action->act_conf.queue.index = act_q->index; + + if (filter_action->act_conf.queue.index >= + ad->eth_dev->data->nb_rx_queues) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + actions, "Invalid queue for FDIR."); + return -rte_errno; + } + + filter->add_fltr.rule_cfg.action_set.count = ++number; + break; + + case RTE_FLOW_ACTION_TYPE_RSS: + dest_num++; + + filter_action = &filter->add_fltr.rule_cfg.action_set.actions[number]; + + filter_action->type = VIRTCHNL_ACTION_Q_REGION; + + ret = iavf_fdir_parse_action_qregion(ad, + error, actions, filter_action); + if (ret) + return ret; + + filter->add_fltr.rule_cfg.action_set.count = ++number; + break; + + case RTE_FLOW_ACTION_TYPE_MARK: + mark_num++; + + filter->mark_flag = 1; + mark_spec = actions->conf; + filter_action = &filter->add_fltr.rule_cfg.action_set.actions[number]; + + filter_action->type = VIRTCHNL_ACTION_MARK; + filter_action->act_conf.mark_id = mark_spec->id; + + filter->add_fltr.rule_cfg.action_set.count = ++number; + break; + + default: + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, actions, + "Invalid action."); + return -rte_errno; + } + } + + if (number > VIRTCHNL_MAX_NUM_ACTIONS) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, actions, + "Action numbers exceed the maximum value"); + return -rte_errno; + } + + if (dest_num >= 2) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, actions, + "Unsupported action combination"); + return -rte_errno; + } + + if (mark_num >= 2) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, actions, + "Too many mark actions"); + return -rte_errno; + } + + if (dest_num + mark_num == 0) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, actions, + "Empty action"); + return -rte_errno; + } + + /* Mark only is equal to mark + passthru. */ + if (dest_num == 0) { + filter_action = &filter->add_fltr.rule_cfg.action_set.actions[number]; + filter_action->type = VIRTCHNL_ACTION_PASSTHRU; + filter->add_fltr.rule_cfg.action_set.count = ++number; + } + + return 0; +} + +static int +iavf_fdir_parse_pattern(__rte_unused struct iavf_adapter *ad, + const struct rte_flow_item pattern[], + struct rte_flow_error *error, + struct iavf_fdir_conf *filter) +{ + const struct rte_flow_item *item = pattern; + enum rte_flow_item_type item_type; + enum rte_flow_item_type l3 = RTE_FLOW_ITEM_TYPE_END; + const struct rte_flow_item_eth *eth_spec, *eth_mask; + const struct rte_flow_item_ipv4 *ipv4_spec, *ipv4_mask; + const struct rte_flow_item_ipv6 *ipv6_spec, *ipv6_mask; + const struct rte_flow_item_udp *udp_spec, *udp_mask; + const struct rte_flow_item_tcp *tcp_spec, *tcp_mask; + const struct rte_flow_item_sctp *sctp_spec, *sctp_mask; + const struct rte_flow_item_gtp *gtp_spec, *gtp_mask; + const struct rte_flow_item_gtp_psc *gtp_psc_spec, *gtp_psc_mask; + const struct rte_flow_item_l2tpv3oip *l2tpv3oip_spec, *l2tpv3oip_mask; + const struct rte_flow_item_esp *esp_spec, *esp_mask; + const struct rte_flow_item_ah *ah_spec, *ah_mask; + const struct rte_flow_item_pfcp *pfcp_spec, *pfcp_mask; + uint64_t input_set = IAVF_INSET_NONE; + + enum rte_flow_item_type next_type; + uint16_t ether_type; + + int layer = 0; + struct virtchnl_proto_hdr *hdr; + + uint8_t ipv6_addr_mask[16] = { + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF + }; + + for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "Not support range"); + } + + item_type = item->type; + + switch (item_type) { + case RTE_FLOW_ITEM_TYPE_ETH: + eth_spec = item->spec; + eth_mask = item->mask; + next_type = (item + 1)->type; + + hdr = &filter->add_fltr.rule_cfg.proto_hdrs.proto_hdr[layer]; + + VIRTCHNL_SET_PROTO_HDR_TYPE(hdr, ETH); + + if (next_type == RTE_FLOW_ITEM_TYPE_END && + (!eth_spec || !eth_mask)) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "NULL eth spec/mask."); + return -rte_errno; + } + + if (eth_spec && eth_mask) { + if (!rte_is_zero_ether_addr(ð_mask->src) || + !rte_is_zero_ether_addr(ð_mask->dst)) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "Invalid MAC_addr mask."); + return -rte_errno; + } + } + + if (eth_spec && eth_mask && eth_mask->type) { + if (eth_mask->type != RTE_BE16(0xffff)) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Invalid type mask."); + return -rte_errno; + } + + ether_type = rte_be_to_cpu_16(eth_spec->type); + if (ether_type == RTE_ETHER_TYPE_IPV4 || + ether_type == RTE_ETHER_TYPE_IPV6) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Unsupported ether_type."); + return -rte_errno; + } + + input_set |= IAVF_INSET_ETHERTYPE; + VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, ETH, ETHERTYPE); + + rte_memcpy(hdr->buffer, + eth_spec, sizeof(*eth_spec)); + } + + filter->add_fltr.rule_cfg.proto_hdrs.count = ++layer; + break; + + case RTE_FLOW_ITEM_TYPE_IPV4: + l3 = RTE_FLOW_ITEM_TYPE_IPV4; + ipv4_spec = item->spec; + ipv4_mask = item->mask; + + hdr = &filter->add_fltr.rule_cfg.proto_hdrs.proto_hdr[layer]; + + VIRTCHNL_SET_PROTO_HDR_TYPE(hdr, IPV4); + + if (ipv4_spec && ipv4_mask) { + if (ipv4_mask->hdr.version_ihl || + ipv4_mask->hdr.total_length || + ipv4_mask->hdr.packet_id || + ipv4_mask->hdr.fragment_offset || + ipv4_mask->hdr.hdr_checksum) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Invalid IPv4 mask."); + return -rte_errno; + } + + if (ipv4_mask->hdr.type_of_service == + UINT8_MAX) { + input_set |= IAVF_INSET_IPV4_TOS; + VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, IPV4, DSCP); + } + if (ipv4_mask->hdr.next_proto_id == UINT8_MAX) { + input_set |= IAVF_INSET_IPV4_PROTO; + VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, IPV4, PROT); + } + if (ipv4_mask->hdr.time_to_live == UINT8_MAX) { + input_set |= IAVF_INSET_IPV4_TTL; + VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, IPV4, TTL); + } + if (ipv4_mask->hdr.src_addr == UINT32_MAX) { + input_set |= IAVF_INSET_IPV4_SRC; + VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, IPV4, SRC); + } + if (ipv4_mask->hdr.dst_addr == UINT32_MAX) { + input_set |= IAVF_INSET_IPV4_DST; + VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, IPV4, DST); + } + + rte_memcpy(hdr->buffer, + &ipv4_spec->hdr, + sizeof(ipv4_spec->hdr)); + } + + filter->add_fltr.rule_cfg.proto_hdrs.count = ++layer; + break; + + case RTE_FLOW_ITEM_TYPE_IPV6: + l3 = RTE_FLOW_ITEM_TYPE_IPV6; + ipv6_spec = item->spec; + ipv6_mask = item->mask; + + hdr = &filter->add_fltr.rule_cfg.proto_hdrs.proto_hdr[layer]; + + VIRTCHNL_SET_PROTO_HDR_TYPE(hdr, IPV6); + + if (ipv6_spec && ipv6_mask) { + if (ipv6_mask->hdr.payload_len) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Invalid IPv6 mask"); + return -rte_errno; + } + + if ((ipv6_mask->hdr.vtc_flow & + rte_cpu_to_be_32(IAVF_IPV6_TC_MASK)) + == rte_cpu_to_be_32(IAVF_IPV6_TC_MASK)) { + input_set |= IAVF_INSET_IPV6_TC; + VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, IPV6, TC); + } + if (ipv6_mask->hdr.proto == UINT8_MAX) { + input_set |= IAVF_INSET_IPV6_NEXT_HDR; + VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, IPV6, PROT); + } + if (ipv6_mask->hdr.hop_limits == UINT8_MAX) { + input_set |= IAVF_INSET_IPV6_HOP_LIMIT; + VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, IPV6, HOP_LIMIT); + } + if (!memcmp(ipv6_mask->hdr.src_addr, + ipv6_addr_mask, + RTE_DIM(ipv6_mask->hdr.src_addr))) { + input_set |= IAVF_INSET_IPV6_SRC; + VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, IPV6, SRC); + } + if (!memcmp(ipv6_mask->hdr.dst_addr, + ipv6_addr_mask, + RTE_DIM(ipv6_mask->hdr.dst_addr))) { + input_set |= IAVF_INSET_IPV6_DST; + VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, IPV6, DST); + } + + rte_memcpy(hdr->buffer, + &ipv6_spec->hdr, + sizeof(ipv6_spec->hdr)); + } + + filter->add_fltr.rule_cfg.proto_hdrs.count = ++layer; + break; + + case RTE_FLOW_ITEM_TYPE_UDP: + udp_spec = item->spec; + udp_mask = item->mask; + + hdr = &filter->add_fltr.rule_cfg.proto_hdrs.proto_hdr[layer]; + + VIRTCHNL_SET_PROTO_HDR_TYPE(hdr, UDP); + + if (udp_spec && udp_mask) { + if (udp_mask->hdr.dgram_len || + udp_mask->hdr.dgram_cksum) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "Invalid UDP mask"); + return -rte_errno; + } + + if (udp_mask->hdr.src_port == UINT16_MAX) { + input_set |= IAVF_INSET_UDP_SRC_PORT; + VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, UDP, SRC_PORT); + } + if (udp_mask->hdr.dst_port == UINT16_MAX) { + input_set |= IAVF_INSET_UDP_DST_PORT; + VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, UDP, DST_PORT); + } + + if (l3 == RTE_FLOW_ITEM_TYPE_IPV4) + rte_memcpy(hdr->buffer, + &udp_spec->hdr, + sizeof(udp_spec->hdr)); + else if (l3 == RTE_FLOW_ITEM_TYPE_IPV6) + rte_memcpy(hdr->buffer, + &udp_spec->hdr, + sizeof(udp_spec->hdr)); + } + + filter->add_fltr.rule_cfg.proto_hdrs.count = ++layer; + break; + + case RTE_FLOW_ITEM_TYPE_TCP: + tcp_spec = item->spec; + tcp_mask = item->mask; + + hdr = &filter->add_fltr.rule_cfg.proto_hdrs.proto_hdr[layer]; + + VIRTCHNL_SET_PROTO_HDR_TYPE(hdr, TCP); + + if (tcp_spec && tcp_mask) { + if (tcp_mask->hdr.sent_seq || + tcp_mask->hdr.recv_ack || + tcp_mask->hdr.data_off || + tcp_mask->hdr.tcp_flags || + tcp_mask->hdr.rx_win || + tcp_mask->hdr.cksum || + tcp_mask->hdr.tcp_urp) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "Invalid TCP mask"); + return -rte_errno; + } + + if (tcp_mask->hdr.src_port == UINT16_MAX) { + input_set |= IAVF_INSET_TCP_SRC_PORT; + VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, TCP, SRC_PORT); + } + if (tcp_mask->hdr.dst_port == UINT16_MAX) { + input_set |= IAVF_INSET_TCP_DST_PORT; + VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, TCP, DST_PORT); + } + + if (l3 == RTE_FLOW_ITEM_TYPE_IPV4) + rte_memcpy(hdr->buffer, + &tcp_spec->hdr, + sizeof(tcp_spec->hdr)); + else if (l3 == RTE_FLOW_ITEM_TYPE_IPV6) + rte_memcpy(hdr->buffer, + &tcp_spec->hdr, + sizeof(tcp_spec->hdr)); + } + + filter->add_fltr.rule_cfg.proto_hdrs.count = ++layer; + break; + + case RTE_FLOW_ITEM_TYPE_SCTP: + sctp_spec = item->spec; + sctp_mask = item->mask; + + hdr = &filter->add_fltr.rule_cfg.proto_hdrs.proto_hdr[layer]; + + VIRTCHNL_SET_PROTO_HDR_TYPE(hdr, SCTP); + + if (sctp_spec && sctp_mask) { + if (sctp_mask->hdr.cksum) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "Invalid UDP mask"); + return -rte_errno; + } + + if (sctp_mask->hdr.src_port == UINT16_MAX) { + input_set |= IAVF_INSET_SCTP_SRC_PORT; + VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, SCTP, SRC_PORT); + } + if (sctp_mask->hdr.dst_port == UINT16_MAX) { + input_set |= IAVF_INSET_SCTP_DST_PORT; + VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, SCTP, DST_PORT); + } + + if (l3 == RTE_FLOW_ITEM_TYPE_IPV4) + rte_memcpy(hdr->buffer, + &sctp_spec->hdr, + sizeof(sctp_spec->hdr)); + else if (l3 == RTE_FLOW_ITEM_TYPE_IPV6) + rte_memcpy(hdr->buffer, + &sctp_spec->hdr, + sizeof(sctp_spec->hdr)); + } + + filter->add_fltr.rule_cfg.proto_hdrs.count = ++layer; + break; + + case RTE_FLOW_ITEM_TYPE_GTPU: + gtp_spec = item->spec; + gtp_mask = item->mask; + + hdr = &filter->add_fltr.rule_cfg.proto_hdrs.proto_hdr[layer]; + + VIRTCHNL_SET_PROTO_HDR_TYPE(hdr, GTPU_IP); + + if (gtp_spec && gtp_mask) { + if (gtp_mask->v_pt_rsv_flags || + gtp_mask->msg_type || + gtp_mask->msg_len) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Invalid GTP mask"); + return -rte_errno; + } + + if (gtp_mask->teid == UINT32_MAX) { + input_set |= IAVF_INSET_GTPU_TEID; + VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, GTPU_IP, TEID); + } + + rte_memcpy(hdr->buffer, + gtp_spec, sizeof(*gtp_spec)); + } + + filter->add_fltr.rule_cfg.proto_hdrs.count = ++layer; + break; + + case RTE_FLOW_ITEM_TYPE_GTP_PSC: + gtp_psc_spec = item->spec; + gtp_psc_mask = item->mask; + + hdr = &filter->add_fltr.rule_cfg.proto_hdrs.proto_hdr[layer]; + + VIRTCHNL_SET_PROTO_HDR_TYPE(hdr, GTPU_EH); + + if (gtp_psc_spec && gtp_psc_mask) { + if (gtp_psc_mask->qfi == UINT8_MAX) { + input_set |= IAVF_INSET_GTPU_QFI; + VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, GTPU_EH, QFI); + } + + rte_memcpy(hdr->buffer, gtp_psc_spec, + sizeof(*gtp_psc_spec)); + } + + filter->add_fltr.rule_cfg.proto_hdrs.count = ++layer; + break; + + case RTE_FLOW_ITEM_TYPE_L2TPV3OIP: + l2tpv3oip_spec = item->spec; + l2tpv3oip_mask = item->mask; + + hdr = &filter->add_fltr.rule_cfg.proto_hdrs.proto_hdr[layer]; + + VIRTCHNL_SET_PROTO_HDR_TYPE(hdr, L2TPV3); + + if (l2tpv3oip_spec && l2tpv3oip_mask) { + if (l2tpv3oip_mask->session_id == UINT32_MAX) { + input_set |= IAVF_L2TPV3OIP_SESSION_ID; + VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, L2TPV3, SESS_ID); + } + + rte_memcpy(hdr->buffer, l2tpv3oip_spec, + sizeof(*l2tpv3oip_spec)); + } + + filter->add_fltr.rule_cfg.proto_hdrs.count = ++layer; + break; + + case RTE_FLOW_ITEM_TYPE_ESP: + esp_spec = item->spec; + esp_mask = item->mask; + + hdr = &filter->add_fltr.rule_cfg.proto_hdrs.proto_hdr[layer]; + + VIRTCHNL_SET_PROTO_HDR_TYPE(hdr, ESP); + + if (esp_spec && esp_mask) { + if (esp_mask->hdr.spi == UINT32_MAX) { + input_set |= IAVF_INSET_ESP_SPI; + VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, ESP, SPI); + } + + rte_memcpy(hdr->buffer, &esp_spec->hdr, + sizeof(esp_spec->hdr)); + } + + filter->add_fltr.rule_cfg.proto_hdrs.count = ++layer; + break; + + case RTE_FLOW_ITEM_TYPE_AH: + ah_spec = item->spec; + ah_mask = item->mask; + + hdr = &filter->add_fltr.rule_cfg.proto_hdrs.proto_hdr[layer]; + + VIRTCHNL_SET_PROTO_HDR_TYPE(hdr, AH); + + if (ah_spec && ah_mask) { + if (ah_mask->spi == UINT32_MAX) { + input_set |= IAVF_INSET_AH_SPI; + VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, AH, SPI); + } + + rte_memcpy(hdr->buffer, ah_spec, + sizeof(*ah_spec)); + } + + filter->add_fltr.rule_cfg.proto_hdrs.count = ++layer; + break; + + case RTE_FLOW_ITEM_TYPE_PFCP: + pfcp_spec = item->spec; + pfcp_mask = item->mask; + + hdr = &filter->add_fltr.rule_cfg.proto_hdrs.proto_hdr[layer]; + + VIRTCHNL_SET_PROTO_HDR_TYPE(hdr, PFCP); + + if (pfcp_spec && pfcp_mask) { + if (pfcp_mask->s_field == UINT8_MAX) { + input_set |= IAVF_INSET_PFCP_S_FIELD; + VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, PFCP, S_FIELD); + } + + rte_memcpy(hdr->buffer, pfcp_spec, + sizeof(*pfcp_spec)); + } + + filter->add_fltr.rule_cfg.proto_hdrs.count = ++layer; + break; + + case RTE_FLOW_ITEM_TYPE_VOID: + break; + + default: + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "Invalid pattern item."); + return -rte_errno; + } + } + + if (layer > VIRTCHNL_MAX_NUM_PROTO_HDRS) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "Protocol header layers exceed the maximum value"); + return -rte_errno; + } + + filter->input_set = input_set; + + return 0; +} + +static int +iavf_fdir_parse(struct iavf_adapter *ad, + struct iavf_pattern_match_item *array, + uint32_t array_len, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + void **meta, + struct rte_flow_error *error) +{ + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad); + struct iavf_fdir_conf *filter = &vf->fdir.conf; + struct iavf_pattern_match_item *item = NULL; + uint64_t input_set; + int ret; + + memset(filter, 0, sizeof(*filter)); + + item = iavf_search_pattern_match_item(pattern, array, array_len, error); + if (!item) + return -rte_errno; + + ret = iavf_fdir_parse_pattern(ad, pattern, error, filter); + if (ret) + goto error; + + input_set = filter->input_set; + if (!input_set || input_set & ~item->input_set_mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM_SPEC, pattern, + "Invalid input set"); + ret = -rte_errno; + goto error; + } + + ret = iavf_fdir_parse_action(ad, actions, error, filter); + if (ret) + goto error; + + if (meta) + *meta = filter; + +error: + rte_free(item); + return ret; +} + +static struct iavf_flow_parser iavf_fdir_parser = { + .engine = &iavf_fdir_engine, + .array = iavf_fdir_pattern, + .array_len = RTE_DIM(iavf_fdir_pattern), + .parse_pattern_action = iavf_fdir_parse, + .stage = IAVF_FLOW_STAGE_DISTRIBUTOR, +}; + +RTE_INIT(iavf_fdir_engine_register) +{ + iavf_register_flow_engine(&iavf_fdir_engine); +} diff --git a/src/spdk/dpdk/drivers/net/iavf/iavf_generic_flow.c b/src/spdk/dpdk/drivers/net/iavf/iavf_generic_flow.c new file mode 100644 index 000000000..b6c26c4fd --- /dev/null +++ b/src/spdk/dpdk/drivers/net/iavf/iavf_generic_flow.c @@ -0,0 +1,1044 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2019 Intel Corporation + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "iavf.h" +#include "iavf_generic_flow.h" + +static struct iavf_engine_list engine_list = + TAILQ_HEAD_INITIALIZER(engine_list); + +static int iavf_flow_validate(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error); +static struct rte_flow *iavf_flow_create(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error); +static int iavf_flow_destroy(struct rte_eth_dev *dev, + struct rte_flow *flow, + struct rte_flow_error *error); +static int iavf_flow_query(struct rte_eth_dev *dev, + struct rte_flow *flow, + const struct rte_flow_action *actions, + void *data, + struct rte_flow_error *error); + +const struct rte_flow_ops iavf_flow_ops = { + .validate = iavf_flow_validate, + .create = iavf_flow_create, + .destroy = iavf_flow_destroy, + .flush = iavf_flow_flush, + .query = iavf_flow_query, +}; + +/* empty */ +enum rte_flow_item_type iavf_pattern_empty[] = { + RTE_FLOW_ITEM_TYPE_END, +}; + +/* L2 */ +enum rte_flow_item_type iavf_pattern_ethertype[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_ethertype_vlan[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_ethertype_qinq[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_END, +}; + +/* ARP */ +enum rte_flow_item_type iavf_pattern_eth_arp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_ARP_ETH_IPV4, + RTE_FLOW_ITEM_TYPE_END, +}; + +/* non-tunnel IPv4 */ +enum rte_flow_item_type iavf_pattern_eth_ipv4[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_vlan_ipv4[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_qinq_ipv4[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_ipv4_udp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_vlan_ipv4_udp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_qinq_ipv4_udp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_ipv4_tcp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_vlan_ipv4_tcp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_qinq_ipv4_tcp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_ipv4_sctp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_vlan_ipv4_sctp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_qinq_ipv4_sctp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_ipv4_icmp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_ICMP, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_vlan_ipv4_icmp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_ICMP, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_qinq_ipv4_icmp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_ICMP, + RTE_FLOW_ITEM_TYPE_END, +}; + +/* non-tunnel IPv6 */ +enum rte_flow_item_type iavf_pattern_eth_ipv6[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_vlan_ipv6[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_qinq_ipv6[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_ipv6_udp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_vlan_ipv6_udp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_qinq_ipv6_udp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_ipv6_tcp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_vlan_ipv6_tcp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_qinq_ipv6_tcp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_ipv6_sctp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_vlan_ipv6_sctp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_qinq_ipv6_sctp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_ipv6_icmp6[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_ICMP6, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_vlan_ipv6_icmp6[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_ICMP6, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_qinq_ipv6_icmp6[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_ICMP6, + RTE_FLOW_ITEM_TYPE_END, +}; + +/* GTPU */ +enum rte_flow_item_type iavf_pattern_eth_ipv4_gtpu[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_GTPU, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_ipv4_gtpu_eh[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_GTPU, + RTE_FLOW_ITEM_TYPE_GTP_PSC, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_ipv4_gtpu_ipv4[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_GTPU, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_ipv4_gtpu_eh_ipv4[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_GTPU, + RTE_FLOW_ITEM_TYPE_GTP_PSC, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_ipv4_gtpu_eh_ipv4_udp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_GTPU, + RTE_FLOW_ITEM_TYPE_GTP_PSC, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_ipv4_gtpu_eh_ipv4_tcp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_GTPU, + RTE_FLOW_ITEM_TYPE_GTP_PSC, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_END, + +}; + +enum rte_flow_item_type iavf_pattern_eth_ipv4_gtpu_eh_ipv4_icmp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_GTPU, + RTE_FLOW_ITEM_TYPE_GTP_PSC, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_ICMP, + RTE_FLOW_ITEM_TYPE_END, +}; + +/* ESP */ +enum rte_flow_item_type iavf_pattern_eth_ipv4_esp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_ESP, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_ipv4_udp_esp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_ESP, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_ipv6_esp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_ESP, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_ipv6_udp_esp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_ESP, + RTE_FLOW_ITEM_TYPE_END, +}; + +/* AH */ +enum rte_flow_item_type iavf_pattern_eth_ipv4_ah[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_AH, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_ipv6_ah[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_AH, + RTE_FLOW_ITEM_TYPE_END, +}; + +/* L2TPV3 */ +enum rte_flow_item_type iavf_pattern_eth_ipv4_l2tpv3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_L2TPV3OIP, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_ipv6_l2tpv3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_L2TPV3OIP, + RTE_FLOW_ITEM_TYPE_END, +}; + +/* PFCP */ +enum rte_flow_item_type iavf_pattern_eth_ipv4_pfcp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_PFCP, + RTE_FLOW_ITEM_TYPE_END, +}; + +enum rte_flow_item_type iavf_pattern_eth_ipv6_pfcp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_PFCP, + RTE_FLOW_ITEM_TYPE_END, +}; + +typedef struct iavf_flow_engine * (*parse_engine_t)(struct iavf_adapter *ad, + struct rte_flow *flow, + struct iavf_parser_list *parser_list, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error); + +void +iavf_register_flow_engine(struct iavf_flow_engine *engine) +{ + TAILQ_INSERT_TAIL(&engine_list, engine, node); +} + +int +iavf_flow_init(struct iavf_adapter *ad) +{ + int ret; + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad); + void *temp; + struct iavf_flow_engine *engine; + + TAILQ_INIT(&vf->flow_list); + TAILQ_INIT(&vf->rss_parser_list); + TAILQ_INIT(&vf->dist_parser_list); + rte_spinlock_init(&vf->flow_ops_lock); + + TAILQ_FOREACH_SAFE(engine, &engine_list, node, temp) { + if (engine->init == NULL) { + PMD_INIT_LOG(ERR, "Invalid engine type (%d)", + engine->type); + return -ENOTSUP; + } + + ret = engine->init(ad); + if (ret && ret != -ENOTSUP) { + PMD_INIT_LOG(ERR, "Failed to initialize engine %d", + engine->type); + return ret; + } + } + return 0; +} + +void +iavf_flow_uninit(struct iavf_adapter *ad) +{ + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad); + struct iavf_flow_engine *engine; + struct rte_flow *p_flow; + struct iavf_flow_parser_node *p_parser; + void *temp; + + TAILQ_FOREACH_SAFE(engine, &engine_list, node, temp) { + if (engine->uninit) + engine->uninit(ad); + } + + /* Remove all flows */ + while ((p_flow = TAILQ_FIRST(&vf->flow_list))) { + TAILQ_REMOVE(&vf->flow_list, p_flow, node); + if (p_flow->engine->free) + p_flow->engine->free(p_flow); + rte_free(p_flow); + } + + /* Cleanup parser list */ + while ((p_parser = TAILQ_FIRST(&vf->rss_parser_list))) { + TAILQ_REMOVE(&vf->rss_parser_list, p_parser, node); + rte_free(p_parser); + } + + while ((p_parser = TAILQ_FIRST(&vf->dist_parser_list))) { + TAILQ_REMOVE(&vf->dist_parser_list, p_parser, node); + rte_free(p_parser); + } +} + +int +iavf_register_parser(struct iavf_flow_parser *parser, + struct iavf_adapter *ad) +{ + struct iavf_parser_list *list = NULL; + struct iavf_flow_parser_node *parser_node; + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad); + + parser_node = rte_zmalloc("iavf_parser", sizeof(*parser_node), 0); + if (parser_node == NULL) { + PMD_DRV_LOG(ERR, "Failed to allocate memory."); + return -ENOMEM; + } + parser_node->parser = parser; + + if (parser->engine->type == IAVF_FLOW_ENGINE_HASH) { + list = &vf->rss_parser_list; + TAILQ_INSERT_TAIL(list, parser_node, node); + } else if (parser->engine->type == IAVF_FLOW_ENGINE_FDIR) { + list = &vf->dist_parser_list; + TAILQ_INSERT_HEAD(list, parser_node, node); + } else { + return -EINVAL; + } + + return 0; +} + +void +iavf_unregister_parser(struct iavf_flow_parser *parser, + struct iavf_adapter *ad) +{ + struct iavf_parser_list *list = NULL; + struct iavf_flow_parser_node *p_parser; + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad); + void *temp; + + if (parser->engine->type == IAVF_FLOW_ENGINE_HASH) + list = &vf->rss_parser_list; + else if (parser->engine->type == IAVF_FLOW_ENGINE_FDIR) + list = &vf->dist_parser_list; + + if (list == NULL) + return; + + TAILQ_FOREACH_SAFE(p_parser, list, node, temp) { + if (p_parser->parser->engine->type == parser->engine->type) { + TAILQ_REMOVE(list, p_parser, node); + rte_free(p_parser); + } + } +} + +static int +iavf_flow_valid_attr(const struct rte_flow_attr *attr, + struct rte_flow_error *error) +{ + /* Must be input direction */ + if (!attr->ingress) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, + attr, "Only support ingress."); + return -rte_errno; + } + + /* Not supported */ + if (attr->egress) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, + attr, "Not support egress."); + return -rte_errno; + } + + /* Not supported */ + if (attr->priority) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, + attr, "Not support priority."); + return -rte_errno; + } + + /* Not supported */ + if (attr->group) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_GROUP, + attr, "Not support group."); + return -rte_errno; + } + + return 0; +} + +/* Find the first VOID or non-VOID item pointer */ +static const struct rte_flow_item * +iavf_find_first_item(const struct rte_flow_item *item, bool is_void) +{ + bool is_find; + + while (item->type != RTE_FLOW_ITEM_TYPE_END) { + if (is_void) + is_find = item->type == RTE_FLOW_ITEM_TYPE_VOID; + else + is_find = item->type != RTE_FLOW_ITEM_TYPE_VOID; + if (is_find) + break; + item++; + } + return item; +} + +/* Skip all VOID items of the pattern */ +static void +iavf_pattern_skip_void_item(struct rte_flow_item *items, + const struct rte_flow_item *pattern) +{ + uint32_t cpy_count = 0; + const struct rte_flow_item *pb = pattern, *pe = pattern; + + for (;;) { + /* Find a non-void item first */ + pb = iavf_find_first_item(pb, false); + if (pb->type == RTE_FLOW_ITEM_TYPE_END) { + pe = pb; + break; + } + + /* Find a void item */ + pe = iavf_find_first_item(pb + 1, true); + + cpy_count = pe - pb; + rte_memcpy(items, pb, sizeof(struct rte_flow_item) * cpy_count); + + items += cpy_count; + + if (pe->type == RTE_FLOW_ITEM_TYPE_END) + break; + + pb = pe + 1; + } + /* Copy the END item. */ + rte_memcpy(items, pe, sizeof(struct rte_flow_item)); +} + +/* Check if the pattern matches a supported item type array */ +static bool +iavf_match_pattern(enum rte_flow_item_type *item_array, + const struct rte_flow_item *pattern) +{ + const struct rte_flow_item *item = pattern; + + while ((*item_array == item->type) && + (*item_array != RTE_FLOW_ITEM_TYPE_END)) { + item_array++; + item++; + } + + return (*item_array == RTE_FLOW_ITEM_TYPE_END && + item->type == RTE_FLOW_ITEM_TYPE_END); +} + +struct iavf_pattern_match_item * +iavf_search_pattern_match_item(const struct rte_flow_item pattern[], + struct iavf_pattern_match_item *array, + uint32_t array_len, + struct rte_flow_error *error) +{ + uint16_t i = 0; + struct iavf_pattern_match_item *pattern_match_item; + /* need free by each filter */ + struct rte_flow_item *items; /* used for pattern without VOID items */ + uint32_t item_num = 0; /* non-void item number */ + + /* Get the non-void item number of pattern */ + while ((pattern + i)->type != RTE_FLOW_ITEM_TYPE_END) { + if ((pattern + i)->type != RTE_FLOW_ITEM_TYPE_VOID) + item_num++; + i++; + } + item_num++; + + items = rte_zmalloc("iavf_pattern", + item_num * sizeof(struct rte_flow_item), 0); + if (!items) { + rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_ITEM_NUM, + NULL, "No memory for PMD internal items."); + return NULL; + } + pattern_match_item = rte_zmalloc("iavf_pattern_match_item", + sizeof(struct iavf_pattern_match_item), 0); + if (!pattern_match_item) { + rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "Failed to allocate memory."); + return NULL; + } + + iavf_pattern_skip_void_item(items, pattern); + + for (i = 0; i < array_len; i++) + if (iavf_match_pattern(array[i].pattern_list, + items)) { + pattern_match_item->input_set_mask = + array[i].input_set_mask; + pattern_match_item->pattern_list = + array[i].pattern_list; + pattern_match_item->meta = array[i].meta; + rte_free(items); + return pattern_match_item; + } + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, + pattern, "Unsupported pattern"); + + rte_free(items); + rte_free(pattern_match_item); + return NULL; +} + +static struct iavf_flow_engine * +iavf_parse_engine_create(struct iavf_adapter *ad, + struct rte_flow *flow, + struct iavf_parser_list *parser_list, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + struct iavf_flow_engine *engine = NULL; + struct iavf_flow_parser_node *parser_node; + void *temp; + void *meta = NULL; + + TAILQ_FOREACH_SAFE(parser_node, parser_list, node, temp) { + if (parser_node->parser->parse_pattern_action(ad, + parser_node->parser->array, + parser_node->parser->array_len, + pattern, actions, &meta, error) < 0) + continue; + + engine = parser_node->parser->engine; + + RTE_ASSERT(engine->create != NULL); + if (!(engine->create(ad, flow, meta, error))) + return engine; + } + return NULL; +} + +static struct iavf_flow_engine * +iavf_parse_engine_validate(struct iavf_adapter *ad, + struct rte_flow *flow, + struct iavf_parser_list *parser_list, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + struct iavf_flow_engine *engine = NULL; + struct iavf_flow_parser_node *parser_node; + void *temp; + void *meta = NULL; + + TAILQ_FOREACH_SAFE(parser_node, parser_list, node, temp) { + if (parser_node->parser->parse_pattern_action(ad, + parser_node->parser->array, + parser_node->parser->array_len, + pattern, actions, &meta, error) < 0) + continue; + + engine = parser_node->parser->engine; + if (engine->validation == NULL) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "Validation not support"); + continue; + } + + if (engine->validation(ad, flow, meta, error)) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "Validation failed"); + break; + } + } + return engine; +} + + +static int +iavf_flow_process_filter(struct rte_eth_dev *dev, + struct rte_flow *flow, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct iavf_flow_engine **engine, + parse_engine_t iavf_parse_engine, + struct rte_flow_error *error) +{ + int ret = IAVF_ERR_CONFIG; + struct iavf_adapter *ad = + IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad); + + if (!pattern) { + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_NUM, + NULL, "NULL pattern."); + return -rte_errno; + } + + if (!actions) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_NUM, + NULL, "NULL action."); + return -rte_errno; + } + + if (!attr) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR, + NULL, "NULL attribute."); + return -rte_errno; + } + + ret = iavf_flow_valid_attr(attr, error); + if (ret) + return ret; + + *engine = iavf_parse_engine(ad, flow, &vf->rss_parser_list, pattern, + actions, error); + if (*engine) + return 0; + + *engine = iavf_parse_engine(ad, flow, &vf->dist_parser_list, pattern, + actions, error); + + if (!*engine) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "Failed to create parser engine."); + return -rte_errno; + } + + return 0; +} + +static int +iavf_flow_validate(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + struct iavf_flow_engine *engine; + + return iavf_flow_process_filter(dev, NULL, attr, pattern, actions, + &engine, iavf_parse_engine_validate, error); +} + +static struct rte_flow * +iavf_flow_create(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + struct iavf_adapter *ad = + IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad); + struct iavf_flow_engine *engine = NULL; + struct rte_flow *flow = NULL; + int ret; + + flow = rte_zmalloc("iavf_flow", sizeof(struct rte_flow), 0); + if (!flow) { + rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "Failed to allocate memory"); + return flow; + } + + ret = iavf_flow_process_filter(dev, flow, attr, pattern, actions, + &engine, iavf_parse_engine_create, error); + if (ret < 0) { + PMD_DRV_LOG(ERR, "Failed to create flow"); + rte_free(flow); + flow = NULL; + goto free_flow; + } + + flow->engine = engine; + TAILQ_INSERT_TAIL(&vf->flow_list, flow, node); + PMD_DRV_LOG(INFO, "Succeeded to create (%d) flow", engine->type); + +free_flow: + rte_spinlock_unlock(&vf->flow_ops_lock); + return flow; +} + +static bool +iavf_flow_is_valid(struct rte_flow *flow) +{ + struct iavf_flow_engine *engine; + void *temp; + + if (flow && flow->engine) { + TAILQ_FOREACH_SAFE(engine, &engine_list, node, temp) { + if (engine == flow->engine) + return true; + } + } + + return false; +} + +static int +iavf_flow_destroy(struct rte_eth_dev *dev, + struct rte_flow *flow, + struct rte_flow_error *error) +{ + struct iavf_adapter *ad = + IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad); + int ret = 0; + + if (!iavf_flow_is_valid(flow) || !flow->engine->destroy) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "Invalid flow destroy"); + return -rte_errno; + } + + rte_spinlock_lock(&vf->flow_ops_lock); + + ret = flow->engine->destroy(ad, flow, error); + + if (!ret) { + TAILQ_REMOVE(&vf->flow_list, flow, node); + rte_free(flow); + } else { + PMD_DRV_LOG(ERR, "Failed to destroy flow"); + } + + rte_spinlock_unlock(&vf->flow_ops_lock); + + return ret; +} + +int +iavf_flow_flush(struct rte_eth_dev *dev, + struct rte_flow_error *error) +{ + struct iavf_adapter *ad = + IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad); + struct rte_flow *p_flow; + void *temp; + int ret = 0; + + TAILQ_FOREACH_SAFE(p_flow, &vf->flow_list, node, temp) { + ret = iavf_flow_destroy(dev, p_flow, error); + if (ret) { + PMD_DRV_LOG(ERR, "Failed to flush flows"); + return -EINVAL; + } + } + + return ret; +} + +static int +iavf_flow_query(struct rte_eth_dev *dev, + struct rte_flow *flow, + const struct rte_flow_action *actions, + void *data, + struct rte_flow_error *error) +{ + int ret = -EINVAL; + struct iavf_adapter *ad = + IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct rte_flow_query_count *count = data; + + if (!iavf_flow_is_valid(flow) || !flow->engine->query_count) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "Invalid flow query"); + return -rte_errno; + } + + for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { + switch (actions->type) { + case RTE_FLOW_ACTION_TYPE_VOID: + break; + case RTE_FLOW_ACTION_TYPE_COUNT: + ret = flow->engine->query_count(ad, flow, count, error); + break; + default: + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + actions, + "action not supported"); + } + } + return ret; +} + diff --git a/src/spdk/dpdk/drivers/net/iavf/iavf_generic_flow.h b/src/spdk/dpdk/drivers/net/iavf/iavf_generic_flow.h new file mode 100644 index 000000000..978d0716b --- /dev/null +++ b/src/spdk/dpdk/drivers/net/iavf/iavf_generic_flow.h @@ -0,0 +1,320 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2019 Intel Corporation + */ + +#ifndef _IAVF_GENERIC_FLOW_H_ +#define _IAVF_GENERIC_FLOW_H_ + +#include + +/* protocol */ + +#define IAVF_PROT_MAC_INNER (1ULL << 1) +#define IAVF_PROT_MAC_OUTER (1ULL << 2) +#define IAVF_PROT_VLAN_INNER (1ULL << 3) +#define IAVF_PROT_VLAN_OUTER (1ULL << 4) +#define IAVF_PROT_IPV4_INNER (1ULL << 5) +#define IAVF_PROT_IPV4_OUTER (1ULL << 6) +#define IAVF_PROT_IPV6_INNER (1ULL << 7) +#define IAVF_PROT_IPV6_OUTER (1ULL << 8) +#define IAVF_PROT_TCP_INNER (1ULL << 9) +#define IAVF_PROT_TCP_OUTER (1ULL << 10) +#define IAVF_PROT_UDP_INNER (1ULL << 11) +#define IAVF_PROT_UDP_OUTER (1ULL << 12) +#define IAVF_PROT_SCTP_INNER (1ULL << 13) +#define IAVF_PROT_SCTP_OUTER (1ULL << 14) +#define IAVF_PROT_ICMP4_INNER (1ULL << 15) +#define IAVF_PROT_ICMP4_OUTER (1ULL << 16) +#define IAVF_PROT_ICMP6_INNER (1ULL << 17) +#define IAVF_PROT_ICMP6_OUTER (1ULL << 18) +#define IAVF_PROT_VXLAN (1ULL << 19) +#define IAVF_PROT_NVGRE (1ULL << 20) +#define IAVF_PROT_GTPU (1ULL << 21) +#define IAVF_PROT_ESP (1ULL << 22) +#define IAVF_PROT_AH (1ULL << 23) +#define IAVF_PROT_L2TPV3OIP (1ULL << 24) +#define IAVF_PROT_PFCP (1ULL << 25) + + +/* field */ + +#define IAVF_SMAC (1ULL << 63) +#define IAVF_DMAC (1ULL << 62) +#define IAVF_ETHERTYPE (1ULL << 61) +#define IAVF_IP_SRC (1ULL << 60) +#define IAVF_IP_DST (1ULL << 59) +#define IAVF_IP_PROTO (1ULL << 58) +#define IAVF_IP_TTL (1ULL << 57) +#define IAVF_IP_TOS (1ULL << 56) +#define IAVF_SPORT (1ULL << 55) +#define IAVF_DPORT (1ULL << 54) +#define IAVF_ICMP_TYPE (1ULL << 53) +#define IAVF_ICMP_CODE (1ULL << 52) +#define IAVF_VXLAN_VNI (1ULL << 51) +#define IAVF_NVGRE_TNI (1ULL << 50) +#define IAVF_GTPU_TEID (1ULL << 49) +#define IAVF_GTPU_QFI (1ULL << 48) +#define IAVF_ESP_SPI (1ULL << 47) +#define IAVF_AH_SPI (1ULL << 46) +#define IAVF_L2TPV3OIP_SESSION_ID (1ULL << 45) +#define IAVF_PFCP_S_FIELD (1ULL << 44) +#define IAVF_PFCP_SEID (1ULL << 43) + +/* input set */ + +#define IAVF_INSET_NONE 0ULL + +/* non-tunnel */ + +#define IAVF_INSET_SMAC (IAVF_PROT_MAC_OUTER | IAVF_SMAC) +#define IAVF_INSET_DMAC (IAVF_PROT_MAC_OUTER | IAVF_DMAC) +#define IAVF_INSET_VLAN_INNER (IAVF_PROT_VLAN_INNER) +#define IAVF_INSET_VLAN_OUTER (IAVF_PROT_VLAN_OUTER) +#define IAVF_INSET_ETHERTYPE (IAVF_ETHERTYPE) + +#define IAVF_INSET_IPV4_SRC \ + (IAVF_PROT_IPV4_OUTER | IAVF_IP_SRC) +#define IAVF_INSET_IPV4_DST \ + (IAVF_PROT_IPV4_OUTER | IAVF_IP_DST) +#define IAVF_INSET_IPV4_TOS \ + (IAVF_PROT_IPV4_OUTER | IAVF_IP_TOS) +#define IAVF_INSET_IPV4_PROTO \ + (IAVF_PROT_IPV4_OUTER | IAVF_IP_PROTO) +#define IAVF_INSET_IPV4_TTL \ + (IAVF_PROT_IPV4_OUTER | IAVF_IP_TTL) +#define IAVF_INSET_IPV6_SRC \ + (IAVF_PROT_IPV6_OUTER | IAVF_IP_SRC) +#define IAVF_INSET_IPV6_DST \ + (IAVF_PROT_IPV6_OUTER | IAVF_IP_DST) +#define IAVF_INSET_IPV6_NEXT_HDR \ + (IAVF_PROT_IPV6_OUTER | IAVF_IP_PROTO) +#define IAVF_INSET_IPV6_HOP_LIMIT \ + (IAVF_PROT_IPV6_OUTER | IAVF_IP_TTL) +#define IAVF_INSET_IPV6_TC \ + (IAVF_PROT_IPV6_OUTER | IAVF_IP_TOS) + +#define IAVF_INSET_TCP_SRC_PORT \ + (IAVF_PROT_TCP_OUTER | IAVF_SPORT) +#define IAVF_INSET_TCP_DST_PORT \ + (IAVF_PROT_TCP_OUTER | IAVF_DPORT) +#define IAVF_INSET_UDP_SRC_PORT \ + (IAVF_PROT_UDP_OUTER | IAVF_SPORT) +#define IAVF_INSET_UDP_DST_PORT \ + (IAVF_PROT_UDP_OUTER | IAVF_DPORT) +#define IAVF_INSET_SCTP_SRC_PORT \ + (IAVF_PROT_SCTP_OUTER | IAVF_SPORT) +#define IAVF_INSET_SCTP_DST_PORT \ + (IAVF_PROT_SCTP_OUTER | IAVF_DPORT) +#define IAVF_INSET_ICMP4_SRC_PORT \ + (IAVF_PROT_ICMP4_OUTER | IAVF_SPORT) +#define IAVF_INSET_ICMP4_DST_PORT \ + (IAVF_PROT_ICMP4_OUTER | IAVF_DPORT) +#define IAVF_INSET_ICMP6_SRC_PORT \ + (IAVF_PROT_ICMP6_OUTER | IAVF_SPORT) +#define IAVF_INSET_ICMP6_DST_PORT \ + (IAVF_PROT_ICMP6_OUTER | IAVF_DPORT) +#define IAVF_INSET_ICMP4_TYPE \ + (IAVF_PROT_ICMP4_OUTER | IAVF_ICMP_TYPE) +#define IAVF_INSET_ICMP4_CODE \ + (IAVF_PROT_ICMP4_OUTER | IAVF_ICMP_CODE) +#define IAVF_INSET_ICMP6_TYPE \ + (IAVF_PROT_ICMP6_OUTER | IAVF_ICMP_TYPE) +#define IAVF_INSET_ICMP6_CODE \ + (IAVF_PROT_ICMP6_OUTER | IAVF_ICMP_CODE) +#define IAVF_INSET_GTPU_TEID \ + (IAVF_PROT_GTPU | IAVF_GTPU_TEID) +#define IAVF_INSET_GTPU_QFI \ + (IAVF_PROT_GTPU | IAVF_GTPU_QFI) +#define IAVF_INSET_ESP_SPI \ + (IAVF_PROT_ESP | IAVF_ESP_SPI) +#define IAVF_INSET_AH_SPI \ + (IAVF_PROT_AH | IAVF_AH_SPI) +#define IAVF_INSET_L2TPV3OIP_SESSION_ID \ + (IAVF_PROT_L2TPV3OIP | IAVF_L2TPV3OIP_SESSION_ID) +#define IAVF_INSET_PFCP_S_FIELD \ + (IAVF_PROT_PFCP | IAVF_PFCP_S_FIELD) +#define IAVF_INSET_PFCP_SEID \ + (IAVF_PROT_PFCP | IAVF_PFCP_S_FIELD | IAVF_PFCP_SEID) + + +/* empty pattern */ +extern enum rte_flow_item_type iavf_pattern_empty[]; + +/* L2 */ +extern enum rte_flow_item_type iavf_pattern_ethertype[]; +extern enum rte_flow_item_type iavf_pattern_ethertype_vlan[]; +extern enum rte_flow_item_type iavf_pattern_ethertype_qinq[]; + +/* ARP */ +extern enum rte_flow_item_type iavf_pattern_eth_arp[]; + +/* non-tunnel IPv4 */ +extern enum rte_flow_item_type iavf_pattern_eth_ipv4[]; +extern enum rte_flow_item_type iavf_pattern_eth_vlan_ipv4[]; +extern enum rte_flow_item_type iavf_pattern_eth_qinq_ipv4[]; +extern enum rte_flow_item_type iavf_pattern_eth_ipv4_udp[]; +extern enum rte_flow_item_type iavf_pattern_eth_vlan_ipv4_udp[]; +extern enum rte_flow_item_type iavf_pattern_eth_qinq_ipv4_udp[]; +extern enum rte_flow_item_type iavf_pattern_eth_ipv4_tcp[]; +extern enum rte_flow_item_type iavf_pattern_eth_vlan_ipv4_tcp[]; +extern enum rte_flow_item_type iavf_pattern_eth_qinq_ipv4_tcp[]; +extern enum rte_flow_item_type iavf_pattern_eth_ipv4_sctp[]; +extern enum rte_flow_item_type iavf_pattern_eth_vlan_ipv4_sctp[]; +extern enum rte_flow_item_type iavf_pattern_eth_qinq_ipv4_sctp[]; +extern enum rte_flow_item_type iavf_pattern_eth_ipv4_icmp[]; +extern enum rte_flow_item_type iavf_pattern_eth_vlan_ipv4_icmp[]; +extern enum rte_flow_item_type iavf_pattern_eth_qinq_ipv4_icmp[]; + +/* non-tunnel IPv6 */ +extern enum rte_flow_item_type iavf_pattern_eth_ipv6[]; +extern enum rte_flow_item_type iavf_pattern_eth_vlan_ipv6[]; +extern enum rte_flow_item_type iavf_pattern_eth_qinq_ipv6[]; +extern enum rte_flow_item_type iavf_pattern_eth_ipv6_udp[]; +extern enum rte_flow_item_type iavf_pattern_eth_vlan_ipv6_udp[]; +extern enum rte_flow_item_type iavf_pattern_eth_qinq_ipv6_udp[]; +extern enum rte_flow_item_type iavf_pattern_eth_ipv6_tcp[]; +extern enum rte_flow_item_type iavf_pattern_eth_vlan_ipv6_tcp[]; +extern enum rte_flow_item_type iavf_pattern_eth_qinq_ipv6_tcp[]; +extern enum rte_flow_item_type iavf_pattern_eth_ipv6_sctp[]; +extern enum rte_flow_item_type iavf_pattern_eth_vlan_ipv6_sctp[]; +extern enum rte_flow_item_type iavf_pattern_eth_qinq_ipv6_sctp[]; +extern enum rte_flow_item_type iavf_pattern_eth_ipv6_icmp6[]; +extern enum rte_flow_item_type iavf_pattern_eth_vlan_ipv6_icmp6[]; +extern enum rte_flow_item_type iavf_pattern_eth_qinq_ipv6_icmp6[]; + +/* GTPU */ +extern enum rte_flow_item_type iavf_pattern_eth_ipv4_gtpu[]; +extern enum rte_flow_item_type iavf_pattern_eth_ipv4_gtpu_ipv4[]; +extern enum rte_flow_item_type iavf_pattern_eth_ipv4_gtpu_eh[]; +extern enum rte_flow_item_type iavf_pattern_eth_ipv4_gtpu_eh_ipv4[]; +extern enum rte_flow_item_type iavf_pattern_eth_ipv4_gtpu_eh_ipv4_udp[]; +extern enum rte_flow_item_type iavf_pattern_eth_ipv4_gtpu_eh_ipv4_tcp[]; +extern enum rte_flow_item_type iavf_pattern_eth_ipv4_gtpu_eh_ipv4_icmp[]; + +/* ESP */ +extern enum rte_flow_item_type iavf_pattern_eth_ipv4_esp[]; +extern enum rte_flow_item_type iavf_pattern_eth_ipv4_udp_esp[]; +extern enum rte_flow_item_type iavf_pattern_eth_ipv6_esp[]; +extern enum rte_flow_item_type iavf_pattern_eth_ipv6_udp_esp[]; + +/* AH */ +extern enum rte_flow_item_type iavf_pattern_eth_ipv4_ah[]; +extern enum rte_flow_item_type iavf_pattern_eth_ipv6_ah[]; + +/* L2TPV3 */ +extern enum rte_flow_item_type iavf_pattern_eth_ipv4_l2tpv3[]; +extern enum rte_flow_item_type iavf_pattern_eth_ipv6_l2tpv3[]; + +/* PFCP */ +extern enum rte_flow_item_type iavf_pattern_eth_ipv4_pfcp[]; +extern enum rte_flow_item_type iavf_pattern_eth_ipv6_pfcp[]; + + +extern const struct rte_flow_ops iavf_flow_ops; + +/* pattern structure */ +struct iavf_pattern_match_item { + enum rte_flow_item_type *pattern_list; + /* pattern_list must end with RTE_FLOW_ITEM_TYPE_END */ + uint64_t input_set_mask; + void *meta; +}; + +typedef int (*engine_init_t)(struct iavf_adapter *ad); +typedef void (*engine_uninit_t)(struct iavf_adapter *ad); +typedef int (*engine_validation_t)(struct iavf_adapter *ad, + struct rte_flow *flow, + void *meta, + struct rte_flow_error *error); +typedef int (*engine_create_t)(struct iavf_adapter *ad, + struct rte_flow *flow, + void *meta, + struct rte_flow_error *error); +typedef int (*engine_destroy_t)(struct iavf_adapter *ad, + struct rte_flow *flow, + struct rte_flow_error *error); +typedef int (*engine_query_t)(struct iavf_adapter *ad, + struct rte_flow *flow, + struct rte_flow_query_count *count, + struct rte_flow_error *error); +typedef void (*engine_free_t) (struct rte_flow *flow); +typedef int (*parse_pattern_action_t)(struct iavf_adapter *ad, + struct iavf_pattern_match_item *array, + uint32_t array_len, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + void **meta, + struct rte_flow_error *error); + +/* engine types. */ +enum iavf_flow_engine_type { + IAVF_FLOW_ENGINE_NONE = 0, + IAVF_FLOW_ENGINE_FDIR, + IAVF_FLOW_ENGINE_HASH, + IAVF_FLOW_ENGINE_MAX, +}; + +/** + * classification stages. + * for non-pipeline mode, we have two classification stages: Distributor/RSS + * for pipeline-mode we have three classification stages: + * Permission/Distributor/RSS + */ +enum iavf_flow_classification_stage { + IAVF_FLOW_STAGE_NONE = 0, + IAVF_FLOW_STAGE_RSS, + IAVF_FLOW_STAGE_DISTRIBUTOR, + IAVF_FLOW_STAGE_MAX, +}; + +/* Struct to store engine created. */ +struct iavf_flow_engine { + TAILQ_ENTRY(iavf_flow_engine) node; + engine_init_t init; + engine_uninit_t uninit; + engine_validation_t validation; + engine_create_t create; + engine_destroy_t destroy; + engine_query_t query_count; + engine_free_t free; + enum iavf_flow_engine_type type; +}; + +TAILQ_HEAD(iavf_engine_list, iavf_flow_engine); + +/* Struct to store flow created. */ +struct rte_flow { + TAILQ_ENTRY(rte_flow) node; + struct iavf_flow_engine *engine; + void *rule; +}; + +struct iavf_flow_parser { + struct iavf_flow_engine *engine; + struct iavf_pattern_match_item *array; + uint32_t array_len; + parse_pattern_action_t parse_pattern_action; + enum iavf_flow_classification_stage stage; +}; + +/* Struct to store parser created. */ +struct iavf_flow_parser_node { + TAILQ_ENTRY(iavf_flow_parser_node) node; + struct iavf_flow_parser *parser; +}; + +void iavf_register_flow_engine(struct iavf_flow_engine *engine); +int iavf_flow_init(struct iavf_adapter *ad); +void iavf_flow_uninit(struct iavf_adapter *ad); +int iavf_flow_flush(struct rte_eth_dev *dev, + struct rte_flow_error *error); +int iavf_register_parser(struct iavf_flow_parser *parser, + struct iavf_adapter *ad); +void iavf_unregister_parser(struct iavf_flow_parser *parser, + struct iavf_adapter *ad); +struct iavf_pattern_match_item * +iavf_search_pattern_match_item(const struct rte_flow_item pattern[], + struct iavf_pattern_match_item *array, + uint32_t array_len, + struct rte_flow_error *error); +#endif diff --git a/src/spdk/dpdk/drivers/net/iavf/iavf_hash.c b/src/spdk/dpdk/drivers/net/iavf/iavf_hash.c new file mode 100644 index 000000000..af528863b --- /dev/null +++ b/src/spdk/dpdk/drivers/net/iavf/iavf_hash.c @@ -0,0 +1,1236 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2020 Intel Corporation + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "iavf_log.h" +#include "iavf.h" +#include "iavf_generic_flow.h" + +enum iavf_pattern_hint_type { + IAVF_PATTERN_HINT_NONE, + IAVF_PATTERN_HINT_IPV4, + IAVF_PATTERN_HINT_IPV4_UDP, + IAVF_PATTERN_HINT_IPV4_TCP, + IAVF_PATTERN_HINT_IPV4_SCTP, + IAVF_PATTERN_HINT_IPV6, + IAVF_PATTERN_HINT_IPV6_UDP, + IAVF_PATTERN_HINT_IPV6_TCP, + IAVF_PATTERN_HINT_IPV6_SCTP, +}; + +struct iavf_pattern_match_type { + enum iavf_pattern_hint_type phint_type; +}; + +struct iavf_hash_match_type { + enum iavf_pattern_hint_type phint_type; + uint64_t hash_type; + struct virtchnl_proto_hdrs *proto_hdrs; +}; + +struct iavf_rss_meta { + struct virtchnl_proto_hdrs *proto_hdrs; + enum virtchnl_rss_algorithm rss_algorithm; +}; + +struct iavf_hash_flow_cfg { + struct virtchnl_rss_cfg *rss_cfg; + bool simple_xor; +}; + +static int +iavf_hash_init(struct iavf_adapter *ad); +static int +iavf_hash_create(struct iavf_adapter *ad, struct rte_flow *flow, void *meta, + struct rte_flow_error *error); +static int +iavf_hash_destroy(struct iavf_adapter *ad, struct rte_flow *flow, + struct rte_flow_error *error); +static void +iavf_hash_uninit(struct iavf_adapter *ad); +static void +iavf_hash_free(struct rte_flow *flow); +static int +iavf_hash_parse_pattern_action(struct iavf_adapter *ad, + struct iavf_pattern_match_item *array, + uint32_t array_len, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + void **meta, + struct rte_flow_error *error); + +struct iavf_pattern_match_type phint_empty = { + IAVF_PATTERN_HINT_NONE}; +struct iavf_pattern_match_type phint_eth_ipv4 = { + IAVF_PATTERN_HINT_IPV4}; +struct iavf_pattern_match_type phint_eth_ipv4_udp = { + IAVF_PATTERN_HINT_IPV4_UDP}; +struct iavf_pattern_match_type phint_eth_ipv4_tcp = { + IAVF_PATTERN_HINT_IPV4_TCP}; +struct iavf_pattern_match_type phint_eth_ipv4_sctp = { + IAVF_PATTERN_HINT_IPV4_SCTP}; +struct iavf_pattern_match_type phint_eth_ipv4_gtpu_eh = { + IAVF_PATTERN_HINT_IPV4_UDP}; +struct iavf_pattern_match_type phint_eth_ipv4_esp = { + IAVF_PATTERN_HINT_IPV4}; +struct iavf_pattern_match_type phint_eth_ipv4_ah = { + IAVF_PATTERN_HINT_IPV4}; +struct iavf_pattern_match_type phint_eth_ipv4_l2tpv3 = { + IAVF_PATTERN_HINT_IPV4}; +struct iavf_pattern_match_type phint_eth_ipv4_pfcp = { + IAVF_PATTERN_HINT_IPV4_UDP}; +struct iavf_pattern_match_type phint_eth_ipv6 = { + IAVF_PATTERN_HINT_IPV6}; +struct iavf_pattern_match_type phint_eth_ipv6_udp = { + IAVF_PATTERN_HINT_IPV6_UDP}; +struct iavf_pattern_match_type phint_eth_ipv6_tcp = { + IAVF_PATTERN_HINT_IPV6_TCP}; +struct iavf_pattern_match_type phint_eth_ipv6_sctp = { + IAVF_PATTERN_HINT_IPV6_SCTP}; +struct iavf_pattern_match_type phint_eth_ipv6_esp = { + IAVF_PATTERN_HINT_IPV6}; +struct iavf_pattern_match_type phint_eth_ipv6_ah = { + IAVF_PATTERN_HINT_IPV6}; +struct iavf_pattern_match_type phint_eth_ipv6_l2tpv3 = { + IAVF_PATTERN_HINT_IPV6}; +struct iavf_pattern_match_type phint_eth_ipv6_pfcp = { + IAVF_PATTERN_HINT_IPV6_UDP}; + +/** + * Supported pattern for hash. + * The first member is pattern item type, + * the second member is input set mask, + * the third member is pattern hint for hash. + */ +static struct iavf_pattern_match_item iavf_hash_pattern_list[] = { + {iavf_pattern_eth_ipv4, IAVF_INSET_NONE, &phint_eth_ipv4}, + {iavf_pattern_eth_ipv4_udp, IAVF_INSET_NONE, &phint_eth_ipv4_udp}, + {iavf_pattern_eth_ipv4_tcp, IAVF_INSET_NONE, &phint_eth_ipv4_tcp}, + {iavf_pattern_eth_ipv4_sctp, IAVF_INSET_NONE, &phint_eth_ipv4_sctp}, + {iavf_pattern_eth_ipv6, IAVF_INSET_NONE, &phint_eth_ipv6}, + {iavf_pattern_eth_ipv4_gtpu_eh_ipv4, IAVF_INSET_NONE, + &phint_eth_ipv4_gtpu_eh}, + {iavf_pattern_eth_ipv4_gtpu_eh_ipv4_udp, IAVF_INSET_NONE, + &phint_eth_ipv4_gtpu_eh}, + {iavf_pattern_eth_ipv4_gtpu_eh_ipv4_tcp, IAVF_INSET_NONE, + &phint_eth_ipv4_gtpu_eh}, + {iavf_pattern_eth_ipv4_esp, IAVF_INSET_NONE, &phint_eth_ipv4_esp}, + {iavf_pattern_eth_ipv4_ah, IAVF_INSET_NONE, &phint_eth_ipv4_ah}, + {iavf_pattern_eth_ipv4_l2tpv3, IAVF_INSET_NONE, + &phint_eth_ipv4_l2tpv3}, + {iavf_pattern_eth_ipv4_pfcp, IAVF_INSET_NONE, &phint_eth_ipv4_pfcp}, + {iavf_pattern_eth_ipv6_udp, IAVF_INSET_NONE, &phint_eth_ipv6_udp}, + {iavf_pattern_eth_ipv6_tcp, IAVF_INSET_NONE, &phint_eth_ipv6_tcp}, + {iavf_pattern_eth_ipv6_sctp, IAVF_INSET_NONE, &phint_eth_ipv6_sctp}, + {iavf_pattern_eth_ipv6_esp, IAVF_INSET_NONE, &phint_eth_ipv6_esp}, + {iavf_pattern_eth_ipv6_ah, IAVF_INSET_NONE, &phint_eth_ipv6_ah}, + {iavf_pattern_eth_ipv6_l2tpv3, IAVF_INSET_NONE, + &phint_eth_ipv6_l2tpv3}, + {iavf_pattern_eth_ipv6_pfcp, IAVF_INSET_NONE, &phint_eth_ipv6_pfcp}, + {iavf_pattern_empty, IAVF_INSET_NONE, &phint_empty}, +}; + +#define GTP_EH_PDU_LINK_UP 1 +#define GTP_EH_PDU_LINK_DWN 0 + +#define TUNNEL_LEVEL_OUTER 0 +#define TUNNEL_LEVEL_FIRST_INNER 1 + +#define PROTO_COUNT_ONE 1 +#define PROTO_COUNT_TWO 2 +#define PROTO_COUNT_THREE 3 + +#define BUFF_NOUSED 0 +#define FIELD_FOR_PROTO_ONLY 0 + +#define proto_hint_eth_src { \ + VIRTCHNL_PROTO_HDR_ETH, VIRTCHNL_PROTO_HDR_ETH_SRC, {BUFF_NOUSED } } + +#define proto_hint_eth_dst { \ + VIRTCHNL_PROTO_HDR_ETH, VIRTCHNL_PROTO_HDR_ETH_DST, {BUFF_NOUSED } } + +#define proto_hint_eth_only { \ + VIRTCHNL_PROTO_HDR_ETH, FIELD_FOR_PROTO_ONLY, {BUFF_NOUSED } } + +#define proto_hint_eth { \ + VIRTCHNL_PROTO_HDR_ETH, \ + VIRTCHNL_PROTO_HDR_ETH_SRC | VIRTCHNL_PROTO_HDR_ETH_DST, \ + {BUFF_NOUSED } } + +#define proto_hint_svlan { \ + VIRTCHNL_PROTO_HDR_S_VLAN, VIRTCHNL_PROTO_HDR_S_VLAN_ID, \ + {BUFF_NOUSED } } + +#define proto_hint_cvlan { \ + VIRTCHNL_PROTO_HDR_C_VLAN, VIRTCHNL_PROTO_HDR_C_VLAN_ID, \ + {BUFF_NOUSED } } + +#define proto_hint_ipv4_src { \ + VIRTCHNL_PROTO_HDR_IPV4, VIRTCHNL_PROTO_HDR_IPV4_SRC, {BUFF_NOUSED } } + +#define proto_hint_ipv4_dst { \ + VIRTCHNL_PROTO_HDR_IPV4, VIRTCHNL_PROTO_HDR_IPV4_DST, {BUFF_NOUSED } } + +#define proto_hint_ipv4_only { \ + VIRTCHNL_PROTO_HDR_IPV4, FIELD_FOR_PROTO_ONLY, {BUFF_NOUSED } } + +#define proto_hint_ipv4 { \ + VIRTCHNL_PROTO_HDR_IPV4, \ + VIRTCHNL_PROTO_HDR_IPV4_SRC | VIRTCHNL_PROTO_HDR_IPV4_DST, \ + {BUFF_NOUSED } } + +#define proto_hint_udp_src_port { \ + VIRTCHNL_PROTO_HDR_UDP, VIRTCHNL_PROTO_HDR_UDP_SRC_PORT, \ + {BUFF_NOUSED } } + +#define proto_hint_udp_dst_port { \ + VIRTCHNL_PROTO_HDR_UDP, VIRTCHNL_PROTO_HDR_UDP_DST_PORT, \ + {BUFF_NOUSED } } + +#define proto_hint_udp_only { \ + VIRTCHNL_PROTO_HDR_UDP, FIELD_FOR_PROTO_ONLY, {BUFF_NOUSED } } + +#define proto_hint_udp { \ + VIRTCHNL_PROTO_HDR_UDP, \ + VIRTCHNL_PROTO_HDR_UDP_SRC_PORT | VIRTCHNL_PROTO_HDR_UDP_DST_PORT, \ + {BUFF_NOUSED } } + +#define proto_hint_tcp_src_port { \ + VIRTCHNL_PROTO_HDR_TCP, VIRTCHNL_PROTO_HDR_TCP_SRC_PORT, \ + {BUFF_NOUSED } } + +#define proto_hint_tcp_dst_port { \ + VIRTCHNL_PROTO_HDR_TCP, VIRTCHNL_PROTO_HDR_TCP_DST_PORT, \ + {BUFF_NOUSED } } + +#define proto_hint_tcp_only { \ + VIRTCHNL_PROTO_HDR_TCP, FIELD_FOR_PROTO_ONLY, {BUFF_NOUSED } } + +#define proto_hint_tcp { \ + VIRTCHNL_PROTO_HDR_TCP, \ + VIRTCHNL_PROTO_HDR_TCP_SRC_PORT | VIRTCHNL_PROTO_HDR_TCP_DST_PORT, \ + {BUFF_NOUSED } } + +#define proto_hint_sctp_src_port { \ + VIRTCHNL_PROTO_HDR_SCTP, VIRTCHNL_PROTO_HDR_SCTP_SRC_PORT, \ + {BUFF_NOUSED } } + +#define proto_hint_sctp_dst_port { \ + VIRTCHNL_PROTO_HDR_SCTP, VIRTCHNL_PROTO_HDR_SCTP_DST_PORT, \ + {BUFF_NOUSED } } + +#define proto_hint_sctp_only { \ + VIRTCHNL_PROTO_HDR_SCTP, FIELD_FOR_PROTO_ONLY, {BUFF_NOUSED } } + +#define proto_hint_sctp { \ + VIRTCHNL_PROTO_HDR_SCTP, \ + VIRTCHNL_PROTO_HDR_SCTP_SRC_PORT | VIRTCHNL_PROTO_HDR_SCTP_DST_PORT, \ + {BUFF_NOUSED } } + +#define proto_hint_ipv6_src { \ + VIRTCHNL_PROTO_HDR_IPV6, VIRTCHNL_PROTO_HDR_IPV6_SRC, {BUFF_NOUSED } } + +#define proto_hint_ipv6_dst { \ + VIRTCHNL_PROTO_HDR_IPV6, VIRTCHNL_PROTO_HDR_IPV6_DST, {BUFF_NOUSED } } + +#define proto_hint_ipv6_only { \ + VIRTCHNL_PROTO_HDR_IPV6, FIELD_FOR_PROTO_ONLY, {BUFF_NOUSED } } + +#define proto_hint_ipv6 { \ + VIRTCHNL_PROTO_HDR_IPV6, \ + VIRTCHNL_PROTO_HDR_IPV6_SRC | VIRTCHNL_PROTO_HDR_IPV6_DST, \ + {BUFF_NOUSED } } + +#define proto_hint_gtpu_up_only { \ + VIRTCHNL_PROTO_HDR_GTPU_EH_PDU_UP, \ + FIELD_FOR_PROTO_ONLY, {BUFF_NOUSED } } + +#define proto_hint_gtpu_dwn_only { \ + VIRTCHNL_PROTO_HDR_GTPU_EH_PDU_DWN, \ + FIELD_FOR_PROTO_ONLY, {BUFF_NOUSED } } + +#define proto_hint_esp { \ + VIRTCHNL_PROTO_HDR_ESP, \ + VIRTCHNL_PROTO_HDR_ESP_SPI, {BUFF_NOUSED } } + +#define proto_hint_ah { \ + VIRTCHNL_PROTO_HDR_AH, \ + VIRTCHNL_PROTO_HDR_AH_SPI, {BUFF_NOUSED } } + +#define proto_hint_l2tpv3 { \ + VIRTCHNL_PROTO_HDR_L2TPV3, \ + VIRTCHNL_PROTO_HDR_L2TPV3_SESS_ID, {BUFF_NOUSED } } + +#define proto_hint_pfcp { \ + VIRTCHNL_PROTO_HDR_PFCP, VIRTCHNL_PROTO_HDR_PFCP_SEID, {BUFF_NOUSED } } + +struct virtchnl_proto_hdrs hdrs_hint_eth_src = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_ONE, {proto_hint_eth_src } +}; + +struct virtchnl_proto_hdrs hdrs_hint_eth_dst = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_ONE, {proto_hint_eth_dst } +}; + +struct virtchnl_proto_hdrs hdrs_hint_eth = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_ONE, {proto_hint_eth } +}; + +struct virtchnl_proto_hdrs hdrs_hint_svlan = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_ONE, {proto_hint_svlan } +}; + +struct virtchnl_proto_hdrs hdrs_hint_cvlan = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_ONE, {proto_hint_cvlan } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv4_src = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_ONE, {proto_hint_ipv4_src } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv4_dst = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_ONE, {proto_hint_ipv4_dst } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv4_src_gtpu_up = { + TUNNEL_LEVEL_FIRST_INNER, PROTO_COUNT_TWO, {proto_hint_gtpu_up_only, + proto_hint_ipv4_src } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv4_dst_gtpu_dwn = { + TUNNEL_LEVEL_FIRST_INNER, PROTO_COUNT_TWO, {proto_hint_gtpu_dwn_only, + proto_hint_ipv4_dst } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv4_esp = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv4_only, + proto_hint_esp } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv4_ah = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv4_only, + proto_hint_ah } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv4_l2tpv3 = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv4_only, + proto_hint_l2tpv3 } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv4_pfcp = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv4_only, + proto_hint_pfcp } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv4 = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_ONE, {proto_hint_ipv4 } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv4_src_udp_src_port = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv4_src, + proto_hint_udp_src_port } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv4_src_udp_dst_port = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv4_src, + proto_hint_udp_dst_port } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv4_dst_udp_src_port = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv4_dst, + proto_hint_udp_src_port } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv4_dst_udp_dst_port = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv4_dst, + proto_hint_udp_dst_port } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv4_udp_src_port = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv4_only, + proto_hint_udp_src_port } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv4_udp_dst_port = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv4_only, + proto_hint_udp_dst_port } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv4_udp = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv4, + proto_hint_udp } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv4_src_tcp_src_port = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv4_src, + proto_hint_tcp_src_port } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv4_src_tcp_dst_port = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv4_src, + proto_hint_tcp_dst_port } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv4_dst_tcp_src_port = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv4_dst, + proto_hint_tcp_src_port } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv4_dst_tcp_dst_port = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv4_dst, + proto_hint_tcp_dst_port } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv4_tcp_src_port = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv4_only, + proto_hint_tcp_src_port } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv4_tcp_dst_port = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv4_only, + proto_hint_tcp_dst_port } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv4_tcp = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv4, + proto_hint_tcp } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv4_src_sctp_src_port = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv4_src, + proto_hint_sctp_src_port } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv4_src_sctp_dst_port = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv4_src, + proto_hint_sctp_dst_port } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv4_dst_sctp_src_port = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv4_dst, + proto_hint_sctp_src_port } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv4_dst_sctp_dst_port = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv4_dst, + proto_hint_sctp_dst_port } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv4_sctp_src_port = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv4_only, + proto_hint_sctp_src_port } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv4_sctp_dst_port = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv4_only, + proto_hint_sctp_dst_port } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv4_sctp = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv4, + proto_hint_sctp } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv6_src = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_ONE, {proto_hint_ipv6_src } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv6_dst = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_ONE, {proto_hint_ipv6_dst } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv6_esp = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv6_only, + proto_hint_esp } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv6_ah = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv6_only, + proto_hint_ah } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv6_l2tpv3 = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv6_only, + proto_hint_l2tpv3 } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv6_pfcp = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv6_only, + proto_hint_pfcp } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv6 = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_ONE, {proto_hint_ipv6 } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv6_src_udp_src_port = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv6_src, + proto_hint_udp_src_port } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv6_src_udp_dst_port = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv6_src, + proto_hint_udp_dst_port } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv6_dst_udp_src_port = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv6_dst, + proto_hint_udp_src_port } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv6_dst_udp_dst_port = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv6_dst, + proto_hint_udp_dst_port } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv6_udp_src_port = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_udp_only, + proto_hint_udp_src_port } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv6_udp_dst_port = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_udp_only, + proto_hint_udp_dst_port } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv6_udp = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv6, + proto_hint_udp } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv6_src_tcp_src_port = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv6_src, + proto_hint_tcp_src_port } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv6_src_tcp_dst_port = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv6_src, + proto_hint_tcp_dst_port } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv6_dst_tcp_src_port = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv6_dst, + proto_hint_tcp_src_port } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv6_dst_tcp_dst_port = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv6_dst, + proto_hint_tcp_dst_port } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv6_tcp_src_port = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_udp_only, + proto_hint_tcp_src_port } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv6_tcp_dst_port = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_udp_only, + proto_hint_tcp_dst_port } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv6_tcp = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv6, + proto_hint_tcp } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv6_src_sctp_src_port = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv6_src, + proto_hint_sctp_src_port } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv6_src_sctp_dst_port = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv6_src, + proto_hint_sctp_dst_port } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv6_dst_sctp_src_port = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv6_dst, + proto_hint_sctp_src_port } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv6_dst_sctp_dst_port = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv6_dst, + proto_hint_sctp_dst_port } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv6_sctp_src_port = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_udp_only, + proto_hint_sctp_src_port } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv6_sctp_dst_port = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_udp_only, + proto_hint_sctp_dst_port } +}; + +struct virtchnl_proto_hdrs hdrs_hint_ipv6_sctp = { + TUNNEL_LEVEL_OUTER, PROTO_COUNT_TWO, {proto_hint_ipv6, + proto_hint_sctp } +}; + +/** + * The first member is pattern hint type, + * the second member is hash type, + * the third member is virtchnl protocol hdrs. + */ +struct iavf_hash_match_type iavf_hash_type_list[] = { + /* IPV4 */ + {IAVF_PATTERN_HINT_IPV4, ETH_RSS_L2_SRC_ONLY, &hdrs_hint_eth_src}, + {IAVF_PATTERN_HINT_IPV4, ETH_RSS_L2_DST_ONLY, &hdrs_hint_eth_dst}, + {IAVF_PATTERN_HINT_IPV4, ETH_RSS_ETH | ETH_RSS_L2_SRC_ONLY, + &hdrs_hint_eth_src}, + {IAVF_PATTERN_HINT_IPV4, ETH_RSS_ETH | ETH_RSS_L2_DST_ONLY, + &hdrs_hint_eth_dst}, + {IAVF_PATTERN_HINT_IPV4, ETH_RSS_ETH, &hdrs_hint_eth}, + {IAVF_PATTERN_HINT_IPV4, ETH_RSS_S_VLAN, &hdrs_hint_svlan}, + {IAVF_PATTERN_HINT_IPV4, ETH_RSS_C_VLAN, &hdrs_hint_cvlan}, + {IAVF_PATTERN_HINT_IPV4, ETH_RSS_L3_SRC_ONLY, &hdrs_hint_ipv4_src}, + {IAVF_PATTERN_HINT_IPV4, ETH_RSS_L3_DST_ONLY, &hdrs_hint_ipv4_dst}, + {IAVF_PATTERN_HINT_IPV4, ETH_RSS_IPV4 | ETH_RSS_L3_SRC_ONLY, + &hdrs_hint_ipv4_src}, + {IAVF_PATTERN_HINT_IPV4, ETH_RSS_IPV4 | ETH_RSS_L3_DST_ONLY, + &hdrs_hint_ipv4_dst}, + {IAVF_PATTERN_HINT_IPV4, ETH_RSS_ESP, &hdrs_hint_ipv4_esp}, + {IAVF_PATTERN_HINT_IPV4, ETH_RSS_AH, &hdrs_hint_ipv4_ah}, + {IAVF_PATTERN_HINT_IPV4, ETH_RSS_L2TPV3, &hdrs_hint_ipv4_l2tpv3}, + {IAVF_PATTERN_HINT_IPV4, ETH_RSS_IPV4, &hdrs_hint_ipv4}, + /* IPV4 UDP */ + {IAVF_PATTERN_HINT_IPV4_UDP, ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_SRC_ONLY, + &hdrs_hint_ipv4_src_udp_src_port}, + {IAVF_PATTERN_HINT_IPV4_UDP, ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_DST_ONLY, + &hdrs_hint_ipv4_src_udp_dst_port}, + {IAVF_PATTERN_HINT_IPV4_UDP, ETH_RSS_L3_SRC_ONLY | ETH_RSS_GTPU, + &hdrs_hint_ipv4_src_gtpu_up}, + {IAVF_PATTERN_HINT_IPV4_UDP, ETH_RSS_L3_SRC_ONLY, + &hdrs_hint_ipv4_src}, + {IAVF_PATTERN_HINT_IPV4_UDP, ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY, + &hdrs_hint_ipv4_dst_udp_src_port}, + {IAVF_PATTERN_HINT_IPV4_UDP, ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_DST_ONLY, + &hdrs_hint_ipv4_dst_udp_dst_port}, + {IAVF_PATTERN_HINT_IPV4_UDP, ETH_RSS_L3_DST_ONLY | ETH_RSS_GTPU, + &hdrs_hint_ipv4_dst_gtpu_dwn}, + {IAVF_PATTERN_HINT_IPV4_UDP, ETH_RSS_L3_DST_ONLY, + &hdrs_hint_ipv4_dst}, + {IAVF_PATTERN_HINT_IPV4_UDP, ETH_RSS_NONFRAG_IPV4_UDP | + ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_SRC_ONLY, + &hdrs_hint_ipv4_src_udp_src_port}, + {IAVF_PATTERN_HINT_IPV4_UDP, ETH_RSS_NONFRAG_IPV4_UDP | + ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_DST_ONLY, + &hdrs_hint_ipv4_src_udp_dst_port}, + {IAVF_PATTERN_HINT_IPV4_UDP, ETH_RSS_NONFRAG_IPV4_UDP | + ETH_RSS_L3_SRC_ONLY | ETH_RSS_GTPU, + &hdrs_hint_ipv4_src_gtpu_up}, + {IAVF_PATTERN_HINT_IPV4_UDP, ETH_RSS_NONFRAG_IPV4_UDP | + ETH_RSS_L3_SRC_ONLY, &hdrs_hint_ipv4_src}, + {IAVF_PATTERN_HINT_IPV4_UDP, ETH_RSS_NONFRAG_IPV4_UDP | + ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY, + &hdrs_hint_ipv4_dst_udp_src_port}, + {IAVF_PATTERN_HINT_IPV4_UDP, ETH_RSS_NONFRAG_IPV4_UDP | + ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_DST_ONLY, + &hdrs_hint_ipv4_dst_udp_dst_port}, + {IAVF_PATTERN_HINT_IPV4_UDP, ETH_RSS_NONFRAG_IPV4_UDP | + ETH_RSS_L3_DST_ONLY | ETH_RSS_GTPU, + &hdrs_hint_ipv4_dst_gtpu_dwn}, + {IAVF_PATTERN_HINT_IPV4_UDP, ETH_RSS_NONFRAG_IPV4_UDP | + ETH_RSS_L3_DST_ONLY, &hdrs_hint_ipv4_dst}, + {IAVF_PATTERN_HINT_IPV4_UDP, ETH_RSS_L4_SRC_ONLY, + &hdrs_hint_ipv4_udp_src_port}, + {IAVF_PATTERN_HINT_IPV4_UDP, ETH_RSS_L4_DST_ONLY, + &hdrs_hint_ipv4_udp_dst_port}, + {IAVF_PATTERN_HINT_IPV4_UDP, ETH_RSS_PFCP, + &hdrs_hint_ipv4_pfcp}, + {IAVF_PATTERN_HINT_IPV4_UDP, ETH_RSS_NONFRAG_IPV4_UDP, + &hdrs_hint_ipv4_udp}, + /* IPV4 TCP */ + {IAVF_PATTERN_HINT_IPV4_TCP, ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_SRC_ONLY, + &hdrs_hint_ipv4_src_tcp_src_port}, + {IAVF_PATTERN_HINT_IPV4_TCP, ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_DST_ONLY, + &hdrs_hint_ipv4_src_tcp_dst_port}, + {IAVF_PATTERN_HINT_IPV4_TCP, ETH_RSS_L3_SRC_ONLY, + &hdrs_hint_ipv4_src}, + {IAVF_PATTERN_HINT_IPV4_TCP, ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY, + &hdrs_hint_ipv4_dst_tcp_src_port}, + {IAVF_PATTERN_HINT_IPV4_TCP, ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_DST_ONLY, + &hdrs_hint_ipv4_dst_tcp_dst_port}, + {IAVF_PATTERN_HINT_IPV4_TCP, ETH_RSS_L3_DST_ONLY, + &hdrs_hint_ipv4_dst}, + {IAVF_PATTERN_HINT_IPV4_TCP, ETH_RSS_NONFRAG_IPV4_TCP | + ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_SRC_ONLY, + &hdrs_hint_ipv4_src_tcp_src_port}, + {IAVF_PATTERN_HINT_IPV4_TCP, ETH_RSS_NONFRAG_IPV4_TCP | + ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_DST_ONLY, + &hdrs_hint_ipv4_src_tcp_dst_port}, + {IAVF_PATTERN_HINT_IPV4_TCP, ETH_RSS_NONFRAG_IPV4_TCP | + ETH_RSS_L3_SRC_ONLY, &hdrs_hint_ipv4_src}, + {IAVF_PATTERN_HINT_IPV4_TCP, ETH_RSS_NONFRAG_IPV4_TCP | + ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY, + &hdrs_hint_ipv4_dst_tcp_src_port}, + {IAVF_PATTERN_HINT_IPV4_TCP, ETH_RSS_NONFRAG_IPV4_TCP | + ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_DST_ONLY, + &hdrs_hint_ipv4_dst_tcp_dst_port}, + {IAVF_PATTERN_HINT_IPV4_TCP, ETH_RSS_NONFRAG_IPV4_TCP | + ETH_RSS_L3_DST_ONLY, &hdrs_hint_ipv4_dst}, + {IAVF_PATTERN_HINT_IPV4_TCP, ETH_RSS_L4_SRC_ONLY, + &hdrs_hint_ipv4_tcp_src_port}, + {IAVF_PATTERN_HINT_IPV4_TCP, ETH_RSS_L4_DST_ONLY, + &hdrs_hint_ipv4_tcp_dst_port}, + {IAVF_PATTERN_HINT_IPV4_TCP, ETH_RSS_NONFRAG_IPV4_TCP, + &hdrs_hint_ipv4_tcp}, + /* IPV4 SCTP */ + {IAVF_PATTERN_HINT_IPV4_SCTP, ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_SRC_ONLY, + &hdrs_hint_ipv4_src_sctp_src_port}, + {IAVF_PATTERN_HINT_IPV4_SCTP, ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_DST_ONLY, + &hdrs_hint_ipv4_src_sctp_dst_port}, + {IAVF_PATTERN_HINT_IPV4_SCTP, ETH_RSS_L3_SRC_ONLY, + &hdrs_hint_ipv4_src}, + {IAVF_PATTERN_HINT_IPV4_SCTP, ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY, + &hdrs_hint_ipv4_dst_sctp_src_port}, + {IAVF_PATTERN_HINT_IPV4_SCTP, ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_DST_ONLY, + &hdrs_hint_ipv4_dst_sctp_dst_port}, + {IAVF_PATTERN_HINT_IPV4_SCTP, ETH_RSS_L3_DST_ONLY, + &hdrs_hint_ipv4_dst}, + {IAVF_PATTERN_HINT_IPV4_SCTP, ETH_RSS_NONFRAG_IPV4_SCTP | + ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_SRC_ONLY, + &hdrs_hint_ipv4_src_sctp_src_port}, + {IAVF_PATTERN_HINT_IPV4_SCTP, ETH_RSS_NONFRAG_IPV4_SCTP | + ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_DST_ONLY, + &hdrs_hint_ipv4_src_sctp_dst_port}, + {IAVF_PATTERN_HINT_IPV4_SCTP, ETH_RSS_NONFRAG_IPV4_SCTP | + ETH_RSS_L3_SRC_ONLY, &hdrs_hint_ipv4_src}, + {IAVF_PATTERN_HINT_IPV4_SCTP, ETH_RSS_NONFRAG_IPV4_SCTP | + ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY, + &hdrs_hint_ipv4_dst_sctp_src_port}, + {IAVF_PATTERN_HINT_IPV4_SCTP, ETH_RSS_NONFRAG_IPV4_SCTP | + ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_DST_ONLY, + &hdrs_hint_ipv4_dst_sctp_dst_port}, + {IAVF_PATTERN_HINT_IPV4_SCTP, ETH_RSS_NONFRAG_IPV4_SCTP | + ETH_RSS_L3_DST_ONLY, &hdrs_hint_ipv4_dst}, + {IAVF_PATTERN_HINT_IPV4_SCTP, ETH_RSS_L4_SRC_ONLY, + &hdrs_hint_ipv4_sctp_src_port}, + {IAVF_PATTERN_HINT_IPV4_SCTP, ETH_RSS_L4_DST_ONLY, + &hdrs_hint_ipv4_sctp_dst_port}, + {IAVF_PATTERN_HINT_IPV4_SCTP, ETH_RSS_NONFRAG_IPV4_SCTP, + &hdrs_hint_ipv4_sctp}, + /* IPV6 */ + {IAVF_PATTERN_HINT_IPV6, ETH_RSS_L2_SRC_ONLY, &hdrs_hint_eth_src}, + {IAVF_PATTERN_HINT_IPV6, ETH_RSS_L2_DST_ONLY, &hdrs_hint_eth_dst}, + {IAVF_PATTERN_HINT_IPV6, ETH_RSS_IPV6 | ETH_RSS_L2_SRC_ONLY, + &hdrs_hint_eth_src}, + {IAVF_PATTERN_HINT_IPV6, ETH_RSS_IPV6 | ETH_RSS_L2_DST_ONLY, + &hdrs_hint_eth_dst}, + {IAVF_PATTERN_HINT_IPV6, ETH_RSS_ETH, &hdrs_hint_eth}, + {IAVF_PATTERN_HINT_IPV6, ETH_RSS_S_VLAN, &hdrs_hint_svlan}, + {IAVF_PATTERN_HINT_IPV4, ETH_RSS_C_VLAN, &hdrs_hint_cvlan}, + {IAVF_PATTERN_HINT_IPV6, ETH_RSS_L3_SRC_ONLY, &hdrs_hint_ipv6_src}, + {IAVF_PATTERN_HINT_IPV6, ETH_RSS_L3_DST_ONLY, &hdrs_hint_ipv6_dst}, + {IAVF_PATTERN_HINT_IPV6, ETH_RSS_IPV6 | ETH_RSS_L3_SRC_ONLY, + &hdrs_hint_ipv6_src}, + {IAVF_PATTERN_HINT_IPV6, ETH_RSS_IPV6 | ETH_RSS_L3_DST_ONLY, + &hdrs_hint_ipv6_dst}, + {IAVF_PATTERN_HINT_IPV6, ETH_RSS_ESP, &hdrs_hint_ipv6_esp}, + {IAVF_PATTERN_HINT_IPV6, ETH_RSS_AH, &hdrs_hint_ipv6_ah}, + {IAVF_PATTERN_HINT_IPV6, ETH_RSS_L2TPV3, &hdrs_hint_ipv6_l2tpv3}, + {IAVF_PATTERN_HINT_IPV6, ETH_RSS_IPV6, &hdrs_hint_ipv6}, + /* IPV6 UDP */ + {IAVF_PATTERN_HINT_IPV6_UDP, ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_SRC_ONLY, + &hdrs_hint_ipv6_src_udp_src_port}, + {IAVF_PATTERN_HINT_IPV6_UDP, ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_DST_ONLY, + &hdrs_hint_ipv6_src_udp_dst_port}, + {IAVF_PATTERN_HINT_IPV6_UDP, ETH_RSS_L3_SRC_ONLY, + &hdrs_hint_ipv6_src}, + {IAVF_PATTERN_HINT_IPV6_UDP, ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY, + &hdrs_hint_ipv6_dst_udp_src_port}, + {IAVF_PATTERN_HINT_IPV6_UDP, ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_DST_ONLY, + &hdrs_hint_ipv6_dst_udp_dst_port}, + {IAVF_PATTERN_HINT_IPV6_UDP, ETH_RSS_L3_DST_ONLY, + &hdrs_hint_ipv6_dst}, + {IAVF_PATTERN_HINT_IPV6_UDP, ETH_RSS_NONFRAG_IPV6_UDP | + ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_SRC_ONLY, + &hdrs_hint_ipv6_src_udp_src_port}, + {IAVF_PATTERN_HINT_IPV6_UDP, ETH_RSS_NONFRAG_IPV6_UDP | + ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_DST_ONLY, + &hdrs_hint_ipv6_src_udp_dst_port}, + {IAVF_PATTERN_HINT_IPV6_UDP, ETH_RSS_NONFRAG_IPV6_UDP | + ETH_RSS_L3_SRC_ONLY, &hdrs_hint_ipv6_src}, + {IAVF_PATTERN_HINT_IPV6_UDP, ETH_RSS_NONFRAG_IPV6_UDP | + ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY, + &hdrs_hint_ipv6_dst_udp_src_port}, + {IAVF_PATTERN_HINT_IPV6_UDP, ETH_RSS_NONFRAG_IPV6_UDP | + ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_DST_ONLY, + &hdrs_hint_ipv6_dst_udp_dst_port}, + {IAVF_PATTERN_HINT_IPV6_UDP, ETH_RSS_NONFRAG_IPV6_UDP | + ETH_RSS_L3_DST_ONLY, &hdrs_hint_ipv6_dst}, + {IAVF_PATTERN_HINT_IPV6_UDP, ETH_RSS_L4_SRC_ONLY, + &hdrs_hint_ipv6_udp_src_port}, + {IAVF_PATTERN_HINT_IPV6_UDP, ETH_RSS_L4_DST_ONLY, + &hdrs_hint_ipv6_udp_dst_port}, + {IAVF_PATTERN_HINT_IPV6_UDP, ETH_RSS_PFCP, + &hdrs_hint_ipv6_pfcp}, + {IAVF_PATTERN_HINT_IPV6_UDP, ETH_RSS_NONFRAG_IPV6_UDP, + &hdrs_hint_ipv6_udp}, + /* IPV6 TCP */ + {IAVF_PATTERN_HINT_IPV6_TCP, ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_SRC_ONLY, + &hdrs_hint_ipv6_src_tcp_src_port}, + {IAVF_PATTERN_HINT_IPV6_TCP, ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_DST_ONLY, + &hdrs_hint_ipv6_src_tcp_dst_port}, + {IAVF_PATTERN_HINT_IPV6_TCP, ETH_RSS_L3_SRC_ONLY, + &hdrs_hint_ipv6_src}, + {IAVF_PATTERN_HINT_IPV6_TCP, ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY, + &hdrs_hint_ipv6_dst_tcp_src_port}, + {IAVF_PATTERN_HINT_IPV6_TCP, ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_DST_ONLY, + &hdrs_hint_ipv6_dst_tcp_dst_port}, + {IAVF_PATTERN_HINT_IPV6_TCP, ETH_RSS_L3_DST_ONLY, + &hdrs_hint_ipv6_dst}, + {IAVF_PATTERN_HINT_IPV6_TCP, ETH_RSS_NONFRAG_IPV6_TCP | + ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_SRC_ONLY, + &hdrs_hint_ipv6_src_tcp_src_port}, + {IAVF_PATTERN_HINT_IPV6_TCP, ETH_RSS_NONFRAG_IPV6_TCP | + ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_DST_ONLY, + &hdrs_hint_ipv6_src_tcp_dst_port}, + {IAVF_PATTERN_HINT_IPV6_TCP, ETH_RSS_NONFRAG_IPV6_TCP | + ETH_RSS_L3_SRC_ONLY, &hdrs_hint_ipv6_src}, + {IAVF_PATTERN_HINT_IPV6_TCP, ETH_RSS_NONFRAG_IPV6_TCP | + ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY, + &hdrs_hint_ipv6_dst_tcp_src_port}, + {IAVF_PATTERN_HINT_IPV6_TCP, ETH_RSS_NONFRAG_IPV6_TCP | + ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_DST_ONLY, + &hdrs_hint_ipv6_dst_tcp_dst_port}, + {IAVF_PATTERN_HINT_IPV6_TCP, ETH_RSS_NONFRAG_IPV6_TCP | + ETH_RSS_L3_DST_ONLY, &hdrs_hint_ipv6_dst}, + {IAVF_PATTERN_HINT_IPV6_TCP, ETH_RSS_L4_SRC_ONLY, + &hdrs_hint_ipv6_tcp_src_port}, + {IAVF_PATTERN_HINT_IPV6_TCP, ETH_RSS_L4_DST_ONLY, + &hdrs_hint_ipv6_tcp_dst_port}, + {IAVF_PATTERN_HINT_IPV6_TCP, ETH_RSS_NONFRAG_IPV6_TCP, + &hdrs_hint_ipv6_tcp}, + /* IPV6 SCTP */ + {IAVF_PATTERN_HINT_IPV6_SCTP, ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_SRC_ONLY, + &hdrs_hint_ipv6_src_sctp_src_port}, + {IAVF_PATTERN_HINT_IPV6_SCTP, ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_DST_ONLY, + &hdrs_hint_ipv6_src_sctp_dst_port}, + {IAVF_PATTERN_HINT_IPV6_SCTP, ETH_RSS_L3_SRC_ONLY, + &hdrs_hint_ipv6_src}, + {IAVF_PATTERN_HINT_IPV6_SCTP, ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY, + &hdrs_hint_ipv6_dst_sctp_src_port}, + {IAVF_PATTERN_HINT_IPV6_SCTP, ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_DST_ONLY, + &hdrs_hint_ipv6_dst_sctp_dst_port}, + {IAVF_PATTERN_HINT_IPV6_SCTP, ETH_RSS_L3_DST_ONLY, + &hdrs_hint_ipv6_dst}, + {IAVF_PATTERN_HINT_IPV6_SCTP, ETH_RSS_NONFRAG_IPV6_SCTP | + ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_SRC_ONLY, + &hdrs_hint_ipv6_src_sctp_src_port}, + {IAVF_PATTERN_HINT_IPV6_SCTP, ETH_RSS_NONFRAG_IPV6_SCTP | + ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_DST_ONLY, + &hdrs_hint_ipv6_src_sctp_dst_port}, + {IAVF_PATTERN_HINT_IPV6_SCTP, ETH_RSS_NONFRAG_IPV6_SCTP | + ETH_RSS_L3_SRC_ONLY, &hdrs_hint_ipv6_src}, + {IAVF_PATTERN_HINT_IPV6_SCTP, ETH_RSS_NONFRAG_IPV6_SCTP | + ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY, + &hdrs_hint_ipv6_dst_sctp_src_port}, + {IAVF_PATTERN_HINT_IPV6_SCTP, ETH_RSS_NONFRAG_IPV6_SCTP | + ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_DST_ONLY, + &hdrs_hint_ipv6_dst_sctp_dst_port}, + {IAVF_PATTERN_HINT_IPV6_SCTP, ETH_RSS_NONFRAG_IPV6_SCTP | + ETH_RSS_L3_DST_ONLY, &hdrs_hint_ipv6_dst}, + {IAVF_PATTERN_HINT_IPV6_SCTP, ETH_RSS_L4_SRC_ONLY, + &hdrs_hint_ipv6_sctp_src_port}, + {IAVF_PATTERN_HINT_IPV6_SCTP, ETH_RSS_L4_DST_ONLY, + &hdrs_hint_ipv6_sctp_dst_port}, + {IAVF_PATTERN_HINT_IPV6_SCTP, ETH_RSS_NONFRAG_IPV6_SCTP, + &hdrs_hint_ipv6_sctp}, +}; + +struct virtchnl_proto_hdrs *iavf_hash_default_hdrs[] = { + &hdrs_hint_ipv4, + &hdrs_hint_ipv4_udp, + &hdrs_hint_ipv4_tcp, + &hdrs_hint_ipv4_sctp, + &hdrs_hint_ipv6, + &hdrs_hint_ipv6_udp, + &hdrs_hint_ipv6_tcp, + &hdrs_hint_ipv6_sctp, +}; + +static struct iavf_flow_engine iavf_hash_engine = { + .init = iavf_hash_init, + .create = iavf_hash_create, + .destroy = iavf_hash_destroy, + .uninit = iavf_hash_uninit, + .free = iavf_hash_free, + .type = IAVF_FLOW_ENGINE_HASH, +}; + +/* Register parser for comms package. */ +static struct iavf_flow_parser iavf_hash_parser = { + .engine = &iavf_hash_engine, + .array = iavf_hash_pattern_list, + .array_len = RTE_DIM(iavf_hash_pattern_list), + .parse_pattern_action = iavf_hash_parse_pattern_action, + .stage = IAVF_FLOW_STAGE_RSS, +}; + +static int +iavf_hash_default_set(struct iavf_adapter *ad, bool add) +{ + struct virtchnl_rss_cfg *rss_cfg; + uint16_t i; + int ret; + + rss_cfg = rte_zmalloc("iavf rss rule", + sizeof(struct virtchnl_rss_cfg), 0); + if (!rss_cfg) + return -ENOMEM; + + for (i = 0; i < RTE_DIM(iavf_hash_default_hdrs); i++) { + rss_cfg->proto_hdrs = *iavf_hash_default_hdrs[i]; + rss_cfg->rss_algorithm = VIRTCHNL_RSS_ALG_TOEPLITZ_ASYMMETRIC; + + ret = iavf_add_del_rss_cfg(ad, rss_cfg, add); + if (ret) { + PMD_DRV_LOG(ERR, "fail to %s RSS configure", + add ? "add" : "delete"); + rte_free(rss_cfg); + return ret; + } + } + + return ret; +} + +RTE_INIT(iavf_hash_engine_init) +{ + struct iavf_flow_engine *engine = &iavf_hash_engine; + + iavf_register_flow_engine(engine); +} + +static int +iavf_hash_init(struct iavf_adapter *ad) +{ + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad); + struct iavf_flow_parser *parser; + int ret; + + if (!vf->vf_res) + return -EINVAL; + + if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF)) + return -ENOTSUP; + + parser = &iavf_hash_parser; + + ret = iavf_register_parser(parser, ad); + if (ret) { + PMD_DRV_LOG(ERR, "fail to register hash parser"); + return ret; + } + + ret = iavf_hash_default_set(ad, true); + if (ret) { + PMD_DRV_LOG(ERR, "fail to set default RSS"); + iavf_unregister_parser(parser, ad); + } + + return ret; +} + +static int +iavf_hash_check_inset(const struct rte_flow_item pattern[], + struct rte_flow_error *error) +{ + const struct rte_flow_item *item = pattern; + + for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "Not support range"); + return -rte_errno; + } + } + + return 0; +} + +static uint64_t +iavf_hash_refine_type(uint64_t rss_type, const struct rte_flow_item pattern[]) +{ + const struct rte_flow_item *item; + + for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { + if (item->type == RTE_FLOW_ITEM_TYPE_GTP_PSC) { + const struct rte_flow_item_gtp_psc *psc = item->spec; + + if (psc && (psc->pdu_type == GTP_EH_PDU_LINK_UP || + psc->pdu_type == GTP_EH_PDU_LINK_DWN)) { + rss_type |= ETH_RSS_GTPU; + } + } + } + + return rss_type; +} + +static int +iavf_hash_parse_action(struct iavf_pattern_match_item *pattern_match_item, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + void **meta, struct rte_flow_error *error) +{ + struct iavf_rss_meta *rss_meta = (struct iavf_rss_meta *)*meta; + uint32_t type_list_len = RTE_DIM(iavf_hash_type_list); + struct iavf_hash_match_type *type_match_item; + enum rte_flow_action_type action_type; + const struct rte_flow_action_rss *rss; + const struct rte_flow_action *action; + bool item_found = false; + uint64_t rss_type; + uint16_t i; + + struct iavf_pattern_match_type *tt = (struct iavf_pattern_match_type *) + (pattern_match_item->meta); + + /* Supported action is RSS. */ + for (action = actions; action->type != + RTE_FLOW_ACTION_TYPE_END; action++) { + action_type = action->type; + switch (action_type) { + case RTE_FLOW_ACTION_TYPE_RSS: + rss = action->conf; + rss_type = rss->types; + + /** + * Check simultaneous use of SRC_ONLY and DST_ONLY + * of the same level. + */ + rss_type = rte_eth_rss_hf_refine(rss_type); + + /** + * Refine the hash type base on some specific item of + * the pattern, such as identify the gtpu hash. + */ + rss_type = iavf_hash_refine_type(rss_type, pattern); + + /* Check if pattern is empty. */ + if (pattern_match_item->pattern_list != + iavf_pattern_empty && rss->func == + RTE_ETH_HASH_FUNCTION_SIMPLE_XOR) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "Not supported flow"); + + if (rss->level) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "a nonzero RSS encapsulation level is not supported"); + + if (rss->key_len) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "a nonzero RSS key_len is not supported"); + + if (rss->queue_num) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "a non-NULL RSS queue is not supported"); + + /* Check hash function and save it to rss_meta. */ + if (rss->func == RTE_ETH_HASH_FUNCTION_SIMPLE_XOR) + rss_meta->rss_algorithm = + VIRTCHNL_RSS_ALG_XOR_ASYMMETRIC; + else if (rss->func == + RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) + rss_meta->rss_algorithm = + VIRTCHNL_RSS_ALG_TOEPLITZ_SYMMETRIC; + else + rss_meta->rss_algorithm = + VIRTCHNL_RSS_ALG_TOEPLITZ_ASYMMETRIC; + + type_match_item = + rte_zmalloc("iavf_type_match_item", + sizeof(struct iavf_hash_match_type), 0); + if (!type_match_item) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, + "No memory for type_match_item"); + return -ENOMEM; + } + + /* Find matched proto hdrs according to hash type. */ + for (i = 0; i < type_list_len; i++) { + struct iavf_hash_match_type *ht_map = + &iavf_hash_type_list[i]; + if (rss_type == ht_map->hash_type && + tt->phint_type == ht_map->phint_type) { + type_match_item->hash_type = + ht_map->hash_type; + type_match_item->proto_hdrs = + ht_map->proto_hdrs; + rss_meta->proto_hdrs = + type_match_item->proto_hdrs; + item_found = true; + } + } + + rte_free(type_match_item); + + if (!item_found) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "Not supported flow"); + break; + + case RTE_FLOW_ACTION_TYPE_END: + break; + + default: + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "Invalid action."); + return -rte_errno; + } + } + + return 0; +} + +static int +iavf_hash_parse_pattern_action(__rte_unused struct iavf_adapter *ad, + struct iavf_pattern_match_item *array, + uint32_t array_len, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + void **meta, + struct rte_flow_error *error) +{ + struct iavf_pattern_match_item *pattern_match_item; + struct iavf_rss_meta *rss_meta_ptr; + int ret = 0; + + rss_meta_ptr = rte_zmalloc(NULL, sizeof(*rss_meta_ptr), 0); + if (!rss_meta_ptr) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "No memory for rss_meta_ptr"); + return -ENOMEM; + } + + /* Check rss supported pattern and find matched pattern. */ + pattern_match_item = + iavf_search_pattern_match_item(pattern, array, array_len, + error); + if (!pattern_match_item) { + ret = -rte_errno; + goto error; + } + + ret = iavf_hash_check_inset(pattern, error); + if (ret) + goto error; + + /* Check rss action. */ + ret = iavf_hash_parse_action(pattern_match_item, pattern, actions, + (void **)&rss_meta_ptr, error); + +error: + if (!ret && meta) + *meta = rss_meta_ptr; + else + rte_free(rss_meta_ptr); + + rte_free(pattern_match_item); + + return ret; +} + +static int +iavf_hash_create(__rte_unused struct iavf_adapter *ad, + __rte_unused struct rte_flow *flow, void *meta, + __rte_unused struct rte_flow_error *error) +{ + struct iavf_rss_meta *rss_meta = (struct iavf_rss_meta *)meta; + struct virtchnl_rss_cfg *rss_cfg; + int ret = 0; + + rss_cfg = rte_zmalloc("iavf rss rule", + sizeof(struct virtchnl_rss_cfg), 0); + if (!rss_cfg) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "No memory for rss rule"); + return -ENOMEM; + } + + rss_cfg->proto_hdrs = *rss_meta->proto_hdrs; + rss_cfg->rss_algorithm = rss_meta->rss_algorithm; + + ret = iavf_add_del_rss_cfg(ad, rss_cfg, true); + if (!ret) { + flow->rule = rss_cfg; + } else { + PMD_DRV_LOG(ERR, "fail to add RSS configure"); + rte_flow_error_set(error, -ret, + RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "Failed to add rss rule."); + rte_free(rss_cfg); + return -rte_errno; + } + + rte_free(meta); + + return ret; +} + +static int +iavf_hash_destroy(__rte_unused struct iavf_adapter *ad, + struct rte_flow *flow, + __rte_unused struct rte_flow_error *error) +{ + struct virtchnl_rss_cfg *rss_cfg; + int ret = 0; + + rss_cfg = (struct virtchnl_rss_cfg *)flow->rule; + + ret = iavf_add_del_rss_cfg(ad, rss_cfg, false); + if (ret) { + PMD_DRV_LOG(ERR, "fail to del RSS configure"); + rte_flow_error_set(error, -ret, + RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "Failed to delete rss rule."); + return -rte_errno; + } + return ret; +} + +static void +iavf_hash_uninit(struct iavf_adapter *ad) +{ + if (iavf_hash_default_set(ad, false)) + PMD_DRV_LOG(ERR, "fail to delete default RSS"); + + iavf_unregister_parser(&iavf_hash_parser, ad); +} + +static void +iavf_hash_free(struct rte_flow *flow) +{ + rte_free(flow->rule); +} diff --git a/src/spdk/dpdk/drivers/net/iavf/iavf_log.h b/src/spdk/dpdk/drivers/net/iavf/iavf_log.h new file mode 100644 index 000000000..1088ec75f --- /dev/null +++ b/src/spdk/dpdk/drivers/net/iavf/iavf_log.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017 Intel Corporation + */ + +#ifndef _IAVF_LOG_H_ +#define _IAVF_LOG_H_ + +extern int iavf_logtype_init; +#define PMD_INIT_LOG(level, fmt, args...) \ + rte_log(RTE_LOG_ ## level, iavf_logtype_init, "%s(): " fmt "\n", \ + __func__, ## args) +#define PMD_INIT_FUNC_TRACE() PMD_INIT_LOG(DEBUG, " >>") + +extern int iavf_logtype_driver; +#define PMD_DRV_LOG_RAW(level, fmt, args...) \ + rte_log(RTE_LOG_ ## level, iavf_logtype_driver, "%s(): " fmt, \ + __func__, ## args) + +#define PMD_DRV_LOG(level, fmt, args...) \ + PMD_DRV_LOG_RAW(level, fmt "\n", ## args) +#define PMD_DRV_FUNC_TRACE() PMD_DRV_LOG(DEBUG, " >>") + + +#ifdef RTE_LIBRTE_IAVF_DEBUG_RX +extern int iavf_logtype_rx; +#define PMD_RX_LOG(level, fmt, args...) \ + rte_log(RTE_LOG_ ## level, iavf_logtype_rx, \ + "%s(): " fmt "\n", __func__, ## args) +#else +#define PMD_RX_LOG(level, fmt, args...) do { } while (0) +#endif + +#ifdef RTE_LIBRTE_IAVF_DEBUG_TX +extern int iavf_logtype_tx; +#define PMD_TX_LOG(level, fmt, args...) \ + rte_log(RTE_LOG_ ## level, iavf_logtype_tx, \ + "%s(): " fmt "\n", __func__, ## args) +#else +#define PMD_TX_LOG(level, fmt, args...) do { } while (0) +#endif + +#ifdef RTE_LIBRTE_IAVF_DEBUG_TX_FREE +extern int iavf_logtype_tx_free; +#define PMD_TX_FREE_LOG(level, fmt, args...) \ + rte_log(RTE_LOG_ ## level, iavf_logtype_tx_free, \ + "%s(): " fmt "\n", __func__, ## args) +#else +#define PMD_TX_FREE_LOG(level, fmt, args...) do { } while (0) +#endif + +#endif /* _IAVF_LOG_H_ */ diff --git a/src/spdk/dpdk/drivers/net/iavf/iavf_rxtx.c b/src/spdk/dpdk/drivers/net/iavf/iavf_rxtx.c new file mode 100644 index 000000000..05a7dd898 --- /dev/null +++ b/src/spdk/dpdk/drivers/net/iavf/iavf_rxtx.c @@ -0,0 +1,2869 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017 Intel Corporation + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "iavf.h" +#include "iavf_rxtx.h" + +static inline int +check_rx_thresh(uint16_t nb_desc, uint16_t thresh) +{ + /* The following constraints must be satisfied: + * thresh < rxq->nb_rx_desc + */ + if (thresh >= nb_desc) { + PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be less than %u", + thresh, nb_desc); + return -EINVAL; + } + return 0; +} + +static inline int +check_tx_thresh(uint16_t nb_desc, uint16_t tx_rs_thresh, + uint16_t tx_free_thresh) +{ + /* TX descriptors will have their RS bit set after tx_rs_thresh + * descriptors have been used. The TX descriptor ring will be cleaned + * after tx_free_thresh descriptors are used or if the number of + * descriptors required to transmit a packet is greater than the + * number of free TX descriptors. + * + * The following constraints must be satisfied: + * - tx_rs_thresh must be less than the size of the ring minus 2. + * - tx_free_thresh must be less than the size of the ring minus 3. + * - tx_rs_thresh must be less than or equal to tx_free_thresh. + * - tx_rs_thresh must be a divisor of the ring size. + * + * One descriptor in the TX ring is used as a sentinel to avoid a H/W + * race condition, hence the maximum threshold constraints. When set + * to zero use default values. + */ + if (tx_rs_thresh >= (nb_desc - 2)) { + PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be less than the " + "number of TX descriptors (%u) minus 2", + tx_rs_thresh, nb_desc); + return -EINVAL; + } + if (tx_free_thresh >= (nb_desc - 3)) { + PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be less than the " + "number of TX descriptors (%u) minus 3.", + tx_free_thresh, nb_desc); + return -EINVAL; + } + if (tx_rs_thresh > tx_free_thresh) { + PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be less than or " + "equal to tx_free_thresh (%u).", + tx_rs_thresh, tx_free_thresh); + return -EINVAL; + } + if ((nb_desc % tx_rs_thresh) != 0) { + PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be a divisor of the " + "number of TX descriptors (%u).", + tx_rs_thresh, nb_desc); + return -EINVAL; + } + + return 0; +} + +static inline bool +check_rx_vec_allow(struct iavf_rx_queue *rxq) +{ + if (rxq->rx_free_thresh >= IAVF_VPMD_RX_MAX_BURST && + rxq->nb_rx_desc % rxq->rx_free_thresh == 0) { + PMD_INIT_LOG(DEBUG, "Vector Rx can be enabled on this rxq."); + return true; + } + + PMD_INIT_LOG(DEBUG, "Vector Rx cannot be enabled on this rxq."); + return false; +} + +static inline bool +check_tx_vec_allow(struct iavf_tx_queue *txq) +{ + if (!(txq->offloads & IAVF_NO_VECTOR_FLAGS) && + txq->rs_thresh >= IAVF_VPMD_TX_MAX_BURST && + txq->rs_thresh <= IAVF_VPMD_TX_MAX_FREE_BUF) { + PMD_INIT_LOG(DEBUG, "Vector tx can be enabled on this txq."); + return true; + } + PMD_INIT_LOG(DEBUG, "Vector Tx cannot be enabled on this txq."); + return false; +} + +static inline bool +check_rx_bulk_allow(struct iavf_rx_queue *rxq) +{ + int ret = true; + + if (!(rxq->rx_free_thresh >= IAVF_RX_MAX_BURST)) { + PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: " + "rxq->rx_free_thresh=%d, " + "IAVF_RX_MAX_BURST=%d", + rxq->rx_free_thresh, IAVF_RX_MAX_BURST); + ret = false; + } else if (rxq->nb_rx_desc % rxq->rx_free_thresh != 0) { + PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: " + "rxq->nb_rx_desc=%d, " + "rxq->rx_free_thresh=%d", + rxq->nb_rx_desc, rxq->rx_free_thresh); + ret = false; + } + return ret; +} + +static inline void +reset_rx_queue(struct iavf_rx_queue *rxq) +{ + uint16_t len; + uint32_t i; + + if (!rxq) + return; + + len = rxq->nb_rx_desc + IAVF_RX_MAX_BURST; + + for (i = 0; i < len * sizeof(union iavf_rx_desc); i++) + ((volatile char *)rxq->rx_ring)[i] = 0; + + memset(&rxq->fake_mbuf, 0x0, sizeof(rxq->fake_mbuf)); + + for (i = 0; i < IAVF_RX_MAX_BURST; i++) + rxq->sw_ring[rxq->nb_rx_desc + i] = &rxq->fake_mbuf; + + /* for rx bulk */ + rxq->rx_nb_avail = 0; + rxq->rx_next_avail = 0; + rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1); + + rxq->rx_tail = 0; + rxq->nb_rx_hold = 0; + rxq->pkt_first_seg = NULL; + rxq->pkt_last_seg = NULL; +} + +static inline void +reset_tx_queue(struct iavf_tx_queue *txq) +{ + struct iavf_tx_entry *txe; + uint32_t i, size; + uint16_t prev; + + if (!txq) { + PMD_DRV_LOG(DEBUG, "Pointer to txq is NULL"); + return; + } + + txe = txq->sw_ring; + size = sizeof(struct iavf_tx_desc) * txq->nb_tx_desc; + for (i = 0; i < size; i++) + ((volatile char *)txq->tx_ring)[i] = 0; + + prev = (uint16_t)(txq->nb_tx_desc - 1); + for (i = 0; i < txq->nb_tx_desc; i++) { + txq->tx_ring[i].cmd_type_offset_bsz = + rte_cpu_to_le_64(IAVF_TX_DESC_DTYPE_DESC_DONE); + txe[i].mbuf = NULL; + txe[i].last_id = i; + txe[prev].next_id = i; + prev = i; + } + + txq->tx_tail = 0; + txq->nb_used = 0; + + txq->last_desc_cleaned = txq->nb_tx_desc - 1; + txq->nb_free = txq->nb_tx_desc - 1; + + txq->next_dd = txq->rs_thresh - 1; + txq->next_rs = txq->rs_thresh - 1; +} + +static int +alloc_rxq_mbufs(struct iavf_rx_queue *rxq) +{ + volatile union iavf_rx_desc *rxd; + struct rte_mbuf *mbuf = NULL; + uint64_t dma_addr; + uint16_t i; + + for (i = 0; i < rxq->nb_rx_desc; i++) { + mbuf = rte_mbuf_raw_alloc(rxq->mp); + if (unlikely(!mbuf)) { + PMD_DRV_LOG(ERR, "Failed to allocate mbuf for RX"); + return -ENOMEM; + } + + rte_mbuf_refcnt_set(mbuf, 1); + mbuf->next = NULL; + mbuf->data_off = RTE_PKTMBUF_HEADROOM; + mbuf->nb_segs = 1; + mbuf->port = rxq->port_id; + + dma_addr = + rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf)); + + rxd = &rxq->rx_ring[i]; + rxd->read.pkt_addr = dma_addr; + rxd->read.hdr_addr = 0; +#ifndef RTE_LIBRTE_IAVF_16BYTE_RX_DESC + rxd->read.rsvd1 = 0; + rxd->read.rsvd2 = 0; +#endif + + rxq->sw_ring[i] = mbuf; + } + + return 0; +} + +static inline void +release_rxq_mbufs(struct iavf_rx_queue *rxq) +{ + uint16_t i; + + if (!rxq->sw_ring) + return; + + for (i = 0; i < rxq->nb_rx_desc; i++) { + if (rxq->sw_ring[i]) { + rte_pktmbuf_free_seg(rxq->sw_ring[i]); + rxq->sw_ring[i] = NULL; + } + } + + /* for rx bulk */ + if (rxq->rx_nb_avail == 0) + return; + for (i = 0; i < rxq->rx_nb_avail; i++) { + struct rte_mbuf *mbuf; + + mbuf = rxq->rx_stage[rxq->rx_next_avail + i]; + rte_pktmbuf_free_seg(mbuf); + } + rxq->rx_nb_avail = 0; +} + +static inline void +release_txq_mbufs(struct iavf_tx_queue *txq) +{ + uint16_t i; + + if (!txq || !txq->sw_ring) { + PMD_DRV_LOG(DEBUG, "Pointer to rxq or sw_ring is NULL"); + return; + } + + for (i = 0; i < txq->nb_tx_desc; i++) { + if (txq->sw_ring[i].mbuf) { + rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf); + txq->sw_ring[i].mbuf = NULL; + } + } +} + +static const struct iavf_rxq_ops def_rxq_ops = { + .release_mbufs = release_rxq_mbufs, +}; + +static const struct iavf_txq_ops def_txq_ops = { + .release_mbufs = release_txq_mbufs, +}; + +int +iavf_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, + uint16_t nb_desc, unsigned int socket_id, + const struct rte_eth_rxconf *rx_conf, + struct rte_mempool *mp) +{ + struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct iavf_adapter *ad = + IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct iavf_info *vf = + IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); + struct iavf_vsi *vsi = &vf->vsi; + struct iavf_rx_queue *rxq; + const struct rte_memzone *mz; + uint32_t ring_size; + uint16_t len; + uint16_t rx_free_thresh; + + PMD_INIT_FUNC_TRACE(); + + if (nb_desc % IAVF_ALIGN_RING_DESC != 0 || + nb_desc > IAVF_MAX_RING_DESC || + nb_desc < IAVF_MIN_RING_DESC) { + PMD_INIT_LOG(ERR, "Number (%u) of receive descriptors is " + "invalid", nb_desc); + return -EINVAL; + } + + /* Check free threshold */ + rx_free_thresh = (rx_conf->rx_free_thresh == 0) ? + IAVF_DEFAULT_RX_FREE_THRESH : + rx_conf->rx_free_thresh; + if (check_rx_thresh(nb_desc, rx_free_thresh) != 0) + return -EINVAL; + + /* Free memory if needed */ + if (dev->data->rx_queues[queue_idx]) { + iavf_dev_rx_queue_release(dev->data->rx_queues[queue_idx]); + dev->data->rx_queues[queue_idx] = NULL; + } + + /* Allocate the rx queue data structure */ + rxq = rte_zmalloc_socket("iavf rxq", + sizeof(struct iavf_rx_queue), + RTE_CACHE_LINE_SIZE, + socket_id); + if (!rxq) { + PMD_INIT_LOG(ERR, "Failed to allocate memory for " + "rx queue data structure"); + return -ENOMEM; + } + + if (vf->vf_res->vf_cap_flags & + VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC && + vf->supported_rxdid & BIT(IAVF_RXDID_COMMS_OVS_1)) { + rxq->rxdid = IAVF_RXDID_COMMS_OVS_1; + } else { + rxq->rxdid = IAVF_RXDID_LEGACY_1; + } + + rxq->mp = mp; + rxq->nb_rx_desc = nb_desc; + rxq->rx_free_thresh = rx_free_thresh; + rxq->queue_id = queue_idx; + rxq->port_id = dev->data->port_id; + rxq->crc_len = 0; /* crc stripping by default */ + rxq->rx_deferred_start = rx_conf->rx_deferred_start; + rxq->rx_hdr_len = 0; + rxq->vsi = vsi; + + len = rte_pktmbuf_data_room_size(rxq->mp) - RTE_PKTMBUF_HEADROOM; + rxq->rx_buf_len = RTE_ALIGN(len, (1 << IAVF_RXQ_CTX_DBUFF_SHIFT)); + + /* Allocate the software ring. */ + len = nb_desc + IAVF_RX_MAX_BURST; + rxq->sw_ring = + rte_zmalloc_socket("iavf rx sw ring", + sizeof(struct rte_mbuf *) * len, + RTE_CACHE_LINE_SIZE, + socket_id); + if (!rxq->sw_ring) { + PMD_INIT_LOG(ERR, "Failed to allocate memory for SW ring"); + rte_free(rxq); + return -ENOMEM; + } + + /* Allocate the maximun number of RX ring hardware descriptor with + * a liitle more to support bulk allocate. + */ + len = IAVF_MAX_RING_DESC + IAVF_RX_MAX_BURST; + ring_size = RTE_ALIGN(len * sizeof(union iavf_rx_desc), + IAVF_DMA_MEM_ALIGN); + mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx, + ring_size, IAVF_RING_BASE_ALIGN, + socket_id); + if (!mz) { + PMD_INIT_LOG(ERR, "Failed to reserve DMA memory for RX"); + rte_free(rxq->sw_ring); + rte_free(rxq); + return -ENOMEM; + } + /* Zero all the descriptors in the ring. */ + memset(mz->addr, 0, ring_size); + rxq->rx_ring_phys_addr = mz->iova; + rxq->rx_ring = (union iavf_rx_desc *)mz->addr; + + rxq->mz = mz; + reset_rx_queue(rxq); + rxq->q_set = true; + dev->data->rx_queues[queue_idx] = rxq; + rxq->qrx_tail = hw->hw_addr + IAVF_QRX_TAIL1(rxq->queue_id); + rxq->ops = &def_rxq_ops; + + if (check_rx_bulk_allow(rxq) == true) { + PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are " + "satisfied. Rx Burst Bulk Alloc function will be " + "used on port=%d, queue=%d.", + rxq->port_id, rxq->queue_id); + } else { + PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are " + "not satisfied, Scattered Rx is requested " + "on port=%d, queue=%d.", + rxq->port_id, rxq->queue_id); + ad->rx_bulk_alloc_allowed = false; + } + + if (check_rx_vec_allow(rxq) == false) + ad->rx_vec_allowed = false; + + return 0; +} + +int +iavf_dev_tx_queue_setup(struct rte_eth_dev *dev, + uint16_t queue_idx, + uint16_t nb_desc, + unsigned int socket_id, + const struct rte_eth_txconf *tx_conf) +{ + struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct iavf_tx_queue *txq; + const struct rte_memzone *mz; + uint32_t ring_size; + uint16_t tx_rs_thresh, tx_free_thresh; + uint64_t offloads; + + PMD_INIT_FUNC_TRACE(); + + offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads; + + if (nb_desc % IAVF_ALIGN_RING_DESC != 0 || + nb_desc > IAVF_MAX_RING_DESC || + nb_desc < IAVF_MIN_RING_DESC) { + PMD_INIT_LOG(ERR, "Number (%u) of transmit descriptors is " + "invalid", nb_desc); + return -EINVAL; + } + + tx_rs_thresh = (uint16_t)((tx_conf->tx_rs_thresh) ? + tx_conf->tx_rs_thresh : DEFAULT_TX_RS_THRESH); + tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ? + tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH); + check_tx_thresh(nb_desc, tx_rs_thresh, tx_rs_thresh); + + /* Free memory if needed. */ + if (dev->data->tx_queues[queue_idx]) { + iavf_dev_tx_queue_release(dev->data->tx_queues[queue_idx]); + dev->data->tx_queues[queue_idx] = NULL; + } + + /* Allocate the TX queue data structure. */ + txq = rte_zmalloc_socket("iavf txq", + sizeof(struct iavf_tx_queue), + RTE_CACHE_LINE_SIZE, + socket_id); + if (!txq) { + PMD_INIT_LOG(ERR, "Failed to allocate memory for " + "tx queue structure"); + return -ENOMEM; + } + + txq->nb_tx_desc = nb_desc; + txq->rs_thresh = tx_rs_thresh; + txq->free_thresh = tx_free_thresh; + txq->queue_id = queue_idx; + txq->port_id = dev->data->port_id; + txq->offloads = offloads; + txq->tx_deferred_start = tx_conf->tx_deferred_start; + + /* Allocate software ring */ + txq->sw_ring = + rte_zmalloc_socket("iavf tx sw ring", + sizeof(struct iavf_tx_entry) * nb_desc, + RTE_CACHE_LINE_SIZE, + socket_id); + if (!txq->sw_ring) { + PMD_INIT_LOG(ERR, "Failed to allocate memory for SW TX ring"); + rte_free(txq); + return -ENOMEM; + } + + /* Allocate TX hardware ring descriptors. */ + ring_size = sizeof(struct iavf_tx_desc) * IAVF_MAX_RING_DESC; + ring_size = RTE_ALIGN(ring_size, IAVF_DMA_MEM_ALIGN); + mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx, + ring_size, IAVF_RING_BASE_ALIGN, + socket_id); + if (!mz) { + PMD_INIT_LOG(ERR, "Failed to reserve DMA memory for TX"); + rte_free(txq->sw_ring); + rte_free(txq); + return -ENOMEM; + } + txq->tx_ring_phys_addr = mz->iova; + txq->tx_ring = (struct iavf_tx_desc *)mz->addr; + + txq->mz = mz; + reset_tx_queue(txq); + txq->q_set = true; + dev->data->tx_queues[queue_idx] = txq; + txq->qtx_tail = hw->hw_addr + IAVF_QTX_TAIL1(queue_idx); + txq->ops = &def_txq_ops; + + if (check_tx_vec_allow(txq) == false) { + struct iavf_adapter *ad = + IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + ad->tx_vec_allowed = false; + } + + return 0; +} + +int +iavf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id) +{ + struct iavf_adapter *adapter = + IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct iavf_rx_queue *rxq; + int err = 0; + + PMD_DRV_FUNC_TRACE(); + + if (rx_queue_id >= dev->data->nb_rx_queues) + return -EINVAL; + + rxq = dev->data->rx_queues[rx_queue_id]; + + err = alloc_rxq_mbufs(rxq); + if (err) { + PMD_DRV_LOG(ERR, "Failed to allocate RX queue mbuf"); + return err; + } + + rte_wmb(); + + /* Init the RX tail register. */ + IAVF_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1); + IAVF_WRITE_FLUSH(hw); + + /* Ready to switch the queue on */ + err = iavf_switch_queue(adapter, rx_queue_id, true, true); + if (err) + PMD_DRV_LOG(ERR, "Failed to switch RX queue %u on", + rx_queue_id); + else + dev->data->rx_queue_state[rx_queue_id] = + RTE_ETH_QUEUE_STATE_STARTED; + + return err; +} + +int +iavf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id) +{ + struct iavf_adapter *adapter = + IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct iavf_tx_queue *txq; + int err = 0; + + PMD_DRV_FUNC_TRACE(); + + if (tx_queue_id >= dev->data->nb_tx_queues) + return -EINVAL; + + txq = dev->data->tx_queues[tx_queue_id]; + + /* Init the RX tail register. */ + IAVF_PCI_REG_WRITE(txq->qtx_tail, 0); + IAVF_WRITE_FLUSH(hw); + + /* Ready to switch the queue on */ + err = iavf_switch_queue(adapter, tx_queue_id, false, true); + + if (err) + PMD_DRV_LOG(ERR, "Failed to switch TX queue %u on", + tx_queue_id); + else + dev->data->tx_queue_state[tx_queue_id] = + RTE_ETH_QUEUE_STATE_STARTED; + + return err; +} + +int +iavf_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id) +{ + struct iavf_adapter *adapter = + IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct iavf_rx_queue *rxq; + int err; + + PMD_DRV_FUNC_TRACE(); + + if (rx_queue_id >= dev->data->nb_rx_queues) + return -EINVAL; + + err = iavf_switch_queue(adapter, rx_queue_id, true, false); + if (err) { + PMD_DRV_LOG(ERR, "Failed to switch RX queue %u off", + rx_queue_id); + return err; + } + + rxq = dev->data->rx_queues[rx_queue_id]; + rxq->ops->release_mbufs(rxq); + reset_rx_queue(rxq); + dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; + + return 0; +} + +int +iavf_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) +{ + struct iavf_adapter *adapter = + IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct iavf_tx_queue *txq; + int err; + + PMD_DRV_FUNC_TRACE(); + + if (tx_queue_id >= dev->data->nb_tx_queues) + return -EINVAL; + + err = iavf_switch_queue(adapter, tx_queue_id, false, false); + if (err) { + PMD_DRV_LOG(ERR, "Failed to switch TX queue %u off", + tx_queue_id); + return err; + } + + txq = dev->data->tx_queues[tx_queue_id]; + txq->ops->release_mbufs(txq); + reset_tx_queue(txq); + dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; + + return 0; +} + +void +iavf_dev_rx_queue_release(void *rxq) +{ + struct iavf_rx_queue *q = (struct iavf_rx_queue *)rxq; + + if (!q) + return; + + q->ops->release_mbufs(q); + rte_free(q->sw_ring); + rte_memzone_free(q->mz); + rte_free(q); +} + +void +iavf_dev_tx_queue_release(void *txq) +{ + struct iavf_tx_queue *q = (struct iavf_tx_queue *)txq; + + if (!q) + return; + + q->ops->release_mbufs(q); + rte_free(q->sw_ring); + rte_memzone_free(q->mz); + rte_free(q); +} + +void +iavf_stop_queues(struct rte_eth_dev *dev) +{ + struct iavf_adapter *adapter = + IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct iavf_rx_queue *rxq; + struct iavf_tx_queue *txq; + int ret, i; + + /* Stop All queues */ + ret = iavf_disable_queues(adapter); + if (ret) + PMD_DRV_LOG(WARNING, "Fail to stop queues"); + + for (i = 0; i < dev->data->nb_tx_queues; i++) { + txq = dev->data->tx_queues[i]; + if (!txq) + continue; + txq->ops->release_mbufs(txq); + reset_tx_queue(txq); + dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; + } + for (i = 0; i < dev->data->nb_rx_queues; i++) { + rxq = dev->data->rx_queues[i]; + if (!rxq) + continue; + rxq->ops->release_mbufs(rxq); + reset_rx_queue(rxq); + dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; + } +} + +static inline void +iavf_rxd_to_vlan_tci(struct rte_mbuf *mb, volatile union iavf_rx_desc *rxdp) +{ + if (rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len) & + (1 << IAVF_RX_DESC_STATUS_L2TAG1P_SHIFT)) { + mb->ol_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; + mb->vlan_tci = + rte_le_to_cpu_16(rxdp->wb.qword0.lo_dword.l2tag1); + } else { + mb->vlan_tci = 0; + } +} + +static inline void +iavf_flex_rxd_to_vlan_tci(struct rte_mbuf *mb, + volatile union iavf_rx_flex_desc *rxdp) +{ + if (rte_le_to_cpu_64(rxdp->wb.status_error0) & + (1 << IAVF_RX_FLEX_DESC_STATUS0_L2TAG1P_S)) { + mb->ol_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; + mb->vlan_tci = + rte_le_to_cpu_16(rxdp->wb.l2tag1); + } else { + mb->vlan_tci = 0; + } +} + +/* Translate the rx descriptor status and error fields to pkt flags */ +static inline uint64_t +iavf_rxd_to_pkt_flags(uint64_t qword) +{ + uint64_t flags; + uint64_t error_bits = (qword >> IAVF_RXD_QW1_ERROR_SHIFT); + +#define IAVF_RX_ERR_BITS 0x3f + + /* Check if RSS_HASH */ + flags = (((qword >> IAVF_RX_DESC_STATUS_FLTSTAT_SHIFT) & + IAVF_RX_DESC_FLTSTAT_RSS_HASH) == + IAVF_RX_DESC_FLTSTAT_RSS_HASH) ? PKT_RX_RSS_HASH : 0; + + /* Check if FDIR Match */ + flags |= (qword & (1 << IAVF_RX_DESC_STATUS_FLM_SHIFT) ? + PKT_RX_FDIR : 0); + + if (likely((error_bits & IAVF_RX_ERR_BITS) == 0)) { + flags |= (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD); + return flags; + } + + if (unlikely(error_bits & (1 << IAVF_RX_DESC_ERROR_IPE_SHIFT))) + flags |= PKT_RX_IP_CKSUM_BAD; + else + flags |= PKT_RX_IP_CKSUM_GOOD; + + if (unlikely(error_bits & (1 << IAVF_RX_DESC_ERROR_L4E_SHIFT))) + flags |= PKT_RX_L4_CKSUM_BAD; + else + flags |= PKT_RX_L4_CKSUM_GOOD; + + /* TODO: Oversize error bit is not processed here */ + + return flags; +} + +static inline uint64_t +iavf_rxd_build_fdir(volatile union iavf_rx_desc *rxdp, struct rte_mbuf *mb) +{ + uint64_t flags = 0; +#ifndef RTE_LIBRTE_IAVF_16BYTE_RX_DESC + uint16_t flexbh; + + flexbh = (rte_le_to_cpu_32(rxdp->wb.qword2.ext_status) >> + IAVF_RX_DESC_EXT_STATUS_FLEXBH_SHIFT) & + IAVF_RX_DESC_EXT_STATUS_FLEXBH_MASK; + + if (flexbh == IAVF_RX_DESC_EXT_STATUS_FLEXBH_FD_ID) { + mb->hash.fdir.hi = + rte_le_to_cpu_32(rxdp->wb.qword3.hi_dword.fd_id); + flags |= PKT_RX_FDIR_ID; + } +#else + mb->hash.fdir.hi = + rte_le_to_cpu_32(rxdp->wb.qword0.hi_dword.fd_id); + flags |= PKT_RX_FDIR_ID; +#endif + return flags; +} + + +/* Translate the rx flex descriptor status to pkt flags */ +static inline void +iavf_rxd_to_pkt_fields(struct rte_mbuf *mb, + volatile union iavf_rx_flex_desc *rxdp) +{ + volatile struct iavf_32b_rx_flex_desc_comms_ovs *desc = + (volatile struct iavf_32b_rx_flex_desc_comms_ovs *)rxdp; +#ifndef RTE_LIBRTE_IAVF_16BYTE_RX_DESC + uint16_t stat_err; + + stat_err = rte_le_to_cpu_16(desc->status_error0); + if (likely(stat_err & (1 << IAVF_RX_FLEX_DESC_STATUS0_RSS_VALID_S))) { + mb->ol_flags |= PKT_RX_RSS_HASH; + mb->hash.rss = rte_le_to_cpu_32(desc->rss_hash); + } +#endif + + if (desc->flow_id != 0xFFFFFFFF) { + mb->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID; + mb->hash.fdir.hi = rte_le_to_cpu_32(desc->flow_id); + } +} + +#define IAVF_RX_FLEX_ERR0_BITS \ + ((1 << IAVF_RX_FLEX_DESC_STATUS0_HBO_S) | \ + (1 << IAVF_RX_FLEX_DESC_STATUS0_XSUM_IPE_S) | \ + (1 << IAVF_RX_FLEX_DESC_STATUS0_XSUM_L4E_S) | \ + (1 << IAVF_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S) | \ + (1 << IAVF_RX_FLEX_DESC_STATUS0_XSUM_EUDPE_S) | \ + (1 << IAVF_RX_FLEX_DESC_STATUS0_RXE_S)) + +/* Rx L3/L4 checksum */ +static inline uint64_t +iavf_flex_rxd_error_to_pkt_flags(uint16_t stat_err0) +{ + uint64_t flags = 0; + + /* check if HW has decoded the packet and checksum */ + if (unlikely(!(stat_err0 & (1 << IAVF_RX_FLEX_DESC_STATUS0_L3L4P_S)))) + return 0; + + if (likely(!(stat_err0 & IAVF_RX_FLEX_ERR0_BITS))) { + flags |= (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD); + return flags; + } + + if (unlikely(stat_err0 & (1 << IAVF_RX_FLEX_DESC_STATUS0_XSUM_IPE_S))) + flags |= PKT_RX_IP_CKSUM_BAD; + else + flags |= PKT_RX_IP_CKSUM_GOOD; + + if (unlikely(stat_err0 & (1 << IAVF_RX_FLEX_DESC_STATUS0_XSUM_L4E_S))) + flags |= PKT_RX_L4_CKSUM_BAD; + else + flags |= PKT_RX_L4_CKSUM_GOOD; + + if (unlikely(stat_err0 & (1 << IAVF_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S))) + flags |= PKT_RX_EIP_CKSUM_BAD; + + return flags; +} + +/* If the number of free RX descriptors is greater than the RX free + * threshold of the queue, advance the Receive Descriptor Tail (RDT) + * register. Update the RDT with the value of the last processed RX + * descriptor minus 1, to guarantee that the RDT register is never + * equal to the RDH register, which creates a "full" ring situation + * from the hardware point of view. + */ +static inline void +iavf_update_rx_tail(struct iavf_rx_queue *rxq, uint16_t nb_hold, uint16_t rx_id) +{ + nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold); + + if (nb_hold > rxq->rx_free_thresh) { + PMD_RX_LOG(DEBUG, + "port_id=%u queue_id=%u rx_tail=%u nb_hold=%u", + rxq->port_id, rxq->queue_id, rx_id, nb_hold); + rx_id = (uint16_t)((rx_id == 0) ? + (rxq->nb_rx_desc - 1) : (rx_id - 1)); + IAVF_PCI_REG_WRITE(rxq->qrx_tail, rx_id); + nb_hold = 0; + } + rxq->nb_rx_hold = nb_hold; +} + +/* implement recv_pkts */ +uint16_t +iavf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) +{ + volatile union iavf_rx_desc *rx_ring; + volatile union iavf_rx_desc *rxdp; + struct iavf_rx_queue *rxq; + union iavf_rx_desc rxd; + struct rte_mbuf *rxe; + struct rte_eth_dev *dev; + struct rte_mbuf *rxm; + struct rte_mbuf *nmb; + uint16_t nb_rx; + uint32_t rx_status; + uint64_t qword1; + uint16_t rx_packet_len; + uint16_t rx_id, nb_hold; + uint64_t dma_addr; + uint64_t pkt_flags; + const uint32_t *ptype_tbl; + + nb_rx = 0; + nb_hold = 0; + rxq = rx_queue; + rx_id = rxq->rx_tail; + rx_ring = rxq->rx_ring; + ptype_tbl = rxq->vsi->adapter->ptype_tbl; + + while (nb_rx < nb_pkts) { + rxdp = &rx_ring[rx_id]; + qword1 = rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len); + rx_status = (qword1 & IAVF_RXD_QW1_STATUS_MASK) >> + IAVF_RXD_QW1_STATUS_SHIFT; + + /* Check the DD bit first */ + if (!(rx_status & (1 << IAVF_RX_DESC_STATUS_DD_SHIFT))) + break; + IAVF_DUMP_RX_DESC(rxq, rxdp, rx_id); + + nmb = rte_mbuf_raw_alloc(rxq->mp); + if (unlikely(!nmb)) { + dev = &rte_eth_devices[rxq->port_id]; + dev->data->rx_mbuf_alloc_failed++; + PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u " + "queue_id=%u", rxq->port_id, rxq->queue_id); + break; + } + + rxd = *rxdp; + nb_hold++; + rxe = rxq->sw_ring[rx_id]; + rx_id++; + if (unlikely(rx_id == rxq->nb_rx_desc)) + rx_id = 0; + + /* Prefetch next mbuf */ + rte_prefetch0(rxq->sw_ring[rx_id]); + + /* When next RX descriptor is on a cache line boundary, + * prefetch the next 4 RX descriptors and next 8 pointers + * to mbufs. + */ + if ((rx_id & 0x3) == 0) { + rte_prefetch0(&rx_ring[rx_id]); + rte_prefetch0(rxq->sw_ring[rx_id]); + } + rxm = rxe; + dma_addr = + rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); + rxdp->read.hdr_addr = 0; + rxdp->read.pkt_addr = dma_addr; + + rx_packet_len = ((qword1 & IAVF_RXD_QW1_LENGTH_PBUF_MASK) >> + IAVF_RXD_QW1_LENGTH_PBUF_SHIFT) - rxq->crc_len; + + rxm->data_off = RTE_PKTMBUF_HEADROOM; + rte_prefetch0(RTE_PTR_ADD(rxm->buf_addr, RTE_PKTMBUF_HEADROOM)); + rxm->nb_segs = 1; + rxm->next = NULL; + rxm->pkt_len = rx_packet_len; + rxm->data_len = rx_packet_len; + rxm->port = rxq->port_id; + rxm->ol_flags = 0; + iavf_rxd_to_vlan_tci(rxm, &rxd); + pkt_flags = iavf_rxd_to_pkt_flags(qword1); + rxm->packet_type = + ptype_tbl[(uint8_t)((qword1 & + IAVF_RXD_QW1_PTYPE_MASK) >> IAVF_RXD_QW1_PTYPE_SHIFT)]; + + if (pkt_flags & PKT_RX_RSS_HASH) + rxm->hash.rss = + rte_le_to_cpu_32(rxd.wb.qword0.hi_dword.rss); + + if (pkt_flags & PKT_RX_FDIR) + pkt_flags |= iavf_rxd_build_fdir(&rxd, rxm); + + rxm->ol_flags |= pkt_flags; + + rx_pkts[nb_rx++] = rxm; + } + rxq->rx_tail = rx_id; + + iavf_update_rx_tail(rxq, nb_hold, rx_id); + + return nb_rx; +} + +/* implement recv_pkts for flexible Rx descriptor */ +uint16_t +iavf_recv_pkts_flex_rxd(void *rx_queue, + struct rte_mbuf **rx_pkts, uint16_t nb_pkts) +{ + volatile union iavf_rx_desc *rx_ring; + volatile union iavf_rx_flex_desc *rxdp; + struct iavf_rx_queue *rxq; + union iavf_rx_flex_desc rxd; + struct rte_mbuf *rxe; + struct rte_eth_dev *dev; + struct rte_mbuf *rxm; + struct rte_mbuf *nmb; + uint16_t nb_rx; + uint16_t rx_stat_err0; + uint16_t rx_packet_len; + uint16_t rx_id, nb_hold; + uint64_t dma_addr; + uint64_t pkt_flags; + const uint32_t *ptype_tbl; + + nb_rx = 0; + nb_hold = 0; + rxq = rx_queue; + rx_id = rxq->rx_tail; + rx_ring = rxq->rx_ring; + ptype_tbl = rxq->vsi->adapter->ptype_tbl; + + while (nb_rx < nb_pkts) { + rxdp = (volatile union iavf_rx_flex_desc *)&rx_ring[rx_id]; + rx_stat_err0 = rte_le_to_cpu_16(rxdp->wb.status_error0); + + /* Check the DD bit first */ + if (!(rx_stat_err0 & (1 << IAVF_RX_FLEX_DESC_STATUS0_DD_S))) + break; + IAVF_DUMP_RX_DESC(rxq, rxdp, rx_id); + + nmb = rte_mbuf_raw_alloc(rxq->mp); + if (unlikely(!nmb)) { + dev = &rte_eth_devices[rxq->port_id]; + dev->data->rx_mbuf_alloc_failed++; + PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u " + "queue_id=%u", rxq->port_id, rxq->queue_id); + break; + } + + rxd = *rxdp; + nb_hold++; + rxe = rxq->sw_ring[rx_id]; + rx_id++; + if (unlikely(rx_id == rxq->nb_rx_desc)) + rx_id = 0; + + /* Prefetch next mbuf */ + rte_prefetch0(rxq->sw_ring[rx_id]); + + /* When next RX descriptor is on a cache line boundary, + * prefetch the next 4 RX descriptors and next 8 pointers + * to mbufs. + */ + if ((rx_id & 0x3) == 0) { + rte_prefetch0(&rx_ring[rx_id]); + rte_prefetch0(rxq->sw_ring[rx_id]); + } + rxm = rxe; + dma_addr = + rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); + rxdp->read.hdr_addr = 0; + rxdp->read.pkt_addr = dma_addr; + + rx_packet_len = (rte_le_to_cpu_16(rxd.wb.pkt_len) & + IAVF_RX_FLX_DESC_PKT_LEN_M) - rxq->crc_len; + + rxm->data_off = RTE_PKTMBUF_HEADROOM; + rte_prefetch0(RTE_PTR_ADD(rxm->buf_addr, RTE_PKTMBUF_HEADROOM)); + rxm->nb_segs = 1; + rxm->next = NULL; + rxm->pkt_len = rx_packet_len; + rxm->data_len = rx_packet_len; + rxm->port = rxq->port_id; + rxm->ol_flags = 0; + rxm->packet_type = ptype_tbl[IAVF_RX_FLEX_DESC_PTYPE_M & + rte_le_to_cpu_16(rxd.wb.ptype_flex_flags0)]; + iavf_flex_rxd_to_vlan_tci(rxm, &rxd); + iavf_rxd_to_pkt_fields(rxm, &rxd); + pkt_flags = iavf_flex_rxd_error_to_pkt_flags(rx_stat_err0); + rxm->ol_flags |= pkt_flags; + + rx_pkts[nb_rx++] = rxm; + } + rxq->rx_tail = rx_id; + + iavf_update_rx_tail(rxq, nb_hold, rx_id); + + return nb_rx; +} + +/* implement recv_scattered_pkts for flexible Rx descriptor */ +uint16_t +iavf_recv_scattered_pkts_flex_rxd(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + struct iavf_rx_queue *rxq = rx_queue; + union iavf_rx_flex_desc rxd; + struct rte_mbuf *rxe; + struct rte_mbuf *first_seg = rxq->pkt_first_seg; + struct rte_mbuf *last_seg = rxq->pkt_last_seg; + struct rte_mbuf *nmb, *rxm; + uint16_t rx_id = rxq->rx_tail; + uint16_t nb_rx = 0, nb_hold = 0, rx_packet_len; + struct rte_eth_dev *dev; + uint16_t rx_stat_err0; + uint64_t dma_addr; + uint64_t pkt_flags; + + volatile union iavf_rx_desc *rx_ring = rxq->rx_ring; + volatile union iavf_rx_flex_desc *rxdp; + const uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl; + + while (nb_rx < nb_pkts) { + rxdp = (volatile union iavf_rx_flex_desc *)&rx_ring[rx_id]; + rx_stat_err0 = rte_le_to_cpu_16(rxdp->wb.status_error0); + + /* Check the DD bit */ + if (!(rx_stat_err0 & (1 << IAVF_RX_FLEX_DESC_STATUS0_DD_S))) + break; + IAVF_DUMP_RX_DESC(rxq, rxdp, rx_id); + + nmb = rte_mbuf_raw_alloc(rxq->mp); + if (unlikely(!nmb)) { + PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u " + "queue_id=%u", rxq->port_id, rxq->queue_id); + dev = &rte_eth_devices[rxq->port_id]; + dev->data->rx_mbuf_alloc_failed++; + break; + } + + rxd = *rxdp; + nb_hold++; + rxe = rxq->sw_ring[rx_id]; + rx_id++; + if (rx_id == rxq->nb_rx_desc) + rx_id = 0; + + /* Prefetch next mbuf */ + rte_prefetch0(rxq->sw_ring[rx_id]); + + /* When next RX descriptor is on a cache line boundary, + * prefetch the next 4 RX descriptors and next 8 pointers + * to mbufs. + */ + if ((rx_id & 0x3) == 0) { + rte_prefetch0(&rx_ring[rx_id]); + rte_prefetch0(rxq->sw_ring[rx_id]); + } + + rxm = rxe; + dma_addr = + rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); + + /* Set data buffer address and data length of the mbuf */ + rxdp->read.hdr_addr = 0; + rxdp->read.pkt_addr = dma_addr; + rx_packet_len = rte_le_to_cpu_16(rxd.wb.pkt_len) & + IAVF_RX_FLX_DESC_PKT_LEN_M; + rxm->data_len = rx_packet_len; + rxm->data_off = RTE_PKTMBUF_HEADROOM; + + /* If this is the first buffer of the received packet, set the + * pointer to the first mbuf of the packet and initialize its + * context. Otherwise, update the total length and the number + * of segments of the current scattered packet, and update the + * pointer to the last mbuf of the current packet. + */ + if (!first_seg) { + first_seg = rxm; + first_seg->nb_segs = 1; + first_seg->pkt_len = rx_packet_len; + } else { + first_seg->pkt_len = + (uint16_t)(first_seg->pkt_len + + rx_packet_len); + first_seg->nb_segs++; + last_seg->next = rxm; + } + + /* If this is not the last buffer of the received packet, + * update the pointer to the last mbuf of the current scattered + * packet and continue to parse the RX ring. + */ + if (!(rx_stat_err0 & (1 << IAVF_RX_FLEX_DESC_STATUS0_EOF_S))) { + last_seg = rxm; + continue; + } + + /* This is the last buffer of the received packet. If the CRC + * is not stripped by the hardware: + * - Subtract the CRC length from the total packet length. + * - If the last buffer only contains the whole CRC or a part + * of it, free the mbuf associated to the last buffer. If part + * of the CRC is also contained in the previous mbuf, subtract + * the length of that CRC part from the data length of the + * previous mbuf. + */ + rxm->next = NULL; + if (unlikely(rxq->crc_len > 0)) { + first_seg->pkt_len -= RTE_ETHER_CRC_LEN; + if (rx_packet_len <= RTE_ETHER_CRC_LEN) { + rte_pktmbuf_free_seg(rxm); + first_seg->nb_segs--; + last_seg->data_len = + (uint16_t)(last_seg->data_len - + (RTE_ETHER_CRC_LEN - rx_packet_len)); + last_seg->next = NULL; + } else { + rxm->data_len = (uint16_t)(rx_packet_len - + RTE_ETHER_CRC_LEN); + } + } + + first_seg->port = rxq->port_id; + first_seg->ol_flags = 0; + first_seg->packet_type = ptype_tbl[IAVF_RX_FLEX_DESC_PTYPE_M & + rte_le_to_cpu_16(rxd.wb.ptype_flex_flags0)]; + iavf_flex_rxd_to_vlan_tci(first_seg, &rxd); + iavf_rxd_to_pkt_fields(first_seg, &rxd); + pkt_flags = iavf_flex_rxd_error_to_pkt_flags(rx_stat_err0); + + first_seg->ol_flags |= pkt_flags; + + /* Prefetch data of first segment, if configured to do so. */ + rte_prefetch0(RTE_PTR_ADD(first_seg->buf_addr, + first_seg->data_off)); + rx_pkts[nb_rx++] = first_seg; + first_seg = NULL; + } + + /* Record index of the next RX descriptor to probe. */ + rxq->rx_tail = rx_id; + rxq->pkt_first_seg = first_seg; + rxq->pkt_last_seg = last_seg; + + iavf_update_rx_tail(rxq, nb_hold, rx_id); + + return nb_rx; +} + +/* implement recv_scattered_pkts */ +uint16_t +iavf_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + struct iavf_rx_queue *rxq = rx_queue; + union iavf_rx_desc rxd; + struct rte_mbuf *rxe; + struct rte_mbuf *first_seg = rxq->pkt_first_seg; + struct rte_mbuf *last_seg = rxq->pkt_last_seg; + struct rte_mbuf *nmb, *rxm; + uint16_t rx_id = rxq->rx_tail; + uint16_t nb_rx = 0, nb_hold = 0, rx_packet_len; + struct rte_eth_dev *dev; + uint32_t rx_status; + uint64_t qword1; + uint64_t dma_addr; + uint64_t pkt_flags; + + volatile union iavf_rx_desc *rx_ring = rxq->rx_ring; + volatile union iavf_rx_desc *rxdp; + const uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl; + + while (nb_rx < nb_pkts) { + rxdp = &rx_ring[rx_id]; + qword1 = rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len); + rx_status = (qword1 & IAVF_RXD_QW1_STATUS_MASK) >> + IAVF_RXD_QW1_STATUS_SHIFT; + + /* Check the DD bit */ + if (!(rx_status & (1 << IAVF_RX_DESC_STATUS_DD_SHIFT))) + break; + IAVF_DUMP_RX_DESC(rxq, rxdp, rx_id); + + nmb = rte_mbuf_raw_alloc(rxq->mp); + if (unlikely(!nmb)) { + PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u " + "queue_id=%u", rxq->port_id, rxq->queue_id); + dev = &rte_eth_devices[rxq->port_id]; + dev->data->rx_mbuf_alloc_failed++; + break; + } + + rxd = *rxdp; + nb_hold++; + rxe = rxq->sw_ring[rx_id]; + rx_id++; + if (rx_id == rxq->nb_rx_desc) + rx_id = 0; + + /* Prefetch next mbuf */ + rte_prefetch0(rxq->sw_ring[rx_id]); + + /* When next RX descriptor is on a cache line boundary, + * prefetch the next 4 RX descriptors and next 8 pointers + * to mbufs. + */ + if ((rx_id & 0x3) == 0) { + rte_prefetch0(&rx_ring[rx_id]); + rte_prefetch0(rxq->sw_ring[rx_id]); + } + + rxm = rxe; + dma_addr = + rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); + + /* Set data buffer address and data length of the mbuf */ + rxdp->read.hdr_addr = 0; + rxdp->read.pkt_addr = dma_addr; + rx_packet_len = (qword1 & IAVF_RXD_QW1_LENGTH_PBUF_MASK) >> + IAVF_RXD_QW1_LENGTH_PBUF_SHIFT; + rxm->data_len = rx_packet_len; + rxm->data_off = RTE_PKTMBUF_HEADROOM; + + /* If this is the first buffer of the received packet, set the + * pointer to the first mbuf of the packet and initialize its + * context. Otherwise, update the total length and the number + * of segments of the current scattered packet, and update the + * pointer to the last mbuf of the current packet. + */ + if (!first_seg) { + first_seg = rxm; + first_seg->nb_segs = 1; + first_seg->pkt_len = rx_packet_len; + } else { + first_seg->pkt_len = + (uint16_t)(first_seg->pkt_len + + rx_packet_len); + first_seg->nb_segs++; + last_seg->next = rxm; + } + + /* If this is not the last buffer of the received packet, + * update the pointer to the last mbuf of the current scattered + * packet and continue to parse the RX ring. + */ + if (!(rx_status & (1 << IAVF_RX_DESC_STATUS_EOF_SHIFT))) { + last_seg = rxm; + continue; + } + + /* This is the last buffer of the received packet. If the CRC + * is not stripped by the hardware: + * - Subtract the CRC length from the total packet length. + * - If the last buffer only contains the whole CRC or a part + * of it, free the mbuf associated to the last buffer. If part + * of the CRC is also contained in the previous mbuf, subtract + * the length of that CRC part from the data length of the + * previous mbuf. + */ + rxm->next = NULL; + if (unlikely(rxq->crc_len > 0)) { + first_seg->pkt_len -= RTE_ETHER_CRC_LEN; + if (rx_packet_len <= RTE_ETHER_CRC_LEN) { + rte_pktmbuf_free_seg(rxm); + first_seg->nb_segs--; + last_seg->data_len = + (uint16_t)(last_seg->data_len - + (RTE_ETHER_CRC_LEN - rx_packet_len)); + last_seg->next = NULL; + } else + rxm->data_len = (uint16_t)(rx_packet_len - + RTE_ETHER_CRC_LEN); + } + + first_seg->port = rxq->port_id; + first_seg->ol_flags = 0; + iavf_rxd_to_vlan_tci(first_seg, &rxd); + pkt_flags = iavf_rxd_to_pkt_flags(qword1); + first_seg->packet_type = + ptype_tbl[(uint8_t)((qword1 & + IAVF_RXD_QW1_PTYPE_MASK) >> IAVF_RXD_QW1_PTYPE_SHIFT)]; + + if (pkt_flags & PKT_RX_RSS_HASH) + first_seg->hash.rss = + rte_le_to_cpu_32(rxd.wb.qword0.hi_dword.rss); + + if (pkt_flags & PKT_RX_FDIR) + pkt_flags |= iavf_rxd_build_fdir(&rxd, first_seg); + + first_seg->ol_flags |= pkt_flags; + + /* Prefetch data of first segment, if configured to do so. */ + rte_prefetch0(RTE_PTR_ADD(first_seg->buf_addr, + first_seg->data_off)); + rx_pkts[nb_rx++] = first_seg; + first_seg = NULL; + } + + /* Record index of the next RX descriptor to probe. */ + rxq->rx_tail = rx_id; + rxq->pkt_first_seg = first_seg; + rxq->pkt_last_seg = last_seg; + + iavf_update_rx_tail(rxq, nb_hold, rx_id); + + return nb_rx; +} + +#define IAVF_LOOK_AHEAD 8 +static inline int +iavf_rx_scan_hw_ring_flex_rxd(struct iavf_rx_queue *rxq) +{ + volatile union iavf_rx_flex_desc *rxdp; + struct rte_mbuf **rxep; + struct rte_mbuf *mb; + uint16_t stat_err0; + uint16_t pkt_len; + int32_t s[IAVF_LOOK_AHEAD], nb_dd; + int32_t i, j, nb_rx = 0; + uint64_t pkt_flags; + const uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl; + + rxdp = (volatile union iavf_rx_flex_desc *)&rxq->rx_ring[rxq->rx_tail]; + rxep = &rxq->sw_ring[rxq->rx_tail]; + + stat_err0 = rte_le_to_cpu_16(rxdp->wb.status_error0); + + /* Make sure there is at least 1 packet to receive */ + if (!(stat_err0 & (1 << IAVF_RX_FLEX_DESC_STATUS0_DD_S))) + return 0; + + /* Scan LOOK_AHEAD descriptors at a time to determine which + * descriptors reference packets that are ready to be received. + */ + for (i = 0; i < IAVF_RX_MAX_BURST; i += IAVF_LOOK_AHEAD, + rxdp += IAVF_LOOK_AHEAD, rxep += IAVF_LOOK_AHEAD) { + /* Read desc statuses backwards to avoid race condition */ + for (j = IAVF_LOOK_AHEAD - 1; j >= 0; j--) + s[j] = rte_le_to_cpu_16(rxdp[j].wb.status_error0); + + rte_smp_rmb(); + + /* Compute how many status bits were set */ + for (j = 0, nb_dd = 0; j < IAVF_LOOK_AHEAD; j++) + nb_dd += s[j] & (1 << IAVF_RX_FLEX_DESC_STATUS0_DD_S); + + nb_rx += nb_dd; + + /* Translate descriptor info to mbuf parameters */ + for (j = 0; j < nb_dd; j++) { + IAVF_DUMP_RX_DESC(rxq, &rxdp[j], + rxq->rx_tail + + i * IAVF_LOOK_AHEAD + j); + + mb = rxep[j]; + pkt_len = (rte_le_to_cpu_16(rxdp[j].wb.pkt_len) & + IAVF_RX_FLX_DESC_PKT_LEN_M) - rxq->crc_len; + mb->data_len = pkt_len; + mb->pkt_len = pkt_len; + mb->ol_flags = 0; + + mb->packet_type = ptype_tbl[IAVF_RX_FLEX_DESC_PTYPE_M & + rte_le_to_cpu_16(rxdp[j].wb.ptype_flex_flags0)]; + iavf_flex_rxd_to_vlan_tci(mb, &rxdp[j]); + iavf_rxd_to_pkt_fields(mb, &rxdp[j]); + stat_err0 = rte_le_to_cpu_16(rxdp[j].wb.status_error0); + pkt_flags = iavf_flex_rxd_error_to_pkt_flags(stat_err0); + + mb->ol_flags |= pkt_flags; + } + + for (j = 0; j < IAVF_LOOK_AHEAD; j++) + rxq->rx_stage[i + j] = rxep[j]; + + if (nb_dd != IAVF_LOOK_AHEAD) + break; + } + + /* Clear software ring entries */ + for (i = 0; i < nb_rx; i++) + rxq->sw_ring[rxq->rx_tail + i] = NULL; + + return nb_rx; +} + +static inline int +iavf_rx_scan_hw_ring(struct iavf_rx_queue *rxq) +{ + volatile union iavf_rx_desc *rxdp; + struct rte_mbuf **rxep; + struct rte_mbuf *mb; + uint16_t pkt_len; + uint64_t qword1; + uint32_t rx_status; + int32_t s[IAVF_LOOK_AHEAD], nb_dd; + int32_t i, j, nb_rx = 0; + uint64_t pkt_flags; + const uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl; + + rxdp = &rxq->rx_ring[rxq->rx_tail]; + rxep = &rxq->sw_ring[rxq->rx_tail]; + + qword1 = rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len); + rx_status = (qword1 & IAVF_RXD_QW1_STATUS_MASK) >> + IAVF_RXD_QW1_STATUS_SHIFT; + + /* Make sure there is at least 1 packet to receive */ + if (!(rx_status & (1 << IAVF_RX_DESC_STATUS_DD_SHIFT))) + return 0; + + /* Scan LOOK_AHEAD descriptors at a time to determine which + * descriptors reference packets that are ready to be received. + */ + for (i = 0; i < IAVF_RX_MAX_BURST; i += IAVF_LOOK_AHEAD, + rxdp += IAVF_LOOK_AHEAD, rxep += IAVF_LOOK_AHEAD) { + /* Read desc statuses backwards to avoid race condition */ + for (j = IAVF_LOOK_AHEAD - 1; j >= 0; j--) { + qword1 = rte_le_to_cpu_64( + rxdp[j].wb.qword1.status_error_len); + s[j] = (qword1 & IAVF_RXD_QW1_STATUS_MASK) >> + IAVF_RXD_QW1_STATUS_SHIFT; + } + + rte_smp_rmb(); + + /* Compute how many status bits were set */ + for (j = 0, nb_dd = 0; j < IAVF_LOOK_AHEAD; j++) + nb_dd += s[j] & (1 << IAVF_RX_DESC_STATUS_DD_SHIFT); + + nb_rx += nb_dd; + + /* Translate descriptor info to mbuf parameters */ + for (j = 0; j < nb_dd; j++) { + IAVF_DUMP_RX_DESC(rxq, &rxdp[j], + rxq->rx_tail + i * IAVF_LOOK_AHEAD + j); + + mb = rxep[j]; + qword1 = rte_le_to_cpu_64 + (rxdp[j].wb.qword1.status_error_len); + pkt_len = ((qword1 & IAVF_RXD_QW1_LENGTH_PBUF_MASK) >> + IAVF_RXD_QW1_LENGTH_PBUF_SHIFT) - rxq->crc_len; + mb->data_len = pkt_len; + mb->pkt_len = pkt_len; + mb->ol_flags = 0; + iavf_rxd_to_vlan_tci(mb, &rxdp[j]); + pkt_flags = iavf_rxd_to_pkt_flags(qword1); + mb->packet_type = + ptype_tbl[(uint8_t)((qword1 & + IAVF_RXD_QW1_PTYPE_MASK) >> + IAVF_RXD_QW1_PTYPE_SHIFT)]; + + if (pkt_flags & PKT_RX_RSS_HASH) + mb->hash.rss = rte_le_to_cpu_32( + rxdp[j].wb.qword0.hi_dword.rss); + + if (pkt_flags & PKT_RX_FDIR) + pkt_flags |= iavf_rxd_build_fdir(&rxdp[j], mb); + + mb->ol_flags |= pkt_flags; + } + + for (j = 0; j < IAVF_LOOK_AHEAD; j++) + rxq->rx_stage[i + j] = rxep[j]; + + if (nb_dd != IAVF_LOOK_AHEAD) + break; + } + + /* Clear software ring entries */ + for (i = 0; i < nb_rx; i++) + rxq->sw_ring[rxq->rx_tail + i] = NULL; + + return nb_rx; +} + +static inline uint16_t +iavf_rx_fill_from_stage(struct iavf_rx_queue *rxq, + struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + uint16_t i; + struct rte_mbuf **stage = &rxq->rx_stage[rxq->rx_next_avail]; + + nb_pkts = (uint16_t)RTE_MIN(nb_pkts, rxq->rx_nb_avail); + + for (i = 0; i < nb_pkts; i++) + rx_pkts[i] = stage[i]; + + rxq->rx_nb_avail = (uint16_t)(rxq->rx_nb_avail - nb_pkts); + rxq->rx_next_avail = (uint16_t)(rxq->rx_next_avail + nb_pkts); + + return nb_pkts; +} + +static inline int +iavf_rx_alloc_bufs(struct iavf_rx_queue *rxq) +{ + volatile union iavf_rx_desc *rxdp; + struct rte_mbuf **rxep; + struct rte_mbuf *mb; + uint16_t alloc_idx, i; + uint64_t dma_addr; + int diag; + + /* Allocate buffers in bulk */ + alloc_idx = (uint16_t)(rxq->rx_free_trigger - + (rxq->rx_free_thresh - 1)); + rxep = &rxq->sw_ring[alloc_idx]; + diag = rte_mempool_get_bulk(rxq->mp, (void *)rxep, + rxq->rx_free_thresh); + if (unlikely(diag != 0)) { + PMD_RX_LOG(ERR, "Failed to get mbufs in bulk"); + return -ENOMEM; + } + + rxdp = &rxq->rx_ring[alloc_idx]; + for (i = 0; i < rxq->rx_free_thresh; i++) { + if (likely(i < (rxq->rx_free_thresh - 1))) + /* Prefetch next mbuf */ + rte_prefetch0(rxep[i + 1]); + + mb = rxep[i]; + rte_mbuf_refcnt_set(mb, 1); + mb->next = NULL; + mb->data_off = RTE_PKTMBUF_HEADROOM; + mb->nb_segs = 1; + mb->port = rxq->port_id; + dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mb)); + rxdp[i].read.hdr_addr = 0; + rxdp[i].read.pkt_addr = dma_addr; + } + + /* Update rx tail register */ + rte_wmb(); + IAVF_PCI_REG_WRITE_RELAXED(rxq->qrx_tail, rxq->rx_free_trigger); + + rxq->rx_free_trigger = + (uint16_t)(rxq->rx_free_trigger + rxq->rx_free_thresh); + if (rxq->rx_free_trigger >= rxq->nb_rx_desc) + rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1); + + return 0; +} + +static inline uint16_t +rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) +{ + struct iavf_rx_queue *rxq = (struct iavf_rx_queue *)rx_queue; + uint16_t nb_rx = 0; + + if (!nb_pkts) + return 0; + + if (rxq->rx_nb_avail) + return iavf_rx_fill_from_stage(rxq, rx_pkts, nb_pkts); + + if (rxq->rxdid == IAVF_RXDID_COMMS_OVS_1) + nb_rx = (uint16_t)iavf_rx_scan_hw_ring_flex_rxd(rxq); + else + nb_rx = (uint16_t)iavf_rx_scan_hw_ring(rxq); + rxq->rx_next_avail = 0; + rxq->rx_nb_avail = nb_rx; + rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_rx); + + if (rxq->rx_tail > rxq->rx_free_trigger) { + if (iavf_rx_alloc_bufs(rxq) != 0) { + uint16_t i, j; + + /* TODO: count rx_mbuf_alloc_failed here */ + + rxq->rx_nb_avail = 0; + rxq->rx_tail = (uint16_t)(rxq->rx_tail - nb_rx); + for (i = 0, j = rxq->rx_tail; i < nb_rx; i++, j++) + rxq->sw_ring[j] = rxq->rx_stage[i]; + + return 0; + } + } + + if (rxq->rx_tail >= rxq->nb_rx_desc) + rxq->rx_tail = 0; + + PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u, nb_rx=%u", + rxq->port_id, rxq->queue_id, + rxq->rx_tail, nb_rx); + + if (rxq->rx_nb_avail) + return iavf_rx_fill_from_stage(rxq, rx_pkts, nb_pkts); + + return 0; +} + +static uint16_t +iavf_recv_pkts_bulk_alloc(void *rx_queue, + struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + uint16_t nb_rx = 0, n, count; + + if (unlikely(nb_pkts == 0)) + return 0; + + if (likely(nb_pkts <= IAVF_RX_MAX_BURST)) + return rx_recv_pkts(rx_queue, rx_pkts, nb_pkts); + + while (nb_pkts) { + n = RTE_MIN(nb_pkts, IAVF_RX_MAX_BURST); + count = rx_recv_pkts(rx_queue, &rx_pkts[nb_rx], n); + nb_rx = (uint16_t)(nb_rx + count); + nb_pkts = (uint16_t)(nb_pkts - count); + if (count < n) + break; + } + + return nb_rx; +} + +static inline int +iavf_xmit_cleanup(struct iavf_tx_queue *txq) +{ + struct iavf_tx_entry *sw_ring = txq->sw_ring; + uint16_t last_desc_cleaned = txq->last_desc_cleaned; + uint16_t nb_tx_desc = txq->nb_tx_desc; + uint16_t desc_to_clean_to; + uint16_t nb_tx_to_clean; + + volatile struct iavf_tx_desc *txd = txq->tx_ring; + + desc_to_clean_to = (uint16_t)(last_desc_cleaned + txq->rs_thresh); + if (desc_to_clean_to >= nb_tx_desc) + desc_to_clean_to = (uint16_t)(desc_to_clean_to - nb_tx_desc); + + desc_to_clean_to = sw_ring[desc_to_clean_to].last_id; + if ((txd[desc_to_clean_to].cmd_type_offset_bsz & + rte_cpu_to_le_64(IAVF_TXD_QW1_DTYPE_MASK)) != + rte_cpu_to_le_64(IAVF_TX_DESC_DTYPE_DESC_DONE)) { + PMD_TX_FREE_LOG(DEBUG, "TX descriptor %4u is not done " + "(port=%d queue=%d)", desc_to_clean_to, + txq->port_id, txq->queue_id); + return -1; + } + + if (last_desc_cleaned > desc_to_clean_to) + nb_tx_to_clean = (uint16_t)((nb_tx_desc - last_desc_cleaned) + + desc_to_clean_to); + else + nb_tx_to_clean = (uint16_t)(desc_to_clean_to - + last_desc_cleaned); + + txd[desc_to_clean_to].cmd_type_offset_bsz = 0; + + txq->last_desc_cleaned = desc_to_clean_to; + txq->nb_free = (uint16_t)(txq->nb_free + nb_tx_to_clean); + + return 0; +} + +/* Check if the context descriptor is needed for TX offloading */ +static inline uint16_t +iavf_calc_context_desc(uint64_t flags) +{ + static uint64_t mask = PKT_TX_TCP_SEG; + + return (flags & mask) ? 1 : 0; +} + +static inline void +iavf_txd_enable_checksum(uint64_t ol_flags, + uint32_t *td_cmd, + uint32_t *td_offset, + union iavf_tx_offload tx_offload) +{ + /* Set MACLEN */ + *td_offset |= (tx_offload.l2_len >> 1) << + IAVF_TX_DESC_LENGTH_MACLEN_SHIFT; + + /* Enable L3 checksum offloads */ + if (ol_flags & PKT_TX_IP_CKSUM) { + *td_cmd |= IAVF_TX_DESC_CMD_IIPT_IPV4_CSUM; + *td_offset |= (tx_offload.l3_len >> 2) << + IAVF_TX_DESC_LENGTH_IPLEN_SHIFT; + } else if (ol_flags & PKT_TX_IPV4) { + *td_cmd |= IAVF_TX_DESC_CMD_IIPT_IPV4; + *td_offset |= (tx_offload.l3_len >> 2) << + IAVF_TX_DESC_LENGTH_IPLEN_SHIFT; + } else if (ol_flags & PKT_TX_IPV6) { + *td_cmd |= IAVF_TX_DESC_CMD_IIPT_IPV6; + *td_offset |= (tx_offload.l3_len >> 2) << + IAVF_TX_DESC_LENGTH_IPLEN_SHIFT; + } + + if (ol_flags & PKT_TX_TCP_SEG) { + *td_cmd |= IAVF_TX_DESC_CMD_L4T_EOFT_TCP; + *td_offset |= (tx_offload.l4_len >> 2) << + IAVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; + return; + } + + /* Enable L4 checksum offloads */ + switch (ol_flags & PKT_TX_L4_MASK) { + case PKT_TX_TCP_CKSUM: + *td_cmd |= IAVF_TX_DESC_CMD_L4T_EOFT_TCP; + *td_offset |= (sizeof(struct rte_tcp_hdr) >> 2) << + IAVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; + break; + case PKT_TX_SCTP_CKSUM: + *td_cmd |= IAVF_TX_DESC_CMD_L4T_EOFT_SCTP; + *td_offset |= (sizeof(struct rte_sctp_hdr) >> 2) << + IAVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; + break; + case PKT_TX_UDP_CKSUM: + *td_cmd |= IAVF_TX_DESC_CMD_L4T_EOFT_UDP; + *td_offset |= (sizeof(struct rte_udp_hdr) >> 2) << + IAVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; + break; + default: + break; + } +} + +/* set TSO context descriptor + * support IP -> L4 and IP -> IP -> L4 + */ +static inline uint64_t +iavf_set_tso_ctx(struct rte_mbuf *mbuf, union iavf_tx_offload tx_offload) +{ + uint64_t ctx_desc = 0; + uint32_t cd_cmd, hdr_len, cd_tso_len; + + if (!tx_offload.l4_len) { + PMD_TX_LOG(DEBUG, "L4 length set to 0"); + return ctx_desc; + } + + hdr_len = tx_offload.l2_len + + tx_offload.l3_len + + tx_offload.l4_len; + + cd_cmd = IAVF_TX_CTX_DESC_TSO; + cd_tso_len = mbuf->pkt_len - hdr_len; + ctx_desc |= ((uint64_t)cd_cmd << IAVF_TXD_CTX_QW1_CMD_SHIFT) | + ((uint64_t)cd_tso_len << IAVF_TXD_CTX_QW1_TSO_LEN_SHIFT) | + ((uint64_t)mbuf->tso_segsz << IAVF_TXD_CTX_QW1_MSS_SHIFT); + + return ctx_desc; +} + +/* Construct the tx flags */ +static inline uint64_t +iavf_build_ctob(uint32_t td_cmd, uint32_t td_offset, unsigned int size, + uint32_t td_tag) +{ + return rte_cpu_to_le_64(IAVF_TX_DESC_DTYPE_DATA | + ((uint64_t)td_cmd << IAVF_TXD_QW1_CMD_SHIFT) | + ((uint64_t)td_offset << + IAVF_TXD_QW1_OFFSET_SHIFT) | + ((uint64_t)size << + IAVF_TXD_QW1_TX_BUF_SZ_SHIFT) | + ((uint64_t)td_tag << + IAVF_TXD_QW1_L2TAG1_SHIFT)); +} + +/* TX function */ +uint16_t +iavf_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) +{ + volatile struct iavf_tx_desc *txd; + volatile struct iavf_tx_desc *txr; + struct iavf_tx_queue *txq; + struct iavf_tx_entry *sw_ring; + struct iavf_tx_entry *txe, *txn; + struct rte_mbuf *tx_pkt; + struct rte_mbuf *m_seg; + uint16_t tx_id; + uint16_t nb_tx; + uint32_t td_cmd; + uint32_t td_offset; + uint32_t td_tag; + uint64_t ol_flags; + uint16_t nb_used; + uint16_t nb_ctx; + uint16_t tx_last; + uint16_t slen; + uint64_t buf_dma_addr; + union iavf_tx_offload tx_offload = {0}; + + txq = tx_queue; + sw_ring = txq->sw_ring; + txr = txq->tx_ring; + tx_id = txq->tx_tail; + txe = &sw_ring[tx_id]; + + /* Check if the descriptor ring needs to be cleaned. */ + if (txq->nb_free < txq->free_thresh) + iavf_xmit_cleanup(txq); + + for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { + td_cmd = 0; + td_tag = 0; + td_offset = 0; + + tx_pkt = *tx_pkts++; + RTE_MBUF_PREFETCH_TO_FREE(txe->mbuf); + + ol_flags = tx_pkt->ol_flags; + tx_offload.l2_len = tx_pkt->l2_len; + tx_offload.l3_len = tx_pkt->l3_len; + tx_offload.l4_len = tx_pkt->l4_len; + tx_offload.tso_segsz = tx_pkt->tso_segsz; + + /* Calculate the number of context descriptors needed. */ + nb_ctx = iavf_calc_context_desc(ol_flags); + + /* The number of descriptors that must be allocated for + * a packet equals to the number of the segments of that + * packet plus 1 context descriptor if needed. + */ + nb_used = (uint16_t)(tx_pkt->nb_segs + nb_ctx); + tx_last = (uint16_t)(tx_id + nb_used - 1); + + /* Circular ring */ + if (tx_last >= txq->nb_tx_desc) + tx_last = (uint16_t)(tx_last - txq->nb_tx_desc); + + PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u" + " tx_first=%u tx_last=%u", + txq->port_id, txq->queue_id, tx_id, tx_last); + + if (nb_used > txq->nb_free) { + if (iavf_xmit_cleanup(txq)) { + if (nb_tx == 0) + return 0; + goto end_of_tx; + } + if (unlikely(nb_used > txq->rs_thresh)) { + while (nb_used > txq->nb_free) { + if (iavf_xmit_cleanup(txq)) { + if (nb_tx == 0) + return 0; + goto end_of_tx; + } + } + } + } + + /* Descriptor based VLAN insertion */ + if (ol_flags & PKT_TX_VLAN_PKT) { + td_cmd |= IAVF_TX_DESC_CMD_IL2TAG1; + td_tag = tx_pkt->vlan_tci; + } + + /* According to datasheet, the bit2 is reserved and must be + * set to 1. + */ + td_cmd |= 0x04; + + /* Enable checksum offloading */ + if (ol_flags & IAVF_TX_CKSUM_OFFLOAD_MASK) + iavf_txd_enable_checksum(ol_flags, &td_cmd, + &td_offset, tx_offload); + + if (nb_ctx) { + /* Setup TX context descriptor if required */ + uint64_t cd_type_cmd_tso_mss = + IAVF_TX_DESC_DTYPE_CONTEXT; + volatile struct iavf_tx_context_desc *ctx_txd = + (volatile struct iavf_tx_context_desc *) + &txr[tx_id]; + + txn = &sw_ring[txe->next_id]; + RTE_MBUF_PREFETCH_TO_FREE(txn->mbuf); + if (txe->mbuf) { + rte_pktmbuf_free_seg(txe->mbuf); + txe->mbuf = NULL; + } + + /* TSO enabled */ + if (ol_flags & PKT_TX_TCP_SEG) + cd_type_cmd_tso_mss |= + iavf_set_tso_ctx(tx_pkt, tx_offload); + + ctx_txd->type_cmd_tso_mss = + rte_cpu_to_le_64(cd_type_cmd_tso_mss); + + IAVF_DUMP_TX_DESC(txq, &txr[tx_id], tx_id); + txe->last_id = tx_last; + tx_id = txe->next_id; + txe = txn; + } + + m_seg = tx_pkt; + do { + txd = &txr[tx_id]; + txn = &sw_ring[txe->next_id]; + + if (txe->mbuf) + rte_pktmbuf_free_seg(txe->mbuf); + txe->mbuf = m_seg; + + /* Setup TX Descriptor */ + slen = m_seg->data_len; + buf_dma_addr = rte_mbuf_data_iova(m_seg); + txd->buffer_addr = rte_cpu_to_le_64(buf_dma_addr); + txd->cmd_type_offset_bsz = iavf_build_ctob(td_cmd, + td_offset, + slen, + td_tag); + + IAVF_DUMP_TX_DESC(txq, txd, tx_id); + txe->last_id = tx_last; + tx_id = txe->next_id; + txe = txn; + m_seg = m_seg->next; + } while (m_seg); + + /* The last packet data descriptor needs End Of Packet (EOP) */ + td_cmd |= IAVF_TX_DESC_CMD_EOP; + txq->nb_used = (uint16_t)(txq->nb_used + nb_used); + txq->nb_free = (uint16_t)(txq->nb_free - nb_used); + + if (txq->nb_used >= txq->rs_thresh) { + PMD_TX_LOG(DEBUG, "Setting RS bit on TXD id=" + "%4u (port=%d queue=%d)", + tx_last, txq->port_id, txq->queue_id); + + td_cmd |= IAVF_TX_DESC_CMD_RS; + + /* Update txq RS bit counters */ + txq->nb_used = 0; + } + + txd->cmd_type_offset_bsz |= + rte_cpu_to_le_64(((uint64_t)td_cmd) << + IAVF_TXD_QW1_CMD_SHIFT); + IAVF_DUMP_TX_DESC(txq, txd, tx_id); + } + +end_of_tx: + rte_wmb(); + + PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u", + txq->port_id, txq->queue_id, tx_id, nb_tx); + + IAVF_PCI_REG_WRITE_RELAXED(txq->qtx_tail, tx_id); + txq->tx_tail = tx_id; + + return nb_tx; +} + +/* TX prep functions */ +uint16_t +iavf_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + int i, ret; + uint64_t ol_flags; + struct rte_mbuf *m; + + for (i = 0; i < nb_pkts; i++) { + m = tx_pkts[i]; + ol_flags = m->ol_flags; + + /* Check condition for nb_segs > IAVF_TX_MAX_MTU_SEG. */ + if (!(ol_flags & PKT_TX_TCP_SEG)) { + if (m->nb_segs > IAVF_TX_MAX_MTU_SEG) { + rte_errno = EINVAL; + return i; + } + } else if ((m->tso_segsz < IAVF_MIN_TSO_MSS) || + (m->tso_segsz > IAVF_MAX_TSO_MSS)) { + /* MSS outside the range are considered malicious */ + rte_errno = EINVAL; + return i; + } + + if (ol_flags & IAVF_TX_OFFLOAD_NOTSUP_MASK) { + rte_errno = ENOTSUP; + return i; + } + +#ifdef RTE_LIBRTE_ETHDEV_DEBUG + ret = rte_validate_tx_offload(m); + if (ret != 0) { + rte_errno = -ret; + return i; + } +#endif + ret = rte_net_intel_cksum_prepare(m); + if (ret != 0) { + rte_errno = -ret; + return i; + } + } + + return i; +} + +/* choose rx function*/ +void +iavf_set_rx_function(struct rte_eth_dev *dev) +{ + struct iavf_adapter *adapter = + IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); +#ifdef RTE_ARCH_X86 + struct iavf_rx_queue *rxq; + int i; + bool use_avx2 = false; + + if (!iavf_rx_vec_dev_check(dev)) { + for (i = 0; i < dev->data->nb_rx_queues; i++) { + rxq = dev->data->rx_queues[i]; + (void)iavf_rxq_vec_setup(rxq); + } + + if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2) == 1 || + rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1) + use_avx2 = true; + + if (dev->data->scattered_rx) { + PMD_DRV_LOG(DEBUG, + "Using %sVector Scattered Rx (port %d).", + use_avx2 ? "avx2 " : "", + dev->data->port_id); + if (vf->vf_res->vf_cap_flags & + VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) + dev->rx_pkt_burst = use_avx2 ? + iavf_recv_scattered_pkts_vec_avx2_flex_rxd : + iavf_recv_scattered_pkts_vec_flex_rxd; + else + dev->rx_pkt_burst = use_avx2 ? + iavf_recv_scattered_pkts_vec_avx2 : + iavf_recv_scattered_pkts_vec; + } else { + PMD_DRV_LOG(DEBUG, "Using %sVector Rx (port %d).", + use_avx2 ? "avx2 " : "", + dev->data->port_id); + if (vf->vf_res->vf_cap_flags & + VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) + dev->rx_pkt_burst = use_avx2 ? + iavf_recv_pkts_vec_avx2_flex_rxd : + iavf_recv_pkts_vec_flex_rxd; + else + dev->rx_pkt_burst = use_avx2 ? + iavf_recv_pkts_vec_avx2 : + iavf_recv_pkts_vec; + } + + return; + } +#endif + + if (dev->data->scattered_rx) { + PMD_DRV_LOG(DEBUG, "Using a Scattered Rx callback (port=%d).", + dev->data->port_id); + if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) + dev->rx_pkt_burst = iavf_recv_scattered_pkts_flex_rxd; + else + dev->rx_pkt_burst = iavf_recv_scattered_pkts; + } else if (adapter->rx_bulk_alloc_allowed) { + PMD_DRV_LOG(DEBUG, "Using bulk Rx callback (port=%d).", + dev->data->port_id); + dev->rx_pkt_burst = iavf_recv_pkts_bulk_alloc; + } else { + PMD_DRV_LOG(DEBUG, "Using Basic Rx callback (port=%d).", + dev->data->port_id); + if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) + dev->rx_pkt_burst = iavf_recv_pkts_flex_rxd; + else + dev->rx_pkt_burst = iavf_recv_pkts; + } +} + +/* choose tx function*/ +void +iavf_set_tx_function(struct rte_eth_dev *dev) +{ +#ifdef RTE_ARCH_X86 + struct iavf_tx_queue *txq; + int i; + bool use_avx2 = false; + + if (!iavf_tx_vec_dev_check(dev)) { + for (i = 0; i < dev->data->nb_tx_queues; i++) { + txq = dev->data->tx_queues[i]; + if (!txq) + continue; + iavf_txq_vec_setup(txq); + } + + if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2) == 1 || + rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1) + use_avx2 = true; + + PMD_DRV_LOG(DEBUG, "Using %sVector Tx (port %d).", + use_avx2 ? "avx2 " : "", + dev->data->port_id); + dev->tx_pkt_burst = use_avx2 ? + iavf_xmit_pkts_vec_avx2 : + iavf_xmit_pkts_vec; + dev->tx_pkt_prepare = NULL; + + return; + } +#endif + + PMD_DRV_LOG(DEBUG, "Using Basic Tx callback (port=%d).", + dev->data->port_id); + dev->tx_pkt_burst = iavf_xmit_pkts; + dev->tx_pkt_prepare = iavf_prep_pkts; +} + +void +iavf_dev_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, + struct rte_eth_rxq_info *qinfo) +{ + struct iavf_rx_queue *rxq; + + rxq = dev->data->rx_queues[queue_id]; + + qinfo->mp = rxq->mp; + qinfo->scattered_rx = dev->data->scattered_rx; + qinfo->nb_desc = rxq->nb_rx_desc; + + qinfo->conf.rx_free_thresh = rxq->rx_free_thresh; + qinfo->conf.rx_drop_en = true; + qinfo->conf.rx_deferred_start = rxq->rx_deferred_start; +} + +void +iavf_dev_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, + struct rte_eth_txq_info *qinfo) +{ + struct iavf_tx_queue *txq; + + txq = dev->data->tx_queues[queue_id]; + + qinfo->nb_desc = txq->nb_tx_desc; + + qinfo->conf.tx_free_thresh = txq->free_thresh; + qinfo->conf.tx_rs_thresh = txq->rs_thresh; + qinfo->conf.offloads = txq->offloads; + qinfo->conf.tx_deferred_start = txq->tx_deferred_start; +} + +/* Get the number of used descriptors of a rx queue */ +uint32_t +iavf_dev_rxq_count(struct rte_eth_dev *dev, uint16_t queue_id) +{ +#define IAVF_RXQ_SCAN_INTERVAL 4 + volatile union iavf_rx_desc *rxdp; + struct iavf_rx_queue *rxq; + uint16_t desc = 0; + + rxq = dev->data->rx_queues[queue_id]; + rxdp = &rxq->rx_ring[rxq->rx_tail]; + + while ((desc < rxq->nb_rx_desc) && + ((rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len) & + IAVF_RXD_QW1_STATUS_MASK) >> IAVF_RXD_QW1_STATUS_SHIFT) & + (1 << IAVF_RX_DESC_STATUS_DD_SHIFT)) { + /* Check the DD bit of a rx descriptor of each 4 in a group, + * to avoid checking too frequently and downgrading performance + * too much. + */ + desc += IAVF_RXQ_SCAN_INTERVAL; + rxdp += IAVF_RXQ_SCAN_INTERVAL; + if (rxq->rx_tail + desc >= rxq->nb_rx_desc) + rxdp = &(rxq->rx_ring[rxq->rx_tail + + desc - rxq->nb_rx_desc]); + } + + return desc; +} + +int +iavf_dev_rx_desc_status(void *rx_queue, uint16_t offset) +{ + struct iavf_rx_queue *rxq = rx_queue; + volatile uint64_t *status; + uint64_t mask; + uint32_t desc; + + if (unlikely(offset >= rxq->nb_rx_desc)) + return -EINVAL; + + if (offset >= rxq->nb_rx_desc - rxq->nb_rx_hold) + return RTE_ETH_RX_DESC_UNAVAIL; + + desc = rxq->rx_tail + offset; + if (desc >= rxq->nb_rx_desc) + desc -= rxq->nb_rx_desc; + + status = &rxq->rx_ring[desc].wb.qword1.status_error_len; + mask = rte_le_to_cpu_64((1ULL << IAVF_RX_DESC_STATUS_DD_SHIFT) + << IAVF_RXD_QW1_STATUS_SHIFT); + if (*status & mask) + return RTE_ETH_RX_DESC_DONE; + + return RTE_ETH_RX_DESC_AVAIL; +} + +int +iavf_dev_tx_desc_status(void *tx_queue, uint16_t offset) +{ + struct iavf_tx_queue *txq = tx_queue; + volatile uint64_t *status; + uint64_t mask, expect; + uint32_t desc; + + if (unlikely(offset >= txq->nb_tx_desc)) + return -EINVAL; + + desc = txq->tx_tail + offset; + /* go to next desc that has the RS bit */ + desc = ((desc + txq->rs_thresh - 1) / txq->rs_thresh) * + txq->rs_thresh; + if (desc >= txq->nb_tx_desc) { + desc -= txq->nb_tx_desc; + if (desc >= txq->nb_tx_desc) + desc -= txq->nb_tx_desc; + } + + status = &txq->tx_ring[desc].cmd_type_offset_bsz; + mask = rte_le_to_cpu_64(IAVF_TXD_QW1_DTYPE_MASK); + expect = rte_cpu_to_le_64( + IAVF_TX_DESC_DTYPE_DESC_DONE << IAVF_TXD_QW1_DTYPE_SHIFT); + if ((*status & mask) == expect) + return RTE_ETH_TX_DESC_DONE; + + return RTE_ETH_TX_DESC_FULL; +} + +const uint32_t * +iavf_get_default_ptype_table(void) +{ + static const uint32_t ptype_tbl[IAVF_MAX_PKT_TYPE] + __rte_cache_aligned = { + /* L2 types */ + /* [0] reserved */ + [1] = RTE_PTYPE_L2_ETHER, + [2] = RTE_PTYPE_L2_ETHER_TIMESYNC, + /* [3] - [5] reserved */ + [6] = RTE_PTYPE_L2_ETHER_LLDP, + /* [7] - [10] reserved */ + [11] = RTE_PTYPE_L2_ETHER_ARP, + /* [12] - [21] reserved */ + + /* Non tunneled IPv4 */ + [22] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_FRAG, + [23] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_NONFRAG, + [24] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_UDP, + /* [25] reserved */ + [26] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_TCP, + [27] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_SCTP, + [28] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_ICMP, + + /* IPv4 --> IPv4 */ + [29] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG, + [30] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG, + [31] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + /* [32] reserved */ + [33] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [34] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_SCTP, + [35] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_ICMP, + + /* IPv4 --> IPv6 */ + [36] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG, + [37] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG, + [38] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + /* [39] reserved */ + [40] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [41] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_SCTP, + [42] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_ICMP, + + /* IPv4 --> GRE/Teredo/VXLAN */ + [43] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT, + + /* IPv4 --> GRE/Teredo/VXLAN --> IPv4 */ + [44] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG, + [45] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG, + [46] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + /* [47] reserved */ + [48] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [49] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_SCTP, + [50] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_ICMP, + + /* IPv4 --> GRE/Teredo/VXLAN --> IPv6 */ + [51] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG, + [52] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG, + [53] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + /* [54] reserved */ + [55] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [56] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_SCTP, + [57] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_ICMP, + + /* IPv4 --> GRE/Teredo/VXLAN --> MAC */ + [58] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER, + + /* IPv4 --> GRE/Teredo/VXLAN --> MAC --> IPv4 */ + [59] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG, + [60] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG, + [61] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + /* [62] reserved */ + [63] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [64] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_SCTP, + [65] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_ICMP, + + /* IPv4 --> GRE/Teredo/VXLAN --> MAC --> IPv6 */ + [66] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG, + [67] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG, + [68] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + /* [69] reserved */ + [70] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [71] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_SCTP, + [72] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_ICMP, + /* [73] - [87] reserved */ + + /* Non tunneled IPv6 */ + [88] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_FRAG, + [89] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_NONFRAG, + [90] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_UDP, + /* [91] reserved */ + [92] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_TCP, + [93] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_SCTP, + [94] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_ICMP, + + /* IPv6 --> IPv4 */ + [95] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG, + [96] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG, + [97] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + /* [98] reserved */ + [99] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [100] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_SCTP, + [101] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_ICMP, + + /* IPv6 --> IPv6 */ + [102] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG, + [103] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG, + [104] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + /* [105] reserved */ + [106] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [107] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_SCTP, + [108] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_ICMP, + + /* IPv6 --> GRE/Teredo/VXLAN */ + [109] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT, + + /* IPv6 --> GRE/Teredo/VXLAN --> IPv4 */ + [110] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG, + [111] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG, + [112] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + /* [113] reserved */ + [114] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [115] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_SCTP, + [116] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_ICMP, + + /* IPv6 --> GRE/Teredo/VXLAN --> IPv6 */ + [117] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG, + [118] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG, + [119] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + /* [120] reserved */ + [121] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [122] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_SCTP, + [123] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_ICMP, + + /* IPv6 --> GRE/Teredo/VXLAN --> MAC */ + [124] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER, + + /* IPv6 --> GRE/Teredo/VXLAN --> MAC --> IPv4 */ + [125] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG, + [126] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG, + [127] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + /* [128] reserved */ + [129] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [130] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_SCTP, + [131] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_ICMP, + + /* IPv6 --> GRE/Teredo/VXLAN --> MAC --> IPv6 */ + [132] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG, + [133] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG, + [134] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + /* [135] reserved */ + [136] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [137] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_SCTP, + [138] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_ICMP, + /* [139] - [299] reserved */ + + /* PPPoE */ + [300] = RTE_PTYPE_L2_ETHER_PPPOE, + [301] = RTE_PTYPE_L2_ETHER_PPPOE, + + /* PPPoE --> IPv4 */ + [302] = RTE_PTYPE_L2_ETHER_PPPOE | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_FRAG, + [303] = RTE_PTYPE_L2_ETHER_PPPOE | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_NONFRAG, + [304] = RTE_PTYPE_L2_ETHER_PPPOE | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_UDP, + [305] = RTE_PTYPE_L2_ETHER_PPPOE | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_TCP, + [306] = RTE_PTYPE_L2_ETHER_PPPOE | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_SCTP, + [307] = RTE_PTYPE_L2_ETHER_PPPOE | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_ICMP, + + /* PPPoE --> IPv6 */ + [308] = RTE_PTYPE_L2_ETHER_PPPOE | + RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_FRAG, + [309] = RTE_PTYPE_L2_ETHER_PPPOE | + RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_NONFRAG, + [310] = RTE_PTYPE_L2_ETHER_PPPOE | + RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_UDP, + [311] = RTE_PTYPE_L2_ETHER_PPPOE | + RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_TCP, + [312] = RTE_PTYPE_L2_ETHER_PPPOE | + RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_SCTP, + [313] = RTE_PTYPE_L2_ETHER_PPPOE | + RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_ICMP, + /* [314] - [324] reserved */ + + /* IPv4/IPv6 --> GTPC/GTPU */ + [325] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GTPC, + [326] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GTPC, + [327] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GTPC, + [328] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GTPC, + [329] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GTPU, + [330] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GTPU, + + /* IPv4 --> GTPU --> IPv4 */ + [331] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GTPU | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG, + [332] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GTPU | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG, + [333] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GTPU | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + [334] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GTPU | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [335] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GTPU | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_ICMP, + + /* IPv6 --> GTPU --> IPv4 */ + [336] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GTPU | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG, + [337] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GTPU | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG, + [338] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GTPU | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + [339] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GTPU | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [340] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GTPU | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_ICMP, + + /* IPv4 --> GTPU --> IPv6 */ + [341] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GTPU | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG, + [342] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GTPU | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG, + [343] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GTPU | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + [344] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GTPU | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [345] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GTPU | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_ICMP, + + /* IPv6 --> GTPU --> IPv6 */ + [346] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GTPU | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG, + [347] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GTPU | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG, + [348] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GTPU | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + [349] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GTPU | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [350] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GTPU | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_ICMP, + /* All others reserved */ + }; + + return ptype_tbl; +} diff --git a/src/spdk/dpdk/drivers/net/iavf/iavf_rxtx.h b/src/spdk/dpdk/drivers/net/iavf/iavf_rxtx.h new file mode 100644 index 000000000..59625a979 --- /dev/null +++ b/src/spdk/dpdk/drivers/net/iavf/iavf_rxtx.h @@ -0,0 +1,534 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017 Intel Corporation + */ + +#ifndef _IAVF_RXTX_H_ +#define _IAVF_RXTX_H_ + +/* In QLEN must be whole number of 32 descriptors. */ +#define IAVF_ALIGN_RING_DESC 32 +#define IAVF_MIN_RING_DESC 64 +#define IAVF_MAX_RING_DESC 4096 +#define IAVF_DMA_MEM_ALIGN 4096 +/* Base address of the HW descriptor ring should be 128B aligned. */ +#define IAVF_RING_BASE_ALIGN 128 + +/* used for Rx Bulk Allocate */ +#define IAVF_RX_MAX_BURST 32 + +/* used for Vector PMD */ +#define IAVF_VPMD_RX_MAX_BURST 32 +#define IAVF_VPMD_TX_MAX_BURST 32 +#define IAVF_RXQ_REARM_THRESH 32 +#define IAVF_VPMD_DESCS_PER_LOOP 4 +#define IAVF_VPMD_TX_MAX_FREE_BUF 64 + +#define IAVF_NO_VECTOR_FLAGS ( \ + DEV_TX_OFFLOAD_MULTI_SEGS | \ + DEV_TX_OFFLOAD_VLAN_INSERT | \ + DEV_TX_OFFLOAD_SCTP_CKSUM | \ + DEV_TX_OFFLOAD_UDP_CKSUM | \ + DEV_TX_OFFLOAD_TCP_TSO | \ + DEV_TX_OFFLOAD_TCP_CKSUM) + +#define DEFAULT_TX_RS_THRESH 32 +#define DEFAULT_TX_FREE_THRESH 32 + +#define IAVF_MIN_TSO_MSS 256 +#define IAVF_MAX_TSO_MSS 9668 +#define IAVF_TSO_MAX_SEG UINT8_MAX +#define IAVF_TX_MAX_MTU_SEG 8 + +#define IAVF_TX_CKSUM_OFFLOAD_MASK ( \ + PKT_TX_IP_CKSUM | \ + PKT_TX_L4_MASK | \ + PKT_TX_TCP_SEG) + +#define IAVF_TX_OFFLOAD_MASK ( \ + PKT_TX_OUTER_IPV6 | \ + PKT_TX_OUTER_IPV4 | \ + PKT_TX_IPV6 | \ + PKT_TX_IPV4 | \ + PKT_TX_VLAN_PKT | \ + PKT_TX_IP_CKSUM | \ + PKT_TX_L4_MASK | \ + PKT_TX_TCP_SEG) + +#define IAVF_TX_OFFLOAD_NOTSUP_MASK \ + (PKT_TX_OFFLOAD_MASK ^ IAVF_TX_OFFLOAD_MASK) + +/* HW desc structure, both 16-byte and 32-byte types are supported */ +#ifdef RTE_LIBRTE_IAVF_16BYTE_RX_DESC +#define iavf_rx_desc iavf_16byte_rx_desc +#define iavf_rx_flex_desc iavf_16b_rx_flex_desc +#else +#define iavf_rx_desc iavf_32byte_rx_desc +#define iavf_rx_flex_desc iavf_32b_rx_flex_desc +#endif + +struct iavf_rxq_ops { + void (*release_mbufs)(struct iavf_rx_queue *rxq); +}; + +struct iavf_txq_ops { + void (*release_mbufs)(struct iavf_tx_queue *txq); +}; + +/* Structure associated with each Rx queue. */ +struct iavf_rx_queue { + struct rte_mempool *mp; /* mbuf pool to populate Rx ring */ + const struct rte_memzone *mz; /* memzone for Rx ring */ + volatile union iavf_rx_desc *rx_ring; /* Rx ring virtual address */ + uint64_t rx_ring_phys_addr; /* Rx ring DMA address */ + struct rte_mbuf **sw_ring; /* address of SW ring */ + uint16_t nb_rx_desc; /* ring length */ + uint16_t rx_tail; /* current value of tail */ + volatile uint8_t *qrx_tail; /* register address of tail */ + uint16_t rx_free_thresh; /* max free RX desc to hold */ + uint16_t nb_rx_hold; /* number of held free RX desc */ + struct rte_mbuf *pkt_first_seg; /* first segment of current packet */ + struct rte_mbuf *pkt_last_seg; /* last segment of current packet */ + struct rte_mbuf fake_mbuf; /* dummy mbuf */ + uint8_t rxdid; + + /* used for VPMD */ + uint16_t rxrearm_nb; /* number of remaining to be re-armed */ + uint16_t rxrearm_start; /* the idx we start the re-arming from */ + uint64_t mbuf_initializer; /* value to init mbufs */ + + /* for rx bulk */ + uint16_t rx_nb_avail; /* number of staged packets ready */ + uint16_t rx_next_avail; /* index of next staged packets */ + uint16_t rx_free_trigger; /* triggers rx buffer allocation */ + struct rte_mbuf *rx_stage[IAVF_RX_MAX_BURST * 2]; /* store mbuf */ + + uint16_t port_id; /* device port ID */ + uint8_t crc_len; /* 0 if CRC stripped, 4 otherwise */ + uint8_t fdir_enabled; /* 0 if FDIR disabled, 1 when enabled */ + uint16_t queue_id; /* Rx queue index */ + uint16_t rx_buf_len; /* The packet buffer size */ + uint16_t rx_hdr_len; /* The header buffer size */ + uint16_t max_pkt_len; /* Maximum packet length */ + struct iavf_vsi *vsi; /**< the VSI this queue belongs to */ + + bool q_set; /* if rx queue has been configured */ + bool rx_deferred_start; /* don't start this queue in dev start */ + const struct iavf_rxq_ops *ops; +}; + +struct iavf_tx_entry { + struct rte_mbuf *mbuf; + uint16_t next_id; + uint16_t last_id; +}; + +/* Structure associated with each TX queue. */ +struct iavf_tx_queue { + const struct rte_memzone *mz; /* memzone for Tx ring */ + volatile struct iavf_tx_desc *tx_ring; /* Tx ring virtual address */ + uint64_t tx_ring_phys_addr; /* Tx ring DMA address */ + struct iavf_tx_entry *sw_ring; /* address array of SW ring */ + uint16_t nb_tx_desc; /* ring length */ + uint16_t tx_tail; /* current value of tail */ + volatile uint8_t *qtx_tail; /* register address of tail */ + /* number of used desc since RS bit set */ + uint16_t nb_used; + uint16_t nb_free; + uint16_t last_desc_cleaned; /* last desc have been cleaned*/ + uint16_t free_thresh; + uint16_t rs_thresh; + + uint16_t port_id; + uint16_t queue_id; + uint64_t offloads; + uint16_t next_dd; /* next to set RS, for VPMD */ + uint16_t next_rs; /* next to check DD, for VPMD */ + + bool q_set; /* if rx queue has been configured */ + bool tx_deferred_start; /* don't start this queue in dev start */ + const struct iavf_txq_ops *ops; +}; + +/* Offload features */ +union iavf_tx_offload { + uint64_t data; + struct { + uint64_t l2_len:7; /* L2 (MAC) Header Length. */ + uint64_t l3_len:9; /* L3 (IP) Header Length. */ + uint64_t l4_len:8; /* L4 Header Length. */ + uint64_t tso_segsz:16; /* TCP TSO segment size */ + /* uint64_t unused : 24; */ + }; +}; + +/* Rx Flex Descriptors + * These descriptors are used instead of the legacy version descriptors + */ +union iavf_16b_rx_flex_desc { + struct { + __le64 pkt_addr; /* Packet buffer address */ + __le64 hdr_addr; /* Header buffer address */ + /* bit 0 of hdr_addr is DD bit */ + } read; + struct { + /* Qword 0 */ + u8 rxdid; /* descriptor builder profile ID */ + u8 mir_id_umb_cast; /* mirror=[5:0], umb=[7:6] */ + __le16 ptype_flex_flags0; /* ptype=[9:0], ff0=[15:10] */ + __le16 pkt_len; /* [15:14] are reserved */ + __le16 hdr_len_sph_flex_flags1; /* header=[10:0] */ + /* sph=[11:11] */ + /* ff1/ext=[15:12] */ + + /* Qword 1 */ + __le16 status_error0; + __le16 l2tag1; + __le16 flex_meta0; + __le16 flex_meta1; + } wb; /* writeback */ +}; + +union iavf_32b_rx_flex_desc { + struct { + __le64 pkt_addr; /* Packet buffer address */ + __le64 hdr_addr; /* Header buffer address */ + /* bit 0 of hdr_addr is DD bit */ + __le64 rsvd1; + __le64 rsvd2; + } read; + struct { + /* Qword 0 */ + u8 rxdid; /* descriptor builder profile ID */ + u8 mir_id_umb_cast; /* mirror=[5:0], umb=[7:6] */ + __le16 ptype_flex_flags0; /* ptype=[9:0], ff0=[15:10] */ + __le16 pkt_len; /* [15:14] are reserved */ + __le16 hdr_len_sph_flex_flags1; /* header=[10:0] */ + /* sph=[11:11] */ + /* ff1/ext=[15:12] */ + + /* Qword 1 */ + __le16 status_error0; + __le16 l2tag1; + __le16 flex_meta0; + __le16 flex_meta1; + + /* Qword 2 */ + __le16 status_error1; + u8 flex_flags2; + u8 time_stamp_low; + __le16 l2tag2_1st; + __le16 l2tag2_2nd; + + /* Qword 3 */ + __le16 flex_meta2; + __le16 flex_meta3; + union { + struct { + __le16 flex_meta4; + __le16 flex_meta5; + } flex; + __le32 ts_high; + } flex_ts; + } wb; /* writeback */ +}; + +/* Rx Flex Descriptor + * RxDID Profile ID 16-21 + * Flex-field 0: RSS hash lower 16-bits + * Flex-field 1: RSS hash upper 16-bits + * Flex-field 2: Flow ID lower 16-bits + * Flex-field 3: Flow ID upper 16-bits + * Flex-field 4: AUX0 + * Flex-field 5: AUX1 + */ +struct iavf_32b_rx_flex_desc_comms { + /* Qword 0 */ + u8 rxdid; + u8 mir_id_umb_cast; + __le16 ptype_flexi_flags0; + __le16 pkt_len; + __le16 hdr_len_sph_flex_flags1; + + /* Qword 1 */ + __le16 status_error0; + __le16 l2tag1; + __le32 rss_hash; + + /* Qword 2 */ + __le16 status_error1; + u8 flexi_flags2; + u8 ts_low; + __le16 l2tag2_1st; + __le16 l2tag2_2nd; + + /* Qword 3 */ + __le32 flow_id; + union { + struct { + __le16 aux0; + __le16 aux1; + } flex; + __le32 ts_high; + } flex_ts; +}; + +/* Rx Flex Descriptor + * RxDID Profile ID 22-23 (swap Hash and FlowID) + * Flex-field 0: Flow ID lower 16-bits + * Flex-field 1: Flow ID upper 16-bits + * Flex-field 2: RSS hash lower 16-bits + * Flex-field 3: RSS hash upper 16-bits + * Flex-field 4: AUX0 + * Flex-field 5: AUX1 + */ +struct iavf_32b_rx_flex_desc_comms_ovs { + /* Qword 0 */ + u8 rxdid; + u8 mir_id_umb_cast; + __le16 ptype_flexi_flags0; + __le16 pkt_len; + __le16 hdr_len_sph_flex_flags1; + + /* Qword 1 */ + __le16 status_error0; + __le16 l2tag1; + __le32 flow_id; + + /* Qword 2 */ + __le16 status_error1; + u8 flexi_flags2; + u8 ts_low; + __le16 l2tag2_1st; + __le16 l2tag2_2nd; + + /* Qword 3 */ + __le32 rss_hash; + union { + struct { + __le16 aux0; + __le16 aux1; + } flex; + __le32 ts_high; + } flex_ts; +}; + +/* Receive Flex Descriptor profile IDs: There are a total + * of 64 profiles where profile IDs 0/1 are for legacy; and + * profiles 2-63 are flex profiles that can be programmed + * with a specific metadata (profile 7 reserved for HW) + */ +enum iavf_rxdid { + IAVF_RXDID_LEGACY_0 = 0, + IAVF_RXDID_LEGACY_1 = 1, + IAVF_RXDID_FLEX_NIC = 2, + IAVF_RXDID_FLEX_NIC_2 = 6, + IAVF_RXDID_HW = 7, + IAVF_RXDID_COMMS_GENERIC = 16, + IAVF_RXDID_COMMS_AUX_VLAN = 17, + IAVF_RXDID_COMMS_AUX_IPV4 = 18, + IAVF_RXDID_COMMS_AUX_IPV6 = 19, + IAVF_RXDID_COMMS_AUX_IPV6_FLOW = 20, + IAVF_RXDID_COMMS_AUX_TCP = 21, + IAVF_RXDID_COMMS_OVS_1 = 22, + IAVF_RXDID_COMMS_OVS_2 = 23, + IAVF_RXDID_LAST = 63, +}; + +enum iavf_rx_flex_desc_status_error_0_bits { + /* Note: These are predefined bit offsets */ + IAVF_RX_FLEX_DESC_STATUS0_DD_S = 0, + IAVF_RX_FLEX_DESC_STATUS0_EOF_S, + IAVF_RX_FLEX_DESC_STATUS0_HBO_S, + IAVF_RX_FLEX_DESC_STATUS0_L3L4P_S, + IAVF_RX_FLEX_DESC_STATUS0_XSUM_IPE_S, + IAVF_RX_FLEX_DESC_STATUS0_XSUM_L4E_S, + IAVF_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S, + IAVF_RX_FLEX_DESC_STATUS0_XSUM_EUDPE_S, + IAVF_RX_FLEX_DESC_STATUS0_LPBK_S, + IAVF_RX_FLEX_DESC_STATUS0_IPV6EXADD_S, + IAVF_RX_FLEX_DESC_STATUS0_RXE_S, + IAVF_RX_FLEX_DESC_STATUS0_CRCP_S, + IAVF_RX_FLEX_DESC_STATUS0_RSS_VALID_S, + IAVF_RX_FLEX_DESC_STATUS0_L2TAG1P_S, + IAVF_RX_FLEX_DESC_STATUS0_XTRMD0_VALID_S, + IAVF_RX_FLEX_DESC_STATUS0_XTRMD1_VALID_S, + IAVF_RX_FLEX_DESC_STATUS0_LAST /* this entry must be last!!! */ +}; + +/* for iavf_32b_rx_flex_desc.ptype_flex_flags0 member */ +#define IAVF_RX_FLEX_DESC_PTYPE_M (0x3FF) /* 10-bits */ + +/* for iavf_32b_rx_flex_desc.pkt_len member */ +#define IAVF_RX_FLX_DESC_PKT_LEN_M (0x3FFF) /* 14-bits */ + +int iavf_dev_rx_queue_setup(struct rte_eth_dev *dev, + uint16_t queue_idx, + uint16_t nb_desc, + unsigned int socket_id, + const struct rte_eth_rxconf *rx_conf, + struct rte_mempool *mp); + +int iavf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id); +int iavf_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id); +void iavf_dev_rx_queue_release(void *rxq); + +int iavf_dev_tx_queue_setup(struct rte_eth_dev *dev, + uint16_t queue_idx, + uint16_t nb_desc, + unsigned int socket_id, + const struct rte_eth_txconf *tx_conf); +int iavf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id); +int iavf_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id); +void iavf_dev_tx_queue_release(void *txq); +void iavf_stop_queues(struct rte_eth_dev *dev); +uint16_t iavf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts); +uint16_t iavf_recv_pkts_flex_rxd(void *rx_queue, + struct rte_mbuf **rx_pkts, + uint16_t nb_pkts); +uint16_t iavf_recv_scattered_pkts(void *rx_queue, + struct rte_mbuf **rx_pkts, + uint16_t nb_pkts); +uint16_t iavf_recv_scattered_pkts_flex_rxd(void *rx_queue, + struct rte_mbuf **rx_pkts, + uint16_t nb_pkts); +uint16_t iavf_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts); +uint16_t iavf_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts); +void iavf_set_rx_function(struct rte_eth_dev *dev); +void iavf_set_tx_function(struct rte_eth_dev *dev); +void iavf_dev_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, + struct rte_eth_rxq_info *qinfo); +void iavf_dev_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, + struct rte_eth_txq_info *qinfo); +uint32_t iavf_dev_rxq_count(struct rte_eth_dev *dev, uint16_t queue_id); +int iavf_dev_rx_desc_status(void *rx_queue, uint16_t offset); +int iavf_dev_tx_desc_status(void *tx_queue, uint16_t offset); + +uint16_t iavf_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts); +uint16_t iavf_recv_pkts_vec_flex_rxd(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts); +uint16_t iavf_recv_scattered_pkts_vec(void *rx_queue, + struct rte_mbuf **rx_pkts, + uint16_t nb_pkts); +uint16_t iavf_recv_scattered_pkts_vec_flex_rxd(void *rx_queue, + struct rte_mbuf **rx_pkts, + uint16_t nb_pkts); +uint16_t iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts); +uint16_t iavf_recv_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts); +uint16_t iavf_recv_pkts_vec_avx2_flex_rxd(void *rx_queue, + struct rte_mbuf **rx_pkts, + uint16_t nb_pkts); +uint16_t iavf_recv_scattered_pkts_vec_avx2(void *rx_queue, + struct rte_mbuf **rx_pkts, + uint16_t nb_pkts); +uint16_t iavf_recv_scattered_pkts_vec_avx2_flex_rxd(void *rx_queue, + struct rte_mbuf **rx_pkts, + uint16_t nb_pkts); +uint16_t iavf_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts); +uint16_t iavf_xmit_pkts_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts); +int iavf_rx_vec_dev_check(struct rte_eth_dev *dev); +int iavf_tx_vec_dev_check(struct rte_eth_dev *dev); +int iavf_rxq_vec_setup(struct iavf_rx_queue *rxq); +int iavf_txq_vec_setup(struct iavf_tx_queue *txq); + +const uint32_t *iavf_get_default_ptype_table(void); + +static inline +void iavf_dump_rx_descriptor(struct iavf_rx_queue *rxq, + const volatile void *desc, + uint16_t rx_id) +{ +#ifdef RTE_LIBRTE_IAVF_16BYTE_RX_DESC + const volatile union iavf_16byte_rx_desc *rx_desc = desc; + + printf("Queue %d Rx_desc %d: QW0: 0x%016"PRIx64" QW1: 0x%016"PRIx64"\n", + rxq->queue_id, rx_id, rx_desc->read.pkt_addr, + rx_desc->read.hdr_addr); +#else + const volatile union iavf_32byte_rx_desc *rx_desc = desc; + + printf("Queue %d Rx_desc %d: QW0: 0x%016"PRIx64" QW1: 0x%016"PRIx64 + " QW2: 0x%016"PRIx64" QW3: 0x%016"PRIx64"\n", rxq->queue_id, + rx_id, rx_desc->read.pkt_addr, rx_desc->read.hdr_addr, + rx_desc->read.rsvd1, rx_desc->read.rsvd2); +#endif +} + +/* All the descriptors are 16 bytes, so just use one of them + * to print the qwords + */ +static inline +void iavf_dump_tx_descriptor(const struct iavf_tx_queue *txq, + const volatile void *desc, uint16_t tx_id) +{ + const char *name; + const volatile struct iavf_tx_desc *tx_desc = desc; + enum iavf_tx_desc_dtype_value type; + + type = (enum iavf_tx_desc_dtype_value)rte_le_to_cpu_64( + tx_desc->cmd_type_offset_bsz & + rte_cpu_to_le_64(IAVF_TXD_QW1_DTYPE_MASK)); + switch (type) { + case IAVF_TX_DESC_DTYPE_DATA: + name = "Tx_data_desc"; + break; + case IAVF_TX_DESC_DTYPE_CONTEXT: + name = "Tx_context_desc"; + break; + default: + name = "unknown_desc"; + break; + } + + printf("Queue %d %s %d: QW0: 0x%016"PRIx64" QW1: 0x%016"PRIx64"\n", + txq->queue_id, name, tx_id, tx_desc->buffer_addr, + tx_desc->cmd_type_offset_bsz); +} + +#define FDIR_PROC_ENABLE_PER_QUEUE(ad, on) do { \ + int i; \ + for (i = 0; i < (ad)->eth_dev->data->nb_rx_queues; i++) { \ + struct iavf_rx_queue *rxq = (ad)->eth_dev->data->rx_queues[i]; \ + if (!rxq) \ + continue; \ + rxq->fdir_enabled = on; \ + } \ + PMD_DRV_LOG(DEBUG, "FDIR processing on RX set to %d", on); \ +} while (0) + +/* Enable/disable flow director Rx processing in data path. */ +static inline +void iavf_fdir_rx_proc_enable(struct iavf_adapter *ad, bool on) +{ + if (on) { + /* enable flow director processing */ + FDIR_PROC_ENABLE_PER_QUEUE(ad, on); + ad->fdir_ref_cnt++; + } else { + if (ad->fdir_ref_cnt >= 1) { + ad->fdir_ref_cnt--; + + if (ad->fdir_ref_cnt == 0) + FDIR_PROC_ENABLE_PER_QUEUE(ad, on); + } + } +} + +#ifdef RTE_LIBRTE_IAVF_DEBUG_DUMP_DESC +#define IAVF_DUMP_RX_DESC(rxq, desc, rx_id) \ + iavf_dump_rx_descriptor(rxq, desc, rx_id) +#define IAVF_DUMP_TX_DESC(txq, desc, tx_id) \ + iavf_dump_tx_descriptor(txq, desc, tx_id) +#else +#define IAVF_DUMP_RX_DESC(rxq, desc, rx_id) do { } while (0) +#define IAVF_DUMP_TX_DESC(txq, desc, tx_id) do { } while (0) +#endif + +#endif /* _IAVF_RXTX_H_ */ diff --git a/src/spdk/dpdk/drivers/net/iavf/iavf_rxtx_vec_avx2.c b/src/spdk/dpdk/drivers/net/iavf/iavf_rxtx_vec_avx2.c new file mode 100644 index 000000000..e5e0fd309 --- /dev/null +++ b/src/spdk/dpdk/drivers/net/iavf/iavf_rxtx_vec_avx2.c @@ -0,0 +1,1541 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2019 Intel Corporation + */ + +#include "iavf_rxtx_vec_common.h" + +#include + +#ifndef __INTEL_COMPILER +#pragma GCC diagnostic ignored "-Wcast-qual" +#endif + +static inline void +iavf_rxq_rearm(struct iavf_rx_queue *rxq) +{ + int i; + uint16_t rx_id; + volatile union iavf_rx_desc *rxdp; + struct rte_mbuf **rxp = &rxq->sw_ring[rxq->rxrearm_start]; + + rxdp = rxq->rx_ring + rxq->rxrearm_start; + + /* Pull 'n' more MBUFs into the software ring */ + if (rte_mempool_get_bulk(rxq->mp, + (void *)rxp, + IAVF_RXQ_REARM_THRESH) < 0) { + if (rxq->rxrearm_nb + IAVF_RXQ_REARM_THRESH >= + rxq->nb_rx_desc) { + __m128i dma_addr0; + + dma_addr0 = _mm_setzero_si128(); + for (i = 0; i < IAVF_VPMD_DESCS_PER_LOOP; i++) { + rxp[i] = &rxq->fake_mbuf; + _mm_store_si128((__m128i *)&rxdp[i].read, + dma_addr0); + } + } + rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed += + IAVF_RXQ_REARM_THRESH; + return; + } + +#ifndef RTE_LIBRTE_IAVF_16BYTE_RX_DESC + struct rte_mbuf *mb0, *mb1; + __m128i dma_addr0, dma_addr1; + __m128i hdr_room = _mm_set_epi64x(RTE_PKTMBUF_HEADROOM, + RTE_PKTMBUF_HEADROOM); + /* Initialize the mbufs in vector, process 2 mbufs in one loop */ + for (i = 0; i < IAVF_RXQ_REARM_THRESH; i += 2, rxp += 2) { + __m128i vaddr0, vaddr1; + + mb0 = rxp[0]; + mb1 = rxp[1]; + + /* load buf_addr(lo 64bit) and buf_physaddr(hi 64bit) */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_physaddr) != + offsetof(struct rte_mbuf, buf_addr) + 8); + vaddr0 = _mm_loadu_si128((__m128i *)&mb0->buf_addr); + vaddr1 = _mm_loadu_si128((__m128i *)&mb1->buf_addr); + + /* convert pa to dma_addr hdr/data */ + dma_addr0 = _mm_unpackhi_epi64(vaddr0, vaddr0); + dma_addr1 = _mm_unpackhi_epi64(vaddr1, vaddr1); + + /* add headroom to pa values */ + dma_addr0 = _mm_add_epi64(dma_addr0, hdr_room); + dma_addr1 = _mm_add_epi64(dma_addr1, hdr_room); + + /* flush desc with pa dma_addr */ + _mm_store_si128((__m128i *)&rxdp++->read, dma_addr0); + _mm_store_si128((__m128i *)&rxdp++->read, dma_addr1); + } +#else + struct rte_mbuf *mb0, *mb1, *mb2, *mb3; + __m256i dma_addr0_1, dma_addr2_3; + __m256i hdr_room = _mm256_set1_epi64x(RTE_PKTMBUF_HEADROOM); + /* Initialize the mbufs in vector, process 4 mbufs in one loop */ + for (i = 0; i < IAVF_RXQ_REARM_THRESH; + i += 4, rxp += 4, rxdp += 4) { + __m128i vaddr0, vaddr1, vaddr2, vaddr3; + __m256i vaddr0_1, vaddr2_3; + + mb0 = rxp[0]; + mb1 = rxp[1]; + mb2 = rxp[2]; + mb3 = rxp[3]; + + /* load buf_addr(lo 64bit) and buf_physaddr(hi 64bit) */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_physaddr) != + offsetof(struct rte_mbuf, buf_addr) + 8); + vaddr0 = _mm_loadu_si128((__m128i *)&mb0->buf_addr); + vaddr1 = _mm_loadu_si128((__m128i *)&mb1->buf_addr); + vaddr2 = _mm_loadu_si128((__m128i *)&mb2->buf_addr); + vaddr3 = _mm_loadu_si128((__m128i *)&mb3->buf_addr); + + /** + * merge 0 & 1, by casting 0 to 256-bit and inserting 1 + * into the high lanes. Similarly for 2 & 3 + */ + vaddr0_1 = + _mm256_inserti128_si256(_mm256_castsi128_si256(vaddr0), + vaddr1, 1); + vaddr2_3 = + _mm256_inserti128_si256(_mm256_castsi128_si256(vaddr2), + vaddr3, 1); + + /* convert pa to dma_addr hdr/data */ + dma_addr0_1 = _mm256_unpackhi_epi64(vaddr0_1, vaddr0_1); + dma_addr2_3 = _mm256_unpackhi_epi64(vaddr2_3, vaddr2_3); + + /* add headroom to pa values */ + dma_addr0_1 = _mm256_add_epi64(dma_addr0_1, hdr_room); + dma_addr2_3 = _mm256_add_epi64(dma_addr2_3, hdr_room); + + /* flush desc with pa dma_addr */ + _mm256_store_si256((__m256i *)&rxdp->read, dma_addr0_1); + _mm256_store_si256((__m256i *)&(rxdp + 2)->read, dma_addr2_3); + } + +#endif + + rxq->rxrearm_start += IAVF_RXQ_REARM_THRESH; + if (rxq->rxrearm_start >= rxq->nb_rx_desc) + rxq->rxrearm_start = 0; + + rxq->rxrearm_nb -= IAVF_RXQ_REARM_THRESH; + + rx_id = (uint16_t)((rxq->rxrearm_start == 0) ? + (rxq->nb_rx_desc - 1) : (rxq->rxrearm_start - 1)); + + /* Update the tail pointer on the NIC */ + IAVF_PCI_REG_WRITE(rxq->qrx_tail, rx_id); +} + +#define PKTLEN_SHIFT 10 + +static inline uint16_t +_iavf_recv_raw_pkts_vec_avx2(struct iavf_rx_queue *rxq, + struct rte_mbuf **rx_pkts, + uint16_t nb_pkts, uint8_t *split_packet) +{ +#define IAVF_DESCS_PER_LOOP_AVX 8 + + /* const uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl; */ + const uint32_t *type_table = rxq->vsi->adapter->ptype_tbl; + + const __m256i mbuf_init = _mm256_set_epi64x(0, 0, + 0, rxq->mbuf_initializer); + /* struct iavf_rx_entry *sw_ring = &rxq->sw_ring[rxq->rx_tail]; */ + struct rte_mbuf **sw_ring = &rxq->sw_ring[rxq->rx_tail]; + volatile union iavf_rx_desc *rxdp = rxq->rx_ring + rxq->rx_tail; + const int avx_aligned = ((rxq->rx_tail & 1) == 0); + + rte_prefetch0(rxdp); + + /* nb_pkts has to be floor-aligned to IAVF_DESCS_PER_LOOP_AVX */ + nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, IAVF_DESCS_PER_LOOP_AVX); + + /* See if we need to rearm the RX queue - gives the prefetch a bit + * of time to act + */ + if (rxq->rxrearm_nb > IAVF_RXQ_REARM_THRESH) + iavf_rxq_rearm(rxq); + + /* Before we start moving massive data around, check to see if + * there is actually a packet available + */ + if (!(rxdp->wb.qword1.status_error_len & + rte_cpu_to_le_32(1 << IAVF_RX_DESC_STATUS_DD_SHIFT))) + return 0; + + /* constants used in processing loop */ + const __m256i crc_adjust = + _mm256_set_epi16 + (/* first descriptor */ + 0, 0, 0, /* ignore non-length fields */ + -rxq->crc_len, /* sub crc on data_len */ + 0, /* ignore high-16bits of pkt_len */ + -rxq->crc_len, /* sub crc on pkt_len */ + 0, 0, /* ignore pkt_type field */ + /* second descriptor */ + 0, 0, 0, /* ignore non-length fields */ + -rxq->crc_len, /* sub crc on data_len */ + 0, /* ignore high-16bits of pkt_len */ + -rxq->crc_len, /* sub crc on pkt_len */ + 0, 0 /* ignore pkt_type field */ + ); + + /* 8 packets DD mask, LSB in each 32-bit value */ + const __m256i dd_check = _mm256_set1_epi32(1); + + /* 8 packets EOP mask, second-LSB in each 32-bit value */ + const __m256i eop_check = _mm256_slli_epi32(dd_check, + IAVF_RX_DESC_STATUS_EOF_SHIFT); + + /* mask to shuffle from desc. to mbuf (2 descriptors)*/ + const __m256i shuf_msk = + _mm256_set_epi8 + (/* first descriptor */ + 7, 6, 5, 4, /* octet 4~7, 32bits rss */ + 3, 2, /* octet 2~3, low 16 bits vlan_macip */ + 15, 14, /* octet 15~14, 16 bits data_len */ + 0xFF, 0xFF, /* skip high 16 bits pkt_len, zero out */ + 15, 14, /* octet 15~14, low 16 bits pkt_len */ + 0xFF, 0xFF, /* pkt_type set as unknown */ + 0xFF, 0xFF, /*pkt_type set as unknown */ + /* second descriptor */ + 7, 6, 5, 4, /* octet 4~7, 32bits rss */ + 3, 2, /* octet 2~3, low 16 bits vlan_macip */ + 15, 14, /* octet 15~14, 16 bits data_len */ + 0xFF, 0xFF, /* skip high 16 bits pkt_len, zero out */ + 15, 14, /* octet 15~14, low 16 bits pkt_len */ + 0xFF, 0xFF, /* pkt_type set as unknown */ + 0xFF, 0xFF /*pkt_type set as unknown */ + ); + /** + * compile-time check the above crc and shuffle layout is correct. + * NOTE: the first field (lowest address) is given last in set_epi + * calls above. + */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, vlan_tci) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 10); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, hash) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 12); + + /* Status/Error flag masks */ + /** + * mask everything except RSS, flow director and VLAN flags + * bit2 is for VLAN tag, bit11 for flow director indication + * bit13:12 for RSS indication. Bits 3-5 of error + * field (bits 22-24) are for IP/L4 checksum errors + */ + const __m256i flags_mask = + _mm256_set1_epi32((1 << 2) | (1 << 11) | + (3 << 12) | (7 << 22)); + /** + * data to be shuffled by result of flag mask. If VLAN bit is set, + * (bit 2), then position 4 in this array will be used in the + * destination + */ + const __m256i vlan_flags_shuf = + _mm256_set_epi32(0, 0, PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED, 0, + 0, 0, PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED, 0); + /** + * data to be shuffled by result of flag mask, shifted down 11. + * If RSS/FDIR bits are set, shuffle moves appropriate flags in + * place. + */ + const __m256i rss_flags_shuf = + _mm256_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, + PKT_RX_RSS_HASH | PKT_RX_FDIR, PKT_RX_RSS_HASH, + 0, 0, 0, 0, PKT_RX_FDIR, 0,/* end up 128-bits */ + 0, 0, 0, 0, 0, 0, 0, 0, + PKT_RX_RSS_HASH | PKT_RX_FDIR, PKT_RX_RSS_HASH, + 0, 0, 0, 0, PKT_RX_FDIR, 0); + + /** + * data to be shuffled by the result of the flags mask shifted by 22 + * bits. This gives use the l3_l4 flags. + */ + const __m256i l3_l4_flags_shuf = _mm256_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, + /* shift right 1 bit to make sure it not exceed 255 */ + (PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD | + PKT_RX_IP_CKSUM_BAD) >> 1, + (PKT_RX_IP_CKSUM_GOOD | PKT_RX_EIP_CKSUM_BAD | + PKT_RX_L4_CKSUM_BAD) >> 1, + (PKT_RX_EIP_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD) >> 1, + (PKT_RX_IP_CKSUM_GOOD | PKT_RX_EIP_CKSUM_BAD) >> 1, + (PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD) >> 1, + (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD) >> 1, + PKT_RX_IP_CKSUM_BAD >> 1, + (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD) >> 1, + /* second 128-bits */ + 0, 0, 0, 0, 0, 0, 0, 0, + (PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD | + PKT_RX_IP_CKSUM_BAD) >> 1, + (PKT_RX_IP_CKSUM_GOOD | PKT_RX_EIP_CKSUM_BAD | + PKT_RX_L4_CKSUM_BAD) >> 1, + (PKT_RX_EIP_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD) >> 1, + (PKT_RX_IP_CKSUM_GOOD | PKT_RX_EIP_CKSUM_BAD) >> 1, + (PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD) >> 1, + (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD) >> 1, + PKT_RX_IP_CKSUM_BAD >> 1, + (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD) >> 1); + + const __m256i cksum_mask = + _mm256_set1_epi32(PKT_RX_IP_CKSUM_GOOD | PKT_RX_IP_CKSUM_BAD | + PKT_RX_L4_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD | + PKT_RX_EIP_CKSUM_BAD); + + RTE_SET_USED(avx_aligned); /* for 32B descriptors we don't use this */ + + uint16_t i, received; + + for (i = 0, received = 0; i < nb_pkts; + i += IAVF_DESCS_PER_LOOP_AVX, + rxdp += IAVF_DESCS_PER_LOOP_AVX) { + /* step 1, copy over 8 mbuf pointers to rx_pkts array */ + _mm256_storeu_si256((void *)&rx_pkts[i], + _mm256_loadu_si256((void *)&sw_ring[i])); +#ifdef RTE_ARCH_X86_64 + _mm256_storeu_si256 + ((void *)&rx_pkts[i + 4], + _mm256_loadu_si256((void *)&sw_ring[i + 4])); +#endif + + __m256i raw_desc0_1, raw_desc2_3, raw_desc4_5, raw_desc6_7; +#ifdef RTE_LIBRTE_IAVF_16BYTE_RX_DESC + /* for AVX we need alignment otherwise loads are not atomic */ + if (avx_aligned) { + /* load in descriptors, 2 at a time, in reverse order */ + raw_desc6_7 = _mm256_load_si256((void *)(rxdp + 6)); + rte_compiler_barrier(); + raw_desc4_5 = _mm256_load_si256((void *)(rxdp + 4)); + rte_compiler_barrier(); + raw_desc2_3 = _mm256_load_si256((void *)(rxdp + 2)); + rte_compiler_barrier(); + raw_desc0_1 = _mm256_load_si256((void *)(rxdp + 0)); + } else +#endif + { + const __m128i raw_desc7 = + _mm_load_si128((void *)(rxdp + 7)); + rte_compiler_barrier(); + const __m128i raw_desc6 = + _mm_load_si128((void *)(rxdp + 6)); + rte_compiler_barrier(); + const __m128i raw_desc5 = + _mm_load_si128((void *)(rxdp + 5)); + rte_compiler_barrier(); + const __m128i raw_desc4 = + _mm_load_si128((void *)(rxdp + 4)); + rte_compiler_barrier(); + const __m128i raw_desc3 = + _mm_load_si128((void *)(rxdp + 3)); + rte_compiler_barrier(); + const __m128i raw_desc2 = + _mm_load_si128((void *)(rxdp + 2)); + rte_compiler_barrier(); + const __m128i raw_desc1 = + _mm_load_si128((void *)(rxdp + 1)); + rte_compiler_barrier(); + const __m128i raw_desc0 = + _mm_load_si128((void *)(rxdp + 0)); + + raw_desc6_7 = + _mm256_inserti128_si256 + (_mm256_castsi128_si256(raw_desc6), + raw_desc7, 1); + raw_desc4_5 = + _mm256_inserti128_si256 + (_mm256_castsi128_si256(raw_desc4), + raw_desc5, 1); + raw_desc2_3 = + _mm256_inserti128_si256 + (_mm256_castsi128_si256(raw_desc2), + raw_desc3, 1); + raw_desc0_1 = + _mm256_inserti128_si256 + (_mm256_castsi128_si256(raw_desc0), + raw_desc1, 1); + } + + if (split_packet) { + int j; + + for (j = 0; j < IAVF_DESCS_PER_LOOP_AVX; j++) + rte_mbuf_prefetch_part2(rx_pkts[i + j]); + } + + /** + * convert descriptors 4-7 into mbufs, adjusting length and + * re-arranging fields. Then write into the mbuf + */ + const __m256i len6_7 = _mm256_slli_epi32(raw_desc6_7, + PKTLEN_SHIFT); + const __m256i len4_5 = _mm256_slli_epi32(raw_desc4_5, + PKTLEN_SHIFT); + const __m256i desc6_7 = _mm256_blend_epi16(raw_desc6_7, + len6_7, 0x80); + const __m256i desc4_5 = _mm256_blend_epi16(raw_desc4_5, + len4_5, 0x80); + __m256i mb6_7 = _mm256_shuffle_epi8(desc6_7, shuf_msk); + __m256i mb4_5 = _mm256_shuffle_epi8(desc4_5, shuf_msk); + + mb6_7 = _mm256_add_epi16(mb6_7, crc_adjust); + mb4_5 = _mm256_add_epi16(mb4_5, crc_adjust); + /** + * to get packet types, shift 64-bit values down 30 bits + * and so ptype is in lower 8-bits in each + */ + const __m256i ptypes6_7 = _mm256_srli_epi64(desc6_7, 30); + const __m256i ptypes4_5 = _mm256_srli_epi64(desc4_5, 30); + const uint8_t ptype7 = _mm256_extract_epi8(ptypes6_7, 24); + const uint8_t ptype6 = _mm256_extract_epi8(ptypes6_7, 8); + const uint8_t ptype5 = _mm256_extract_epi8(ptypes4_5, 24); + const uint8_t ptype4 = _mm256_extract_epi8(ptypes4_5, 8); + + mb6_7 = _mm256_insert_epi32(mb6_7, type_table[ptype7], 4); + mb6_7 = _mm256_insert_epi32(mb6_7, type_table[ptype6], 0); + mb4_5 = _mm256_insert_epi32(mb4_5, type_table[ptype5], 4); + mb4_5 = _mm256_insert_epi32(mb4_5, type_table[ptype4], 0); + /* merge the status bits into one register */ + const __m256i status4_7 = _mm256_unpackhi_epi32(desc6_7, + desc4_5); + + /** + * convert descriptors 0-3 into mbufs, adjusting length and + * re-arranging fields. Then write into the mbuf + */ + const __m256i len2_3 = _mm256_slli_epi32(raw_desc2_3, + PKTLEN_SHIFT); + const __m256i len0_1 = _mm256_slli_epi32(raw_desc0_1, + PKTLEN_SHIFT); + const __m256i desc2_3 = _mm256_blend_epi16(raw_desc2_3, + len2_3, 0x80); + const __m256i desc0_1 = _mm256_blend_epi16(raw_desc0_1, + len0_1, 0x80); + __m256i mb2_3 = _mm256_shuffle_epi8(desc2_3, shuf_msk); + __m256i mb0_1 = _mm256_shuffle_epi8(desc0_1, shuf_msk); + + mb2_3 = _mm256_add_epi16(mb2_3, crc_adjust); + mb0_1 = _mm256_add_epi16(mb0_1, crc_adjust); + /* get the packet types */ + const __m256i ptypes2_3 = _mm256_srli_epi64(desc2_3, 30); + const __m256i ptypes0_1 = _mm256_srli_epi64(desc0_1, 30); + const uint8_t ptype3 = _mm256_extract_epi8(ptypes2_3, 24); + const uint8_t ptype2 = _mm256_extract_epi8(ptypes2_3, 8); + const uint8_t ptype1 = _mm256_extract_epi8(ptypes0_1, 24); + const uint8_t ptype0 = _mm256_extract_epi8(ptypes0_1, 8); + + mb2_3 = _mm256_insert_epi32(mb2_3, type_table[ptype3], 4); + mb2_3 = _mm256_insert_epi32(mb2_3, type_table[ptype2], 0); + mb0_1 = _mm256_insert_epi32(mb0_1, type_table[ptype1], 4); + mb0_1 = _mm256_insert_epi32(mb0_1, type_table[ptype0], 0); + /* merge the status bits into one register */ + const __m256i status0_3 = _mm256_unpackhi_epi32(desc2_3, + desc0_1); + + /** + * take the two sets of status bits and merge to one + * After merge, the packets status flags are in the + * order (hi->lo): [1, 3, 5, 7, 0, 2, 4, 6] + */ + __m256i status0_7 = _mm256_unpacklo_epi64(status4_7, + status0_3); + + /* now do flag manipulation */ + + /* get only flag/error bits we want */ + const __m256i flag_bits = + _mm256_and_si256(status0_7, flags_mask); + /* set vlan and rss flags */ + const __m256i vlan_flags = + _mm256_shuffle_epi8(vlan_flags_shuf, flag_bits); + const __m256i rss_flags = + _mm256_shuffle_epi8(rss_flags_shuf, + _mm256_srli_epi32(flag_bits, 11)); + /** + * l3_l4_error flags, shuffle, then shift to correct adjustment + * of flags in flags_shuf, and finally mask out extra bits + */ + __m256i l3_l4_flags = _mm256_shuffle_epi8(l3_l4_flags_shuf, + _mm256_srli_epi32(flag_bits, 22)); + l3_l4_flags = _mm256_slli_epi32(l3_l4_flags, 1); + l3_l4_flags = _mm256_and_si256(l3_l4_flags, cksum_mask); + + /* merge flags */ + const __m256i mbuf_flags = _mm256_or_si256(l3_l4_flags, + _mm256_or_si256(rss_flags, vlan_flags)); + /** + * At this point, we have the 8 sets of flags in the low 16-bits + * of each 32-bit value in vlan0. + * We want to extract these, and merge them with the mbuf init + * data so we can do a single write to the mbuf to set the flags + * and all the other initialization fields. Extracting the + * appropriate flags means that we have to do a shift and blend + * for each mbuf before we do the write. However, we can also + * add in the previously computed rx_descriptor fields to + * make a single 256-bit write per mbuf + */ + /* check the structure matches expectations */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, ol_flags) != + offsetof(struct rte_mbuf, rearm_data) + 8); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rearm_data) != + RTE_ALIGN(offsetof(struct rte_mbuf, + rearm_data), + 16)); + /* build up data and do writes */ + __m256i rearm0, rearm1, rearm2, rearm3, rearm4, rearm5, + rearm6, rearm7; + rearm6 = _mm256_blend_epi32(mbuf_init, + _mm256_slli_si256(mbuf_flags, 8), + 0x04); + rearm4 = _mm256_blend_epi32(mbuf_init, + _mm256_slli_si256(mbuf_flags, 4), + 0x04); + rearm2 = _mm256_blend_epi32(mbuf_init, mbuf_flags, 0x04); + rearm0 = _mm256_blend_epi32(mbuf_init, + _mm256_srli_si256(mbuf_flags, 4), + 0x04); + /* permute to add in the rx_descriptor e.g. rss fields */ + rearm6 = _mm256_permute2f128_si256(rearm6, mb6_7, 0x20); + rearm4 = _mm256_permute2f128_si256(rearm4, mb4_5, 0x20); + rearm2 = _mm256_permute2f128_si256(rearm2, mb2_3, 0x20); + rearm0 = _mm256_permute2f128_si256(rearm0, mb0_1, 0x20); + /* write to mbuf */ + _mm256_storeu_si256((__m256i *)&rx_pkts[i + 6]->rearm_data, + rearm6); + _mm256_storeu_si256((__m256i *)&rx_pkts[i + 4]->rearm_data, + rearm4); + _mm256_storeu_si256((__m256i *)&rx_pkts[i + 2]->rearm_data, + rearm2); + _mm256_storeu_si256((__m256i *)&rx_pkts[i + 0]->rearm_data, + rearm0); + + /* repeat for the odd mbufs */ + const __m256i odd_flags = + _mm256_castsi128_si256 + (_mm256_extracti128_si256(mbuf_flags, 1)); + rearm7 = _mm256_blend_epi32(mbuf_init, + _mm256_slli_si256(odd_flags, 8), + 0x04); + rearm5 = _mm256_blend_epi32(mbuf_init, + _mm256_slli_si256(odd_flags, 4), + 0x04); + rearm3 = _mm256_blend_epi32(mbuf_init, odd_flags, 0x04); + rearm1 = _mm256_blend_epi32(mbuf_init, + _mm256_srli_si256(odd_flags, 4), + 0x04); + /* since odd mbufs are already in hi 128-bits use blend */ + rearm7 = _mm256_blend_epi32(rearm7, mb6_7, 0xF0); + rearm5 = _mm256_blend_epi32(rearm5, mb4_5, 0xF0); + rearm3 = _mm256_blend_epi32(rearm3, mb2_3, 0xF0); + rearm1 = _mm256_blend_epi32(rearm1, mb0_1, 0xF0); + /* again write to mbufs */ + _mm256_storeu_si256((__m256i *)&rx_pkts[i + 7]->rearm_data, + rearm7); + _mm256_storeu_si256((__m256i *)&rx_pkts[i + 5]->rearm_data, + rearm5); + _mm256_storeu_si256((__m256i *)&rx_pkts[i + 3]->rearm_data, + rearm3); + _mm256_storeu_si256((__m256i *)&rx_pkts[i + 1]->rearm_data, + rearm1); + + /* extract and record EOP bit */ + if (split_packet) { + const __m128i eop_mask = + _mm_set1_epi16(1 << IAVF_RX_DESC_STATUS_EOF_SHIFT); + const __m256i eop_bits256 = _mm256_and_si256(status0_7, + eop_check); + /* pack status bits into a single 128-bit register */ + const __m128i eop_bits = + _mm_packus_epi32 + (_mm256_castsi256_si128(eop_bits256), + _mm256_extractf128_si256(eop_bits256, + 1)); + /** + * flip bits, and mask out the EOP bit, which is now + * a split-packet bit i.e. !EOP, rather than EOP one. + */ + __m128i split_bits = _mm_andnot_si128(eop_bits, + eop_mask); + /** + * eop bits are out of order, so we need to shuffle them + * back into order again. In doing so, only use low 8 + * bits, which acts like another pack instruction + * The original order is (hi->lo): 1,3,5,7,0,2,4,6 + * [Since we use epi8, the 16-bit positions are + * multiplied by 2 in the eop_shuffle value.] + */ + __m128i eop_shuffle = + _mm_set_epi8(/* zero hi 64b */ + 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, + /* move values to lo 64b */ + 8, 0, 10, 2, + 12, 4, 14, 6); + split_bits = _mm_shuffle_epi8(split_bits, eop_shuffle); + *(uint64_t *)split_packet = + _mm_cvtsi128_si64(split_bits); + split_packet += IAVF_DESCS_PER_LOOP_AVX; + } + + /* perform dd_check */ + status0_7 = _mm256_and_si256(status0_7, dd_check); + status0_7 = _mm256_packs_epi32(status0_7, + _mm256_setzero_si256()); + + uint64_t burst = __builtin_popcountll + (_mm_cvtsi128_si64 + (_mm256_extracti128_si256 + (status0_7, 1))); + burst += __builtin_popcountll + (_mm_cvtsi128_si64 + (_mm256_castsi256_si128(status0_7))); + received += burst; + if (burst != IAVF_DESCS_PER_LOOP_AVX) + break; + } + + /* update tail pointers */ + rxq->rx_tail += received; + rxq->rx_tail &= (rxq->nb_rx_desc - 1); + if ((rxq->rx_tail & 1) == 1 && received > 1) { /* keep avx2 aligned */ + rxq->rx_tail--; + received--; + } + rxq->rxrearm_nb += received; + return received; +} + +static inline __m256i +flex_rxd_to_fdir_flags_vec_avx2(const __m256i fdir_id0_7) +{ +#define FDID_MIS_MAGIC 0xFFFFFFFF + RTE_BUILD_BUG_ON(PKT_RX_FDIR != (1 << 2)); + RTE_BUILD_BUG_ON(PKT_RX_FDIR_ID != (1 << 13)); + const __m256i pkt_fdir_bit = _mm256_set1_epi32(PKT_RX_FDIR | + PKT_RX_FDIR_ID); + /* desc->flow_id field == 0xFFFFFFFF means fdir mismatch */ + const __m256i fdir_mis_mask = _mm256_set1_epi32(FDID_MIS_MAGIC); + __m256i fdir_mask = _mm256_cmpeq_epi32(fdir_id0_7, + fdir_mis_mask); + /* this XOR op results to bit-reverse the fdir_mask */ + fdir_mask = _mm256_xor_si256(fdir_mask, fdir_mis_mask); + const __m256i fdir_flags = _mm256_and_si256(fdir_mask, pkt_fdir_bit); + + return fdir_flags; +} + +static inline uint16_t +_iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq, + struct rte_mbuf **rx_pkts, + uint16_t nb_pkts, uint8_t *split_packet) +{ +#define IAVF_DESCS_PER_LOOP_AVX 8 + + const uint32_t *type_table = rxq->vsi->adapter->ptype_tbl; + + const __m256i mbuf_init = _mm256_set_epi64x(0, 0, + 0, rxq->mbuf_initializer); + struct rte_mbuf **sw_ring = &rxq->sw_ring[rxq->rx_tail]; + volatile union iavf_rx_flex_desc *rxdp = + (union iavf_rx_flex_desc *)rxq->rx_ring + rxq->rx_tail; + + rte_prefetch0(rxdp); + + /* nb_pkts has to be floor-aligned to IAVF_DESCS_PER_LOOP_AVX */ + nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, IAVF_DESCS_PER_LOOP_AVX); + + /* See if we need to rearm the RX queue - gives the prefetch a bit + * of time to act + */ + if (rxq->rxrearm_nb > IAVF_RXQ_REARM_THRESH) + iavf_rxq_rearm(rxq); + + /* Before we start moving massive data around, check to see if + * there is actually a packet available + */ + if (!(rxdp->wb.status_error0 & + rte_cpu_to_le_32(1 << IAVF_RX_FLEX_DESC_STATUS0_DD_S))) + return 0; + + /* constants used in processing loop */ + const __m256i crc_adjust = + _mm256_set_epi16 + (/* first descriptor */ + 0, 0, 0, /* ignore non-length fields */ + -rxq->crc_len, /* sub crc on data_len */ + 0, /* ignore high-16bits of pkt_len */ + -rxq->crc_len, /* sub crc on pkt_len */ + 0, 0, /* ignore pkt_type field */ + /* second descriptor */ + 0, 0, 0, /* ignore non-length fields */ + -rxq->crc_len, /* sub crc on data_len */ + 0, /* ignore high-16bits of pkt_len */ + -rxq->crc_len, /* sub crc on pkt_len */ + 0, 0 /* ignore pkt_type field */ + ); + + /* 8 packets DD mask, LSB in each 32-bit value */ + const __m256i dd_check = _mm256_set1_epi32(1); + + /* 8 packets EOP mask, second-LSB in each 32-bit value */ + const __m256i eop_check = _mm256_slli_epi32(dd_check, + IAVF_RX_FLEX_DESC_STATUS0_EOF_S); + + /* mask to shuffle from desc. to mbuf (2 descriptors)*/ + const __m256i shuf_msk = + _mm256_set_epi8 + (/* first descriptor */ + 0xFF, 0xFF, + 0xFF, 0xFF, /* rss hash parsed separately */ + 11, 10, /* octet 10~11, 16 bits vlan_macip */ + 5, 4, /* octet 4~5, 16 bits data_len */ + 0xFF, 0xFF, /* skip hi 16 bits pkt_len, zero out */ + 5, 4, /* octet 4~5, 16 bits pkt_len */ + 0xFF, 0xFF, /* pkt_type set as unknown */ + 0xFF, 0xFF, /*pkt_type set as unknown */ + /* second descriptor */ + 0xFF, 0xFF, + 0xFF, 0xFF, /* rss hash parsed separately */ + 11, 10, /* octet 10~11, 16 bits vlan_macip */ + 5, 4, /* octet 4~5, 16 bits data_len */ + 0xFF, 0xFF, /* skip hi 16 bits pkt_len, zero out */ + 5, 4, /* octet 4~5, 16 bits pkt_len */ + 0xFF, 0xFF, /* pkt_type set as unknown */ + 0xFF, 0xFF /*pkt_type set as unknown */ + ); + /** + * compile-time check the above crc and shuffle layout is correct. + * NOTE: the first field (lowest address) is given last in set_epi + * calls above. + */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, vlan_tci) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 10); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, hash) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 12); + + /* Status/Error flag masks */ + /** + * mask everything except Checksum Reports, RSS indication + * and VLAN indication. + * bit6:4 for IP/L4 checksum errors. + * bit12 is for RSS indication. + * bit13 is for VLAN indication. + */ + const __m256i flags_mask = + _mm256_set1_epi32((7 << 4) | (1 << 12) | (1 << 13)); + /** + * data to be shuffled by the result of the flags mask shifted by 4 + * bits. This gives use the l3_l4 flags. + */ + const __m256i l3_l4_flags_shuf = _mm256_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, + /* shift right 1 bit to make sure it not exceed 255 */ + (PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD | + PKT_RX_IP_CKSUM_BAD) >> 1, + (PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD | + PKT_RX_IP_CKSUM_GOOD) >> 1, + (PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_GOOD | + PKT_RX_IP_CKSUM_BAD) >> 1, + (PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_GOOD | + PKT_RX_IP_CKSUM_GOOD) >> 1, + (PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD) >> 1, + (PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_GOOD) >> 1, + (PKT_RX_L4_CKSUM_GOOD | PKT_RX_IP_CKSUM_BAD) >> 1, + (PKT_RX_L4_CKSUM_GOOD | PKT_RX_IP_CKSUM_GOOD) >> 1, + /* second 128-bits */ + 0, 0, 0, 0, 0, 0, 0, 0, + (PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD | + PKT_RX_IP_CKSUM_BAD) >> 1, + (PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD | + PKT_RX_IP_CKSUM_GOOD) >> 1, + (PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_GOOD | + PKT_RX_IP_CKSUM_BAD) >> 1, + (PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_GOOD | + PKT_RX_IP_CKSUM_GOOD) >> 1, + (PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD) >> 1, + (PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_GOOD) >> 1, + (PKT_RX_L4_CKSUM_GOOD | PKT_RX_IP_CKSUM_BAD) >> 1, + (PKT_RX_L4_CKSUM_GOOD | PKT_RX_IP_CKSUM_GOOD) >> 1); + const __m256i cksum_mask = + _mm256_set1_epi32(PKT_RX_IP_CKSUM_GOOD | PKT_RX_IP_CKSUM_BAD | + PKT_RX_L4_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD | + PKT_RX_EIP_CKSUM_BAD); + /** + * data to be shuffled by result of flag mask, shifted down 12. + * If RSS(bit12)/VLAN(bit13) are set, + * shuffle moves appropriate flags in place. + */ + const __m256i rss_vlan_flags_shuf = _mm256_set_epi8(0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + PKT_RX_RSS_HASH | PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED, + PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED, + PKT_RX_RSS_HASH, 0, + /* end up 128-bits */ + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + PKT_RX_RSS_HASH | PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED, + PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED, + PKT_RX_RSS_HASH, 0); + + uint16_t i, received; + + for (i = 0, received = 0; i < nb_pkts; + i += IAVF_DESCS_PER_LOOP_AVX, + rxdp += IAVF_DESCS_PER_LOOP_AVX) { + /* step 1, copy over 8 mbuf pointers to rx_pkts array */ + _mm256_storeu_si256((void *)&rx_pkts[i], + _mm256_loadu_si256((void *)&sw_ring[i])); +#ifdef RTE_ARCH_X86_64 + _mm256_storeu_si256 + ((void *)&rx_pkts[i + 4], + _mm256_loadu_si256((void *)&sw_ring[i + 4])); +#endif + + __m256i raw_desc0_1, raw_desc2_3, raw_desc4_5, raw_desc6_7; + + const __m128i raw_desc7 = + _mm_load_si128((void *)(rxdp + 7)); + rte_compiler_barrier(); + const __m128i raw_desc6 = + _mm_load_si128((void *)(rxdp + 6)); + rte_compiler_barrier(); + const __m128i raw_desc5 = + _mm_load_si128((void *)(rxdp + 5)); + rte_compiler_barrier(); + const __m128i raw_desc4 = + _mm_load_si128((void *)(rxdp + 4)); + rte_compiler_barrier(); + const __m128i raw_desc3 = + _mm_load_si128((void *)(rxdp + 3)); + rte_compiler_barrier(); + const __m128i raw_desc2 = + _mm_load_si128((void *)(rxdp + 2)); + rte_compiler_barrier(); + const __m128i raw_desc1 = + _mm_load_si128((void *)(rxdp + 1)); + rte_compiler_barrier(); + const __m128i raw_desc0 = + _mm_load_si128((void *)(rxdp + 0)); + + raw_desc6_7 = + _mm256_inserti128_si256 + (_mm256_castsi128_si256(raw_desc6), + raw_desc7, 1); + raw_desc4_5 = + _mm256_inserti128_si256 + (_mm256_castsi128_si256(raw_desc4), + raw_desc5, 1); + raw_desc2_3 = + _mm256_inserti128_si256 + (_mm256_castsi128_si256(raw_desc2), + raw_desc3, 1); + raw_desc0_1 = + _mm256_inserti128_si256 + (_mm256_castsi128_si256(raw_desc0), + raw_desc1, 1); + + if (split_packet) { + int j; + + for (j = 0; j < IAVF_DESCS_PER_LOOP_AVX; j++) + rte_mbuf_prefetch_part2(rx_pkts[i + j]); + } + + /** + * convert descriptors 4-7 into mbufs, re-arrange fields. + * Then write into the mbuf. + */ + __m256i mb6_7 = _mm256_shuffle_epi8(raw_desc6_7, shuf_msk); + __m256i mb4_5 = _mm256_shuffle_epi8(raw_desc4_5, shuf_msk); + + mb6_7 = _mm256_add_epi16(mb6_7, crc_adjust); + mb4_5 = _mm256_add_epi16(mb4_5, crc_adjust); + /** + * to get packet types, ptype is located in bit16-25 + * of each 128bits + */ + const __m256i ptype_mask = + _mm256_set1_epi16(IAVF_RX_FLEX_DESC_PTYPE_M); + const __m256i ptypes6_7 = + _mm256_and_si256(raw_desc6_7, ptype_mask); + const __m256i ptypes4_5 = + _mm256_and_si256(raw_desc4_5, ptype_mask); + const uint16_t ptype7 = _mm256_extract_epi16(ptypes6_7, 9); + const uint16_t ptype6 = _mm256_extract_epi16(ptypes6_7, 1); + const uint16_t ptype5 = _mm256_extract_epi16(ptypes4_5, 9); + const uint16_t ptype4 = _mm256_extract_epi16(ptypes4_5, 1); + + mb6_7 = _mm256_insert_epi32(mb6_7, type_table[ptype7], 4); + mb6_7 = _mm256_insert_epi32(mb6_7, type_table[ptype6], 0); + mb4_5 = _mm256_insert_epi32(mb4_5, type_table[ptype5], 4); + mb4_5 = _mm256_insert_epi32(mb4_5, type_table[ptype4], 0); + /* merge the status bits into one register */ + const __m256i status4_7 = _mm256_unpackhi_epi32(raw_desc6_7, + raw_desc4_5); + + /** + * convert descriptors 0-3 into mbufs, re-arrange fields. + * Then write into the mbuf. + */ + __m256i mb2_3 = _mm256_shuffle_epi8(raw_desc2_3, shuf_msk); + __m256i mb0_1 = _mm256_shuffle_epi8(raw_desc0_1, shuf_msk); + + mb2_3 = _mm256_add_epi16(mb2_3, crc_adjust); + mb0_1 = _mm256_add_epi16(mb0_1, crc_adjust); + /** + * to get packet types, ptype is located in bit16-25 + * of each 128bits + */ + const __m256i ptypes2_3 = + _mm256_and_si256(raw_desc2_3, ptype_mask); + const __m256i ptypes0_1 = + _mm256_and_si256(raw_desc0_1, ptype_mask); + const uint16_t ptype3 = _mm256_extract_epi16(ptypes2_3, 9); + const uint16_t ptype2 = _mm256_extract_epi16(ptypes2_3, 1); + const uint16_t ptype1 = _mm256_extract_epi16(ptypes0_1, 9); + const uint16_t ptype0 = _mm256_extract_epi16(ptypes0_1, 1); + + mb2_3 = _mm256_insert_epi32(mb2_3, type_table[ptype3], 4); + mb2_3 = _mm256_insert_epi32(mb2_3, type_table[ptype2], 0); + mb0_1 = _mm256_insert_epi32(mb0_1, type_table[ptype1], 4); + mb0_1 = _mm256_insert_epi32(mb0_1, type_table[ptype0], 0); + /* merge the status bits into one register */ + const __m256i status0_3 = _mm256_unpackhi_epi32(raw_desc2_3, + raw_desc0_1); + + /** + * take the two sets of status bits and merge to one + * After merge, the packets status flags are in the + * order (hi->lo): [1, 3, 5, 7, 0, 2, 4, 6] + */ + __m256i status0_7 = _mm256_unpacklo_epi64(status4_7, + status0_3); + + /* now do flag manipulation */ + + /* get only flag/error bits we want */ + const __m256i flag_bits = + _mm256_and_si256(status0_7, flags_mask); + /** + * l3_l4_error flags, shuffle, then shift to correct adjustment + * of flags in flags_shuf, and finally mask out extra bits + */ + __m256i l3_l4_flags = _mm256_shuffle_epi8(l3_l4_flags_shuf, + _mm256_srli_epi32(flag_bits, 4)); + l3_l4_flags = _mm256_slli_epi32(l3_l4_flags, 1); + l3_l4_flags = _mm256_and_si256(l3_l4_flags, cksum_mask); + /* set rss and vlan flags */ + const __m256i rss_vlan_flag_bits = + _mm256_srli_epi32(flag_bits, 12); + const __m256i rss_vlan_flags = + _mm256_shuffle_epi8(rss_vlan_flags_shuf, + rss_vlan_flag_bits); + + /* merge flags */ + __m256i mbuf_flags = _mm256_or_si256(l3_l4_flags, + rss_vlan_flags); + + if (rxq->fdir_enabled) { + const __m256i fdir_id4_7 = + _mm256_unpackhi_epi32(raw_desc6_7, raw_desc4_5); + + const __m256i fdir_id0_3 = + _mm256_unpackhi_epi32(raw_desc2_3, raw_desc0_1); + + const __m256i fdir_id0_7 = + _mm256_unpackhi_epi64(fdir_id4_7, fdir_id0_3); + + const __m256i fdir_flags = + flex_rxd_to_fdir_flags_vec_avx2(fdir_id0_7); + + /* merge with fdir_flags */ + mbuf_flags = _mm256_or_si256(mbuf_flags, fdir_flags); + + /* write to mbuf: have to use scalar store here */ + rx_pkts[i + 0]->hash.fdir.hi = + _mm256_extract_epi32(fdir_id0_7, 3); + + rx_pkts[i + 1]->hash.fdir.hi = + _mm256_extract_epi32(fdir_id0_7, 7); + + rx_pkts[i + 2]->hash.fdir.hi = + _mm256_extract_epi32(fdir_id0_7, 2); + + rx_pkts[i + 3]->hash.fdir.hi = + _mm256_extract_epi32(fdir_id0_7, 6); + + rx_pkts[i + 4]->hash.fdir.hi = + _mm256_extract_epi32(fdir_id0_7, 1); + + rx_pkts[i + 5]->hash.fdir.hi = + _mm256_extract_epi32(fdir_id0_7, 5); + + rx_pkts[i + 6]->hash.fdir.hi = + _mm256_extract_epi32(fdir_id0_7, 0); + + rx_pkts[i + 7]->hash.fdir.hi = + _mm256_extract_epi32(fdir_id0_7, 4); + } /* if() on fdir_enabled */ + +#ifndef RTE_LIBRTE_IAVF_16BYTE_RX_DESC + /** + * needs to load 2nd 16B of each desc for RSS hash parsing, + * will cause performance drop to get into this context. + */ + if (rxq->vsi->adapter->eth_dev->data->dev_conf.rxmode.offloads & + DEV_RX_OFFLOAD_RSS_HASH) { + /* load bottom half of every 32B desc */ + const __m128i raw_desc_bh7 = + _mm_load_si128 + ((void *)(&rxdp[7].wb.status_error1)); + rte_compiler_barrier(); + const __m128i raw_desc_bh6 = + _mm_load_si128 + ((void *)(&rxdp[6].wb.status_error1)); + rte_compiler_barrier(); + const __m128i raw_desc_bh5 = + _mm_load_si128 + ((void *)(&rxdp[5].wb.status_error1)); + rte_compiler_barrier(); + const __m128i raw_desc_bh4 = + _mm_load_si128 + ((void *)(&rxdp[4].wb.status_error1)); + rte_compiler_barrier(); + const __m128i raw_desc_bh3 = + _mm_load_si128 + ((void *)(&rxdp[3].wb.status_error1)); + rte_compiler_barrier(); + const __m128i raw_desc_bh2 = + _mm_load_si128 + ((void *)(&rxdp[2].wb.status_error1)); + rte_compiler_barrier(); + const __m128i raw_desc_bh1 = + _mm_load_si128 + ((void *)(&rxdp[1].wb.status_error1)); + rte_compiler_barrier(); + const __m128i raw_desc_bh0 = + _mm_load_si128 + ((void *)(&rxdp[0].wb.status_error1)); + + __m256i raw_desc_bh6_7 = + _mm256_inserti128_si256 + (_mm256_castsi128_si256(raw_desc_bh6), + raw_desc_bh7, 1); + __m256i raw_desc_bh4_5 = + _mm256_inserti128_si256 + (_mm256_castsi128_si256(raw_desc_bh4), + raw_desc_bh5, 1); + __m256i raw_desc_bh2_3 = + _mm256_inserti128_si256 + (_mm256_castsi128_si256(raw_desc_bh2), + raw_desc_bh3, 1); + __m256i raw_desc_bh0_1 = + _mm256_inserti128_si256 + (_mm256_castsi128_si256(raw_desc_bh0), + raw_desc_bh1, 1); + + /** + * to shift the 32b RSS hash value to the + * highest 32b of each 128b before mask + */ + __m256i rss_hash6_7 = + _mm256_slli_epi64(raw_desc_bh6_7, 32); + __m256i rss_hash4_5 = + _mm256_slli_epi64(raw_desc_bh4_5, 32); + __m256i rss_hash2_3 = + _mm256_slli_epi64(raw_desc_bh2_3, 32); + __m256i rss_hash0_1 = + _mm256_slli_epi64(raw_desc_bh0_1, 32); + + __m256i rss_hash_msk = + _mm256_set_epi32(0xFFFFFFFF, 0, 0, 0, + 0xFFFFFFFF, 0, 0, 0); + + rss_hash6_7 = _mm256_and_si256 + (rss_hash6_7, rss_hash_msk); + rss_hash4_5 = _mm256_and_si256 + (rss_hash4_5, rss_hash_msk); + rss_hash2_3 = _mm256_and_si256 + (rss_hash2_3, rss_hash_msk); + rss_hash0_1 = _mm256_and_si256 + (rss_hash0_1, rss_hash_msk); + + mb6_7 = _mm256_or_si256(mb6_7, rss_hash6_7); + mb4_5 = _mm256_or_si256(mb4_5, rss_hash4_5); + mb2_3 = _mm256_or_si256(mb2_3, rss_hash2_3); + mb0_1 = _mm256_or_si256(mb0_1, rss_hash0_1); + } /* if() on RSS hash parsing */ +#endif + + /** + * At this point, we have the 8 sets of flags in the low 16-bits + * of each 32-bit value in vlan0. + * We want to extract these, and merge them with the mbuf init + * data so we can do a single write to the mbuf to set the flags + * and all the other initialization fields. Extracting the + * appropriate flags means that we have to do a shift and blend + * for each mbuf before we do the write. However, we can also + * add in the previously computed rx_descriptor fields to + * make a single 256-bit write per mbuf + */ + /* check the structure matches expectations */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, ol_flags) != + offsetof(struct rte_mbuf, rearm_data) + 8); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rearm_data) != + RTE_ALIGN(offsetof(struct rte_mbuf, + rearm_data), + 16)); + /* build up data and do writes */ + __m256i rearm0, rearm1, rearm2, rearm3, rearm4, rearm5, + rearm6, rearm7; + rearm6 = _mm256_blend_epi32(mbuf_init, + _mm256_slli_si256(mbuf_flags, 8), + 0x04); + rearm4 = _mm256_blend_epi32(mbuf_init, + _mm256_slli_si256(mbuf_flags, 4), + 0x04); + rearm2 = _mm256_blend_epi32(mbuf_init, mbuf_flags, 0x04); + rearm0 = _mm256_blend_epi32(mbuf_init, + _mm256_srli_si256(mbuf_flags, 4), + 0x04); + /* permute to add in the rx_descriptor e.g. rss fields */ + rearm6 = _mm256_permute2f128_si256(rearm6, mb6_7, 0x20); + rearm4 = _mm256_permute2f128_si256(rearm4, mb4_5, 0x20); + rearm2 = _mm256_permute2f128_si256(rearm2, mb2_3, 0x20); + rearm0 = _mm256_permute2f128_si256(rearm0, mb0_1, 0x20); + /* write to mbuf */ + _mm256_storeu_si256((__m256i *)&rx_pkts[i + 6]->rearm_data, + rearm6); + _mm256_storeu_si256((__m256i *)&rx_pkts[i + 4]->rearm_data, + rearm4); + _mm256_storeu_si256((__m256i *)&rx_pkts[i + 2]->rearm_data, + rearm2); + _mm256_storeu_si256((__m256i *)&rx_pkts[i + 0]->rearm_data, + rearm0); + + /* repeat for the odd mbufs */ + const __m256i odd_flags = + _mm256_castsi128_si256 + (_mm256_extracti128_si256(mbuf_flags, 1)); + rearm7 = _mm256_blend_epi32(mbuf_init, + _mm256_slli_si256(odd_flags, 8), + 0x04); + rearm5 = _mm256_blend_epi32(mbuf_init, + _mm256_slli_si256(odd_flags, 4), + 0x04); + rearm3 = _mm256_blend_epi32(mbuf_init, odd_flags, 0x04); + rearm1 = _mm256_blend_epi32(mbuf_init, + _mm256_srli_si256(odd_flags, 4), + 0x04); + /* since odd mbufs are already in hi 128-bits use blend */ + rearm7 = _mm256_blend_epi32(rearm7, mb6_7, 0xF0); + rearm5 = _mm256_blend_epi32(rearm5, mb4_5, 0xF0); + rearm3 = _mm256_blend_epi32(rearm3, mb2_3, 0xF0); + rearm1 = _mm256_blend_epi32(rearm1, mb0_1, 0xF0); + /* again write to mbufs */ + _mm256_storeu_si256((__m256i *)&rx_pkts[i + 7]->rearm_data, + rearm7); + _mm256_storeu_si256((__m256i *)&rx_pkts[i + 5]->rearm_data, + rearm5); + _mm256_storeu_si256((__m256i *)&rx_pkts[i + 3]->rearm_data, + rearm3); + _mm256_storeu_si256((__m256i *)&rx_pkts[i + 1]->rearm_data, + rearm1); + + /* extract and record EOP bit */ + if (split_packet) { + const __m128i eop_mask = + _mm_set1_epi16(1 << + IAVF_RX_FLEX_DESC_STATUS0_EOF_S); + const __m256i eop_bits256 = _mm256_and_si256(status0_7, + eop_check); + /* pack status bits into a single 128-bit register */ + const __m128i eop_bits = + _mm_packus_epi32 + (_mm256_castsi256_si128(eop_bits256), + _mm256_extractf128_si256(eop_bits256, + 1)); + /** + * flip bits, and mask out the EOP bit, which is now + * a split-packet bit i.e. !EOP, rather than EOP one. + */ + __m128i split_bits = _mm_andnot_si128(eop_bits, + eop_mask); + /** + * eop bits are out of order, so we need to shuffle them + * back into order again. In doing so, only use low 8 + * bits, which acts like another pack instruction + * The original order is (hi->lo): 1,3,5,7,0,2,4,6 + * [Since we use epi8, the 16-bit positions are + * multiplied by 2 in the eop_shuffle value.] + */ + __m128i eop_shuffle = + _mm_set_epi8(/* zero hi 64b */ + 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, + /* move values to lo 64b */ + 8, 0, 10, 2, + 12, 4, 14, 6); + split_bits = _mm_shuffle_epi8(split_bits, eop_shuffle); + *(uint64_t *)split_packet = + _mm_cvtsi128_si64(split_bits); + split_packet += IAVF_DESCS_PER_LOOP_AVX; + } + + /* perform dd_check */ + status0_7 = _mm256_and_si256(status0_7, dd_check); + status0_7 = _mm256_packs_epi32(status0_7, + _mm256_setzero_si256()); + + uint64_t burst = __builtin_popcountll + (_mm_cvtsi128_si64 + (_mm256_extracti128_si256 + (status0_7, 1))); + burst += __builtin_popcountll + (_mm_cvtsi128_si64 + (_mm256_castsi256_si128(status0_7))); + received += burst; + if (burst != IAVF_DESCS_PER_LOOP_AVX) + break; + } + + /* update tail pointers */ + rxq->rx_tail += received; + rxq->rx_tail &= (rxq->nb_rx_desc - 1); + if ((rxq->rx_tail & 1) == 1 && received > 1) { /* keep avx2 aligned */ + rxq->rx_tail--; + received--; + } + rxq->rxrearm_nb += received; + return received; +} + +/** + * Notice: + * - nb_pkts < IAVF_DESCS_PER_LOOP, just return no packet + */ +uint16_t +iavf_recv_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + return _iavf_recv_raw_pkts_vec_avx2(rx_queue, rx_pkts, nb_pkts, NULL); +} + +/** + * Notice: + * - nb_pkts < IAVF_DESCS_PER_LOOP, just return no packet + */ +uint16_t +iavf_recv_pkts_vec_avx2_flex_rxd(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + return _iavf_recv_raw_pkts_vec_avx2_flex_rxd(rx_queue, rx_pkts, + nb_pkts, NULL); +} + +/** + * vPMD receive routine that reassembles single burst of 32 scattered packets + * Notice: + * - nb_pkts < IAVF_DESCS_PER_LOOP, just return no packet + */ +static uint16_t +iavf_recv_scattered_burst_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + struct iavf_rx_queue *rxq = rx_queue; + uint8_t split_flags[IAVF_VPMD_RX_MAX_BURST] = {0}; + + /* get some new buffers */ + uint16_t nb_bufs = _iavf_recv_raw_pkts_vec_avx2(rxq, rx_pkts, nb_pkts, + split_flags); + if (nb_bufs == 0) + return 0; + + /* happy day case, full burst + no packets to be joined */ + const uint64_t *split_fl64 = (uint64_t *)split_flags; + + if (!rxq->pkt_first_seg && + split_fl64[0] == 0 && split_fl64[1] == 0 && + split_fl64[2] == 0 && split_fl64[3] == 0) + return nb_bufs; + + /* reassemble any packets that need reassembly*/ + unsigned int i = 0; + + if (!rxq->pkt_first_seg) { + /* find the first split flag, and only reassemble then*/ + while (i < nb_bufs && !split_flags[i]) + i++; + if (i == nb_bufs) + return nb_bufs; + rxq->pkt_first_seg = rx_pkts[i]; + } + return i + reassemble_packets(rxq, &rx_pkts[i], nb_bufs - i, + &split_flags[i]); +} + +/** + * vPMD receive routine that reassembles scattered packets. + * Main receive routine that can handle arbitrary burst sizes + * Notice: + * - nb_pkts < IAVF_DESCS_PER_LOOP, just return no packet + */ +uint16_t +iavf_recv_scattered_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + uint16_t retval = 0; + + while (nb_pkts > IAVF_VPMD_RX_MAX_BURST) { + uint16_t burst = iavf_recv_scattered_burst_vec_avx2(rx_queue, + rx_pkts + retval, IAVF_VPMD_RX_MAX_BURST); + retval += burst; + nb_pkts -= burst; + if (burst < IAVF_VPMD_RX_MAX_BURST) + return retval; + } + return retval + iavf_recv_scattered_burst_vec_avx2(rx_queue, + rx_pkts + retval, nb_pkts); +} + +/** + * vPMD receive routine that reassembles single burst of + * 32 scattered packets for flex RxD + * Notice: + * - nb_pkts < IAVF_DESCS_PER_LOOP, just return no packet + */ +static uint16_t +iavf_recv_scattered_burst_vec_avx2_flex_rxd(void *rx_queue, + struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + struct iavf_rx_queue *rxq = rx_queue; + uint8_t split_flags[IAVF_VPMD_RX_MAX_BURST] = {0}; + + /* get some new buffers */ + uint16_t nb_bufs = _iavf_recv_raw_pkts_vec_avx2_flex_rxd(rxq, + rx_pkts, nb_pkts, split_flags); + if (nb_bufs == 0) + return 0; + + /* happy day case, full burst + no packets to be joined */ + const uint64_t *split_fl64 = (uint64_t *)split_flags; + + if (!rxq->pkt_first_seg && + split_fl64[0] == 0 && split_fl64[1] == 0 && + split_fl64[2] == 0 && split_fl64[3] == 0) + return nb_bufs; + + /* reassemble any packets that need reassembly*/ + unsigned int i = 0; + + if (!rxq->pkt_first_seg) { + /* find the first split flag, and only reassemble then*/ + while (i < nb_bufs && !split_flags[i]) + i++; + if (i == nb_bufs) + return nb_bufs; + rxq->pkt_first_seg = rx_pkts[i]; + } + return i + reassemble_packets(rxq, &rx_pkts[i], nb_bufs - i, + &split_flags[i]); +} + +/** + * vPMD receive routine that reassembles scattered packets for flex RxD. + * Main receive routine that can handle arbitrary burst sizes + * Notice: + * - nb_pkts < IAVF_DESCS_PER_LOOP, just return no packet + */ +uint16_t +iavf_recv_scattered_pkts_vec_avx2_flex_rxd(void *rx_queue, + struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + uint16_t retval = 0; + + while (nb_pkts > IAVF_VPMD_RX_MAX_BURST) { + uint16_t burst = + iavf_recv_scattered_burst_vec_avx2_flex_rxd + (rx_queue, rx_pkts + retval, IAVF_VPMD_RX_MAX_BURST); + retval += burst; + nb_pkts -= burst; + if (burst < IAVF_VPMD_RX_MAX_BURST) + return retval; + } + return retval + iavf_recv_scattered_burst_vec_avx2_flex_rxd(rx_queue, + rx_pkts + retval, nb_pkts); +} + +static inline void +iavf_vtx1(volatile struct iavf_tx_desc *txdp, + struct rte_mbuf *pkt, uint64_t flags) +{ + uint64_t high_qw = + (IAVF_TX_DESC_DTYPE_DATA | + ((uint64_t)flags << IAVF_TXD_QW1_CMD_SHIFT) | + ((uint64_t)pkt->data_len << IAVF_TXD_QW1_TX_BUF_SZ_SHIFT)); + + __m128i descriptor = _mm_set_epi64x(high_qw, + pkt->buf_physaddr + pkt->data_off); + _mm_store_si128((__m128i *)txdp, descriptor); +} + +static inline void +iavf_vtx(volatile struct iavf_tx_desc *txdp, + struct rte_mbuf **pkt, uint16_t nb_pkts, uint64_t flags) +{ + const uint64_t hi_qw_tmpl = (IAVF_TX_DESC_DTYPE_DATA | + ((uint64_t)flags << IAVF_TXD_QW1_CMD_SHIFT)); + + /* if unaligned on 32-bit boundary, do one to align */ + if (((uintptr_t)txdp & 0x1F) != 0 && nb_pkts != 0) { + iavf_vtx1(txdp, *pkt, flags); + nb_pkts--, txdp++, pkt++; + } + + /* do two at a time while possible, in bursts */ + for (; nb_pkts > 3; txdp += 4, pkt += 4, nb_pkts -= 4) { + uint64_t hi_qw3 = + hi_qw_tmpl | + ((uint64_t)pkt[3]->data_len << + IAVF_TXD_QW1_TX_BUF_SZ_SHIFT); + uint64_t hi_qw2 = + hi_qw_tmpl | + ((uint64_t)pkt[2]->data_len << + IAVF_TXD_QW1_TX_BUF_SZ_SHIFT); + uint64_t hi_qw1 = + hi_qw_tmpl | + ((uint64_t)pkt[1]->data_len << + IAVF_TXD_QW1_TX_BUF_SZ_SHIFT); + uint64_t hi_qw0 = + hi_qw_tmpl | + ((uint64_t)pkt[0]->data_len << + IAVF_TXD_QW1_TX_BUF_SZ_SHIFT); + + __m256i desc2_3 = + _mm256_set_epi64x + (hi_qw3, + pkt[3]->buf_physaddr + pkt[3]->data_off, + hi_qw2, + pkt[2]->buf_physaddr + pkt[2]->data_off); + __m256i desc0_1 = + _mm256_set_epi64x + (hi_qw1, + pkt[1]->buf_physaddr + pkt[1]->data_off, + hi_qw0, + pkt[0]->buf_physaddr + pkt[0]->data_off); + _mm256_store_si256((void *)(txdp + 2), desc2_3); + _mm256_store_si256((void *)txdp, desc0_1); + } + + /* do any last ones */ + while (nb_pkts) { + iavf_vtx1(txdp, *pkt, flags); + txdp++, pkt++, nb_pkts--; + } +} + +static inline uint16_t +iavf_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + struct iavf_tx_queue *txq = (struct iavf_tx_queue *)tx_queue; + volatile struct iavf_tx_desc *txdp; + struct iavf_tx_entry *txep; + uint16_t n, nb_commit, tx_id; + /* bit2 is reserved and must be set to 1 according to Spec */ + uint64_t flags = IAVF_TX_DESC_CMD_EOP | IAVF_TX_DESC_CMD_ICRC; + uint64_t rs = IAVF_TX_DESC_CMD_RS | flags; + + /* cross rx_thresh boundary is not allowed */ + nb_pkts = RTE_MIN(nb_pkts, txq->rs_thresh); + + if (txq->nb_free < txq->free_thresh) + iavf_tx_free_bufs(txq); + + nb_commit = nb_pkts = (uint16_t)RTE_MIN(txq->nb_free, nb_pkts); + if (unlikely(nb_pkts == 0)) + return 0; + + tx_id = txq->tx_tail; + txdp = &txq->tx_ring[tx_id]; + txep = &txq->sw_ring[tx_id]; + + txq->nb_free = (uint16_t)(txq->nb_free - nb_pkts); + + n = (uint16_t)(txq->nb_tx_desc - tx_id); + if (nb_commit >= n) { + tx_backlog_entry(txep, tx_pkts, n); + + iavf_vtx(txdp, tx_pkts, n - 1, flags); + tx_pkts += (n - 1); + txdp += (n - 1); + + iavf_vtx1(txdp, *tx_pkts++, rs); + + nb_commit = (uint16_t)(nb_commit - n); + + tx_id = 0; + txq->next_rs = (uint16_t)(txq->rs_thresh - 1); + + /* avoid reach the end of ring */ + txdp = &txq->tx_ring[tx_id]; + txep = &txq->sw_ring[tx_id]; + } + + tx_backlog_entry(txep, tx_pkts, nb_commit); + + iavf_vtx(txdp, tx_pkts, nb_commit, flags); + + tx_id = (uint16_t)(tx_id + nb_commit); + if (tx_id > txq->next_rs) { + txq->tx_ring[txq->next_rs].cmd_type_offset_bsz |= + rte_cpu_to_le_64(((uint64_t)IAVF_TX_DESC_CMD_RS) << + IAVF_TXD_QW1_CMD_SHIFT); + txq->next_rs = + (uint16_t)(txq->next_rs + txq->rs_thresh); + } + + txq->tx_tail = tx_id; + + IAVF_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail); + + return nb_pkts; +} + +uint16_t +iavf_xmit_pkts_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + uint16_t nb_tx = 0; + struct iavf_tx_queue *txq = (struct iavf_tx_queue *)tx_queue; + + while (nb_pkts) { + uint16_t ret, num; + + num = (uint16_t)RTE_MIN(nb_pkts, txq->rs_thresh); + ret = iavf_xmit_fixed_burst_vec_avx2(tx_queue, &tx_pkts[nb_tx], + num); + nb_tx += ret; + nb_pkts -= ret; + if (ret < num) + break; + } + + return nb_tx; +} diff --git a/src/spdk/dpdk/drivers/net/iavf/iavf_rxtx_vec_common.h b/src/spdk/dpdk/drivers/net/iavf/iavf_rxtx_vec_common.h new file mode 100644 index 000000000..25bb502de --- /dev/null +++ b/src/spdk/dpdk/drivers/net/iavf/iavf_rxtx_vec_common.h @@ -0,0 +1,276 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017 Intel Corporation + */ + +#ifndef _IAVF_RXTX_VEC_COMMON_H_ +#define _IAVF_RXTX_VEC_COMMON_H_ +#include +#include +#include + +#include "iavf.h" +#include "iavf_rxtx.h" + +static inline uint16_t +reassemble_packets(struct iavf_rx_queue *rxq, struct rte_mbuf **rx_bufs, + uint16_t nb_bufs, uint8_t *split_flags) +{ + struct rte_mbuf *pkts[IAVF_VPMD_RX_MAX_BURST]; + struct rte_mbuf *start = rxq->pkt_first_seg; + struct rte_mbuf *end = rxq->pkt_last_seg; + unsigned int pkt_idx, buf_idx; + + for (buf_idx = 0, pkt_idx = 0; buf_idx < nb_bufs; buf_idx++) { + if (end) { + /* processing a split packet */ + end->next = rx_bufs[buf_idx]; + rx_bufs[buf_idx]->data_len += rxq->crc_len; + + start->nb_segs++; + start->pkt_len += rx_bufs[buf_idx]->data_len; + end = end->next; + + if (!split_flags[buf_idx]) { + /* it's the last packet of the set */ + start->hash = end->hash; + start->vlan_tci = end->vlan_tci; + start->ol_flags = end->ol_flags; + /* we need to strip crc for the whole packet */ + start->pkt_len -= rxq->crc_len; + if (end->data_len > rxq->crc_len) { + end->data_len -= rxq->crc_len; + } else { + /* free up last mbuf */ + struct rte_mbuf *secondlast = start; + + start->nb_segs--; + while (secondlast->next != end) + secondlast = secondlast->next; + secondlast->data_len -= (rxq->crc_len - + end->data_len); + secondlast->next = NULL; + rte_pktmbuf_free_seg(end); + } + pkts[pkt_idx++] = start; + start = NULL; + end = NULL; + } + } else { + /* not processing a split packet */ + if (!split_flags[buf_idx]) { + /* not a split packet, save and skip */ + pkts[pkt_idx++] = rx_bufs[buf_idx]; + continue; + } + end = start = rx_bufs[buf_idx]; + rx_bufs[buf_idx]->data_len += rxq->crc_len; + rx_bufs[buf_idx]->pkt_len += rxq->crc_len; + } + } + + /* save the partial packet for next time */ + rxq->pkt_first_seg = start; + rxq->pkt_last_seg = end; + memcpy(rx_bufs, pkts, pkt_idx * (sizeof(*pkts))); + return pkt_idx; +} + +static __rte_always_inline int +iavf_tx_free_bufs(struct iavf_tx_queue *txq) +{ + struct iavf_tx_entry *txep; + uint32_t n; + uint32_t i; + int nb_free = 0; + struct rte_mbuf *m, *free[IAVF_VPMD_TX_MAX_FREE_BUF]; + + /* check DD bits on threshold descriptor */ + if ((txq->tx_ring[txq->next_dd].cmd_type_offset_bsz & + rte_cpu_to_le_64(IAVF_TXD_QW1_DTYPE_MASK)) != + rte_cpu_to_le_64(IAVF_TX_DESC_DTYPE_DESC_DONE)) + return 0; + + n = txq->rs_thresh; + + /* first buffer to free from S/W ring is at index + * tx_next_dd - (tx_rs_thresh-1) + */ + txep = &txq->sw_ring[txq->next_dd - (n - 1)]; + m = rte_pktmbuf_prefree_seg(txep[0].mbuf); + if (likely(m != NULL)) { + free[0] = m; + nb_free = 1; + for (i = 1; i < n; i++) { + m = rte_pktmbuf_prefree_seg(txep[i].mbuf); + if (likely(m != NULL)) { + if (likely(m->pool == free[0]->pool)) { + free[nb_free++] = m; + } else { + rte_mempool_put_bulk(free[0]->pool, + (void *)free, + nb_free); + free[0] = m; + nb_free = 1; + } + } + } + rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free); + } else { + for (i = 1; i < n; i++) { + m = rte_pktmbuf_prefree_seg(txep[i].mbuf); + if (m) + rte_mempool_put(m->pool, m); + } + } + + /* buffers were freed, update counters */ + txq->nb_free = (uint16_t)(txq->nb_free + txq->rs_thresh); + txq->next_dd = (uint16_t)(txq->next_dd + txq->rs_thresh); + if (txq->next_dd >= txq->nb_tx_desc) + txq->next_dd = (uint16_t)(txq->rs_thresh - 1); + + return txq->rs_thresh; +} + +static __rte_always_inline void +tx_backlog_entry(struct iavf_tx_entry *txep, + struct rte_mbuf **tx_pkts, uint16_t nb_pkts) +{ + int i; + + for (i = 0; i < (int)nb_pkts; ++i) + txep[i].mbuf = tx_pkts[i]; +} + +static inline void +_iavf_rx_queue_release_mbufs_vec(struct iavf_rx_queue *rxq) +{ + const unsigned int mask = rxq->nb_rx_desc - 1; + unsigned int i; + + if (!rxq->sw_ring || rxq->rxrearm_nb >= rxq->nb_rx_desc) + return; + + /* free all mbufs that are valid in the ring */ + if (rxq->rxrearm_nb == 0) { + for (i = 0; i < rxq->nb_rx_desc; i++) { + if (rxq->sw_ring[i]) + rte_pktmbuf_free_seg(rxq->sw_ring[i]); + } + } else { + for (i = rxq->rx_tail; + i != rxq->rxrearm_start; + i = (i + 1) & mask) { + if (rxq->sw_ring[i]) + rte_pktmbuf_free_seg(rxq->sw_ring[i]); + } + } + + rxq->rxrearm_nb = rxq->nb_rx_desc; + + /* set all entries to NULL */ + memset(rxq->sw_ring, 0, sizeof(rxq->sw_ring[0]) * rxq->nb_rx_desc); +} + +static inline void +_iavf_tx_queue_release_mbufs_vec(struct iavf_tx_queue *txq) +{ + unsigned i; + const uint16_t max_desc = (uint16_t)(txq->nb_tx_desc - 1); + + if (!txq->sw_ring || txq->nb_free == max_desc) + return; + + i = txq->next_dd - txq->rs_thresh + 1; + if (txq->tx_tail < i) { + for (; i < txq->nb_tx_desc; i++) { + rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf); + txq->sw_ring[i].mbuf = NULL; + } + i = 0; + } +} + +static inline int +iavf_rxq_vec_setup_default(struct iavf_rx_queue *rxq) +{ + uintptr_t p; + struct rte_mbuf mb_def = { .buf_addr = 0 }; /* zeroed mbuf */ + + mb_def.nb_segs = 1; + mb_def.data_off = RTE_PKTMBUF_HEADROOM; + mb_def.port = rxq->port_id; + rte_mbuf_refcnt_set(&mb_def, 1); + + /* prevent compiler reordering: rearm_data covers previous fields */ + rte_compiler_barrier(); + p = (uintptr_t)&mb_def.rearm_data; + rxq->mbuf_initializer = *(uint64_t *)p; + return 0; +} + +static inline int +iavf_rx_vec_queue_default(struct iavf_rx_queue *rxq) +{ + if (!rxq) + return -1; + + if (!rte_is_power_of_2(rxq->nb_rx_desc)) + return -1; + + if (rxq->rx_free_thresh < IAVF_VPMD_RX_MAX_BURST) + return -1; + + if (rxq->nb_rx_desc % rxq->rx_free_thresh) + return -1; + + return 0; +} + +static inline int +iavf_tx_vec_queue_default(struct iavf_tx_queue *txq) +{ + if (!txq) + return -1; + + if (txq->offloads & IAVF_NO_VECTOR_FLAGS) + return -1; + + if (txq->rs_thresh < IAVF_VPMD_TX_MAX_BURST || + txq->rs_thresh > IAVF_VPMD_TX_MAX_FREE_BUF) + return -1; + + return 0; +} + +static inline int +iavf_rx_vec_dev_check_default(struct rte_eth_dev *dev) +{ + int i; + struct iavf_rx_queue *rxq; + + for (i = 0; i < dev->data->nb_rx_queues; i++) { + rxq = dev->data->rx_queues[i]; + if (iavf_rx_vec_queue_default(rxq)) + return -1; + } + + return 0; +} + +static inline int +iavf_tx_vec_dev_check_default(struct rte_eth_dev *dev) +{ + int i; + struct iavf_tx_queue *txq; + + for (i = 0; i < dev->data->nb_tx_queues; i++) { + txq = dev->data->tx_queues[i]; + if (iavf_tx_vec_queue_default(txq)) + return -1; + } + + return 0; +} + +#endif diff --git a/src/spdk/dpdk/drivers/net/iavf/iavf_rxtx_vec_sse.c b/src/spdk/dpdk/drivers/net/iavf/iavf_rxtx_vec_sse.c new file mode 100644 index 000000000..85c5bd4af --- /dev/null +++ b/src/spdk/dpdk/drivers/net/iavf/iavf_rxtx_vec_sse.c @@ -0,0 +1,1191 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017 Intel Corporation + */ + +#include +#include +#include + +#include "iavf.h" +#include "iavf_rxtx.h" +#include "iavf_rxtx_vec_common.h" + +#include + +#ifndef __INTEL_COMPILER +#pragma GCC diagnostic ignored "-Wcast-qual" +#endif + +static inline void +iavf_rxq_rearm(struct iavf_rx_queue *rxq) +{ + int i; + uint16_t rx_id; + + volatile union iavf_rx_desc *rxdp; + struct rte_mbuf **rxp = &rxq->sw_ring[rxq->rxrearm_start]; + struct rte_mbuf *mb0, *mb1; + __m128i hdr_room = _mm_set_epi64x(RTE_PKTMBUF_HEADROOM, + RTE_PKTMBUF_HEADROOM); + __m128i dma_addr0, dma_addr1; + + rxdp = rxq->rx_ring + rxq->rxrearm_start; + + /* Pull 'n' more MBUFs into the software ring */ + if (rte_mempool_get_bulk(rxq->mp, (void *)rxp, + rxq->rx_free_thresh) < 0) { + if (rxq->rxrearm_nb + rxq->rx_free_thresh >= rxq->nb_rx_desc) { + dma_addr0 = _mm_setzero_si128(); + for (i = 0; i < IAVF_VPMD_DESCS_PER_LOOP; i++) { + rxp[i] = &rxq->fake_mbuf; + _mm_store_si128((__m128i *)&rxdp[i].read, + dma_addr0); + } + } + rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed += + rxq->rx_free_thresh; + return; + } + + /* Initialize the mbufs in vector, process 2 mbufs in one loop */ + for (i = 0; i < rxq->rx_free_thresh; i += 2, rxp += 2) { + __m128i vaddr0, vaddr1; + + mb0 = rxp[0]; + mb1 = rxp[1]; + + /* load buf_addr(lo 64bit) and buf_iova(hi 64bit) */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) != + offsetof(struct rte_mbuf, buf_addr) + 8); + vaddr0 = _mm_loadu_si128((__m128i *)&mb0->buf_addr); + vaddr1 = _mm_loadu_si128((__m128i *)&mb1->buf_addr); + + /* convert pa to dma_addr hdr/data */ + dma_addr0 = _mm_unpackhi_epi64(vaddr0, vaddr0); + dma_addr1 = _mm_unpackhi_epi64(vaddr1, vaddr1); + + /* add headroom to pa values */ + dma_addr0 = _mm_add_epi64(dma_addr0, hdr_room); + dma_addr1 = _mm_add_epi64(dma_addr1, hdr_room); + + /* flush desc with pa dma_addr */ + _mm_store_si128((__m128i *)&rxdp++->read, dma_addr0); + _mm_store_si128((__m128i *)&rxdp++->read, dma_addr1); + } + + rxq->rxrearm_start += rxq->rx_free_thresh; + if (rxq->rxrearm_start >= rxq->nb_rx_desc) + rxq->rxrearm_start = 0; + + rxq->rxrearm_nb -= rxq->rx_free_thresh; + + rx_id = (uint16_t)((rxq->rxrearm_start == 0) ? + (rxq->nb_rx_desc - 1) : (rxq->rxrearm_start - 1)); + + PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u " + "rearm_start=%u rearm_nb=%u", + rxq->port_id, rxq->queue_id, + rx_id, rxq->rxrearm_start, rxq->rxrearm_nb); + + /* Update the tail pointer on the NIC */ + IAVF_PCI_REG_WRITE(rxq->qrx_tail, rx_id); +} + +static inline void +desc_to_olflags_v(struct iavf_rx_queue *rxq, __m128i descs[4], + struct rte_mbuf **rx_pkts) +{ + const __m128i mbuf_init = _mm_set_epi64x(0, rxq->mbuf_initializer); + __m128i rearm0, rearm1, rearm2, rearm3; + + __m128i vlan0, vlan1, rss, l3_l4e; + + /* mask everything except RSS, flow director and VLAN flags + * bit2 is for VLAN tag, bit11 for flow director indication + * bit13:12 for RSS indication. + */ + const __m128i rss_vlan_msk = _mm_set_epi32( + 0x1c03804, 0x1c03804, 0x1c03804, 0x1c03804); + + const __m128i cksum_mask = _mm_set_epi32( + PKT_RX_IP_CKSUM_GOOD | PKT_RX_IP_CKSUM_BAD | + PKT_RX_L4_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD | + PKT_RX_EIP_CKSUM_BAD, + PKT_RX_IP_CKSUM_GOOD | PKT_RX_IP_CKSUM_BAD | + PKT_RX_L4_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD | + PKT_RX_EIP_CKSUM_BAD, + PKT_RX_IP_CKSUM_GOOD | PKT_RX_IP_CKSUM_BAD | + PKT_RX_L4_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD | + PKT_RX_EIP_CKSUM_BAD, + PKT_RX_IP_CKSUM_GOOD | PKT_RX_IP_CKSUM_BAD | + PKT_RX_L4_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD | + PKT_RX_EIP_CKSUM_BAD); + + /* map rss and vlan type to rss hash and vlan flag */ + const __m128i vlan_flags = _mm_set_epi8(0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED, + 0, 0, 0, 0); + + const __m128i rss_flags = _mm_set_epi8(0, 0, 0, 0, + 0, 0, 0, 0, + PKT_RX_RSS_HASH | PKT_RX_FDIR, PKT_RX_RSS_HASH, 0, 0, + 0, 0, PKT_RX_FDIR, 0); + + const __m128i l3_l4e_flags = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, + /* shift right 1 bit to make sure it not exceed 255 */ + (PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD | + PKT_RX_IP_CKSUM_BAD) >> 1, + (PKT_RX_IP_CKSUM_GOOD | PKT_RX_EIP_CKSUM_BAD | + PKT_RX_L4_CKSUM_BAD) >> 1, + (PKT_RX_EIP_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD) >> 1, + (PKT_RX_IP_CKSUM_GOOD | PKT_RX_EIP_CKSUM_BAD) >> 1, + (PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD) >> 1, + (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD) >> 1, + PKT_RX_IP_CKSUM_BAD >> 1, + (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD) >> 1); + + vlan0 = _mm_unpackhi_epi32(descs[0], descs[1]); + vlan1 = _mm_unpackhi_epi32(descs[2], descs[3]); + vlan0 = _mm_unpacklo_epi64(vlan0, vlan1); + + vlan1 = _mm_and_si128(vlan0, rss_vlan_msk); + vlan0 = _mm_shuffle_epi8(vlan_flags, vlan1); + + rss = _mm_srli_epi32(vlan1, 11); + rss = _mm_shuffle_epi8(rss_flags, rss); + + l3_l4e = _mm_srli_epi32(vlan1, 22); + l3_l4e = _mm_shuffle_epi8(l3_l4e_flags, l3_l4e); + /* then we shift left 1 bit */ + l3_l4e = _mm_slli_epi32(l3_l4e, 1); + /* we need to mask out the reduntant bits */ + l3_l4e = _mm_and_si128(l3_l4e, cksum_mask); + + vlan0 = _mm_or_si128(vlan0, rss); + vlan0 = _mm_or_si128(vlan0, l3_l4e); + + /* At this point, we have the 4 sets of flags in the low 16-bits + * of each 32-bit value in vlan0. + * We want to extract these, and merge them with the mbuf init data + * so we can do a single 16-byte write to the mbuf to set the flags + * and all the other initialization fields. Extracting the + * appropriate flags means that we have to do a shift and blend for + * each mbuf before we do the write. + */ + rearm0 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vlan0, 8), 0x10); + rearm1 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vlan0, 4), 0x10); + rearm2 = _mm_blend_epi16(mbuf_init, vlan0, 0x10); + rearm3 = _mm_blend_epi16(mbuf_init, _mm_srli_si128(vlan0, 4), 0x10); + + /* write the rearm data and the olflags in one write */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, ol_flags) != + offsetof(struct rte_mbuf, rearm_data) + 8); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rearm_data) != + RTE_ALIGN(offsetof(struct rte_mbuf, rearm_data), 16)); + _mm_store_si128((__m128i *)&rx_pkts[0]->rearm_data, rearm0); + _mm_store_si128((__m128i *)&rx_pkts[1]->rearm_data, rearm1); + _mm_store_si128((__m128i *)&rx_pkts[2]->rearm_data, rearm2); + _mm_store_si128((__m128i *)&rx_pkts[3]->rearm_data, rearm3); +} + +static inline __m128i +flex_rxd_to_fdir_flags_vec(const __m128i fdir_id0_3) +{ +#define FDID_MIS_MAGIC 0xFFFFFFFF + RTE_BUILD_BUG_ON(PKT_RX_FDIR != (1 << 2)); + RTE_BUILD_BUG_ON(PKT_RX_FDIR_ID != (1 << 13)); + const __m128i pkt_fdir_bit = _mm_set1_epi32(PKT_RX_FDIR | + PKT_RX_FDIR_ID); + /* desc->flow_id field == 0xFFFFFFFF means fdir mismatch */ + const __m128i fdir_mis_mask = _mm_set1_epi32(FDID_MIS_MAGIC); + __m128i fdir_mask = _mm_cmpeq_epi32(fdir_id0_3, + fdir_mis_mask); + /* this XOR op results to bit-reverse the fdir_mask */ + fdir_mask = _mm_xor_si128(fdir_mask, fdir_mis_mask); + const __m128i fdir_flags = _mm_and_si128(fdir_mask, pkt_fdir_bit); + + return fdir_flags; +} + +static inline void +flex_desc_to_olflags_v(struct iavf_rx_queue *rxq, __m128i descs[4], + struct rte_mbuf **rx_pkts) +{ + const __m128i mbuf_init = _mm_set_epi64x(0, rxq->mbuf_initializer); + __m128i rearm0, rearm1, rearm2, rearm3; + + __m128i tmp_desc, flags, rss_vlan; + + /* mask everything except checksum, RSS and VLAN flags. + * bit6:4 for checksum. + * bit12 for RSS indication. + * bit13 for VLAN indication. + */ + const __m128i desc_mask = _mm_set_epi32(0x3070, 0x3070, + 0x3070, 0x3070); + + const __m128i cksum_mask = _mm_set_epi32(PKT_RX_IP_CKSUM_MASK | + PKT_RX_L4_CKSUM_MASK | + PKT_RX_EIP_CKSUM_BAD, + PKT_RX_IP_CKSUM_MASK | + PKT_RX_L4_CKSUM_MASK | + PKT_RX_EIP_CKSUM_BAD, + PKT_RX_IP_CKSUM_MASK | + PKT_RX_L4_CKSUM_MASK | + PKT_RX_EIP_CKSUM_BAD, + PKT_RX_IP_CKSUM_MASK | + PKT_RX_L4_CKSUM_MASK | + PKT_RX_EIP_CKSUM_BAD); + + /* map the checksum, rss and vlan fields to the checksum, rss + * and vlan flag + */ + const __m128i cksum_flags = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, + /* shift right 1 bit to make sure it not exceed 255 */ + (PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD | + PKT_RX_IP_CKSUM_BAD) >> 1, + (PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD | + PKT_RX_IP_CKSUM_GOOD) >> 1, + (PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_GOOD | + PKT_RX_IP_CKSUM_BAD) >> 1, + (PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_GOOD | + PKT_RX_IP_CKSUM_GOOD) >> 1, + (PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD) >> 1, + (PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_GOOD) >> 1, + (PKT_RX_L4_CKSUM_GOOD | PKT_RX_IP_CKSUM_BAD) >> 1, + (PKT_RX_L4_CKSUM_GOOD | PKT_RX_IP_CKSUM_GOOD) >> 1); + + const __m128i rss_vlan_flags = _mm_set_epi8(0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + PKT_RX_RSS_HASH | PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED, + PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED, + PKT_RX_RSS_HASH, 0); + + /* merge 4 descriptors */ + flags = _mm_unpackhi_epi32(descs[0], descs[1]); + tmp_desc = _mm_unpackhi_epi32(descs[2], descs[3]); + tmp_desc = _mm_unpacklo_epi64(flags, tmp_desc); + tmp_desc = _mm_and_si128(flags, desc_mask); + + /* checksum flags */ + tmp_desc = _mm_srli_epi32(tmp_desc, 4); + flags = _mm_shuffle_epi8(cksum_flags, tmp_desc); + /* then we shift left 1 bit */ + flags = _mm_slli_epi32(flags, 1); + /* we need to mask out the redundant bits introduced by RSS or + * VLAN fields. + */ + flags = _mm_and_si128(flags, cksum_mask); + + /* RSS, VLAN flag */ + tmp_desc = _mm_srli_epi32(tmp_desc, 8); + rss_vlan = _mm_shuffle_epi8(rss_vlan_flags, tmp_desc); + + /* merge the flags */ + flags = _mm_or_si128(flags, rss_vlan); + + if (rxq->fdir_enabled) { + const __m128i fdir_id0_1 = + _mm_unpackhi_epi32(descs[0], descs[1]); + + const __m128i fdir_id2_3 = + _mm_unpackhi_epi32(descs[2], descs[3]); + + const __m128i fdir_id0_3 = + _mm_unpackhi_epi64(fdir_id0_1, fdir_id2_3); + + const __m128i fdir_flags = + flex_rxd_to_fdir_flags_vec(fdir_id0_3); + + /* merge with fdir_flags */ + flags = _mm_or_si128(flags, fdir_flags); + + /* write fdir_id to mbuf */ + rx_pkts[0]->hash.fdir.hi = + _mm_extract_epi32(fdir_id0_3, 0); + + rx_pkts[1]->hash.fdir.hi = + _mm_extract_epi32(fdir_id0_3, 1); + + rx_pkts[2]->hash.fdir.hi = + _mm_extract_epi32(fdir_id0_3, 2); + + rx_pkts[3]->hash.fdir.hi = + _mm_extract_epi32(fdir_id0_3, 3); + } /* if() on fdir_enabled */ + + /** + * At this point, we have the 4 sets of flags in the low 16-bits + * of each 32-bit value in flags. + * We want to extract these, and merge them with the mbuf init data + * so we can do a single 16-byte write to the mbuf to set the flags + * and all the other initialization fields. Extracting the + * appropriate flags means that we have to do a shift and blend for + * each mbuf before we do the write. + */ + rearm0 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(flags, 8), 0x10); + rearm1 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(flags, 4), 0x10); + rearm2 = _mm_blend_epi16(mbuf_init, flags, 0x10); + rearm3 = _mm_blend_epi16(mbuf_init, _mm_srli_si128(flags, 4), 0x10); + + /* write the rearm data and the olflags in one write */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, ol_flags) != + offsetof(struct rte_mbuf, rearm_data) + 8); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rearm_data) != + RTE_ALIGN(offsetof(struct rte_mbuf, rearm_data), 16)); + _mm_store_si128((__m128i *)&rx_pkts[0]->rearm_data, rearm0); + _mm_store_si128((__m128i *)&rx_pkts[1]->rearm_data, rearm1); + _mm_store_si128((__m128i *)&rx_pkts[2]->rearm_data, rearm2); + _mm_store_si128((__m128i *)&rx_pkts[3]->rearm_data, rearm3); +} + +#define PKTLEN_SHIFT 10 + +static inline void +desc_to_ptype_v(__m128i descs[4], struct rte_mbuf **rx_pkts, + const uint32_t *type_table) +{ + __m128i ptype0 = _mm_unpackhi_epi64(descs[0], descs[1]); + __m128i ptype1 = _mm_unpackhi_epi64(descs[2], descs[3]); + + ptype0 = _mm_srli_epi64(ptype0, 30); + ptype1 = _mm_srli_epi64(ptype1, 30); + + rx_pkts[0]->packet_type = type_table[_mm_extract_epi8(ptype0, 0)]; + rx_pkts[1]->packet_type = type_table[_mm_extract_epi8(ptype0, 8)]; + rx_pkts[2]->packet_type = type_table[_mm_extract_epi8(ptype1, 0)]; + rx_pkts[3]->packet_type = type_table[_mm_extract_epi8(ptype1, 8)]; +} + +static inline void +flex_desc_to_ptype_v(__m128i descs[4], struct rte_mbuf **rx_pkts, + const uint32_t *type_table) +{ + const __m128i ptype_mask = _mm_set_epi16(0, IAVF_RX_FLEX_DESC_PTYPE_M, + 0, IAVF_RX_FLEX_DESC_PTYPE_M, + 0, IAVF_RX_FLEX_DESC_PTYPE_M, + 0, IAVF_RX_FLEX_DESC_PTYPE_M); + __m128i ptype_01 = _mm_unpacklo_epi32(descs[0], descs[1]); + __m128i ptype_23 = _mm_unpacklo_epi32(descs[2], descs[3]); + __m128i ptype_all = _mm_unpacklo_epi64(ptype_01, ptype_23); + + ptype_all = _mm_and_si128(ptype_all, ptype_mask); + + rx_pkts[0]->packet_type = type_table[_mm_extract_epi16(ptype_all, 1)]; + rx_pkts[1]->packet_type = type_table[_mm_extract_epi16(ptype_all, 3)]; + rx_pkts[2]->packet_type = type_table[_mm_extract_epi16(ptype_all, 5)]; + rx_pkts[3]->packet_type = type_table[_mm_extract_epi16(ptype_all, 7)]; +} + +/* Notice: + * - nb_pkts < IAVF_VPMD_DESCS_PER_LOOP, just return no packet + * - nb_pkts > IAVF_VPMD_RX_MAX_BURST, only scan IAVF_VPMD_RX_MAX_BURST + * numbers of DD bits + */ +static inline uint16_t +_recv_raw_pkts_vec(struct iavf_rx_queue *rxq, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts, uint8_t *split_packet) +{ + volatile union iavf_rx_desc *rxdp; + struct rte_mbuf **sw_ring; + uint16_t nb_pkts_recd; + int pos; + uint64_t var; + __m128i shuf_msk; + const uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl; + + __m128i crc_adjust = _mm_set_epi16( + 0, 0, 0, /* ignore non-length fields */ + -rxq->crc_len, /* sub crc on data_len */ + 0, /* ignore high-16bits of pkt_len */ + -rxq->crc_len, /* sub crc on pkt_len */ + 0, 0 /* ignore pkt_type field */ + ); + /* compile-time check the above crc_adjust layout is correct. + * NOTE: the first field (lowest address) is given last in set_epi16 + * call above. + */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8); + __m128i dd_check, eop_check; + + /* nb_pkts shall be less equal than IAVF_VPMD_RX_MAX_BURST */ + nb_pkts = RTE_MIN(nb_pkts, IAVF_VPMD_RX_MAX_BURST); + + /* nb_pkts has to be floor-aligned to IAVF_VPMD_DESCS_PER_LOOP */ + nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, IAVF_VPMD_DESCS_PER_LOOP); + + /* Just the act of getting into the function from the application is + * going to cost about 7 cycles + */ + rxdp = rxq->rx_ring + rxq->rx_tail; + + rte_prefetch0(rxdp); + + /* See if we need to rearm the RX queue - gives the prefetch a bit + * of time to act + */ + if (rxq->rxrearm_nb > rxq->rx_free_thresh) + iavf_rxq_rearm(rxq); + + /* Before we start moving massive data around, check to see if + * there is actually a packet available + */ + if (!(rxdp->wb.qword1.status_error_len & + rte_cpu_to_le_32(1 << IAVF_RX_DESC_STATUS_DD_SHIFT))) + return 0; + + /* 4 packets DD mask */ + dd_check = _mm_set_epi64x(0x0000000100000001LL, 0x0000000100000001LL); + + /* 4 packets EOP mask */ + eop_check = _mm_set_epi64x(0x0000000200000002LL, 0x0000000200000002LL); + + /* mask to shuffle from desc. to mbuf */ + shuf_msk = _mm_set_epi8( + 7, 6, 5, 4, /* octet 4~7, 32bits rss */ + 3, 2, /* octet 2~3, low 16 bits vlan_macip */ + 15, 14, /* octet 15~14, 16 bits data_len */ + 0xFF, 0xFF, /* skip high 16 bits pkt_len, zero out */ + 15, 14, /* octet 15~14, low 16 bits pkt_len */ + 0xFF, 0xFF, 0xFF, 0xFF /* pkt_type set as unknown */ + ); + /* Compile-time verify the shuffle mask + * NOTE: some field positions already verified above, but duplicated + * here for completeness in case of future modifications. + */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, vlan_tci) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 10); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, hash) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 12); + + /* Cache is empty -> need to scan the buffer rings, but first move + * the next 'n' mbufs into the cache + */ + sw_ring = &rxq->sw_ring[rxq->rx_tail]; + + /* A. load 4 packet in one loop + * [A*. mask out 4 unused dirty field in desc] + * B. copy 4 mbuf point from swring to rx_pkts + * C. calc the number of DD bits among the 4 packets + * [C*. extract the end-of-packet bit, if requested] + * D. fill info. from desc to mbuf + */ + + for (pos = 0, nb_pkts_recd = 0; pos < nb_pkts; + pos += IAVF_VPMD_DESCS_PER_LOOP, + rxdp += IAVF_VPMD_DESCS_PER_LOOP) { + __m128i descs[IAVF_VPMD_DESCS_PER_LOOP]; + __m128i pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4; + __m128i zero, staterr, sterr_tmp1, sterr_tmp2; + /* 2 64 bit or 4 32 bit mbuf pointers in one XMM reg. */ + __m128i mbp1; +#if defined(RTE_ARCH_X86_64) + __m128i mbp2; +#endif + + /* B.1 load 2 (64 bit) or 4 (32 bit) mbuf points */ + mbp1 = _mm_loadu_si128((__m128i *)&sw_ring[pos]); + /* Read desc statuses backwards to avoid race condition */ + /* A.1 load 4 pkts desc */ + descs[3] = _mm_loadu_si128((__m128i *)(rxdp + 3)); + rte_compiler_barrier(); + + /* B.2 copy 2 64 bit or 4 32 bit mbuf point into rx_pkts */ + _mm_storeu_si128((__m128i *)&rx_pkts[pos], mbp1); + +#if defined(RTE_ARCH_X86_64) + /* B.1 load 2 64 bit mbuf points */ + mbp2 = _mm_loadu_si128((__m128i *)&sw_ring[pos + 2]); +#endif + + descs[2] = _mm_loadu_si128((__m128i *)(rxdp + 2)); + rte_compiler_barrier(); + /* B.1 load 2 mbuf point */ + descs[1] = _mm_loadu_si128((__m128i *)(rxdp + 1)); + rte_compiler_barrier(); + descs[0] = _mm_loadu_si128((__m128i *)(rxdp)); + +#if defined(RTE_ARCH_X86_64) + /* B.2 copy 2 mbuf point into rx_pkts */ + _mm_storeu_si128((__m128i *)&rx_pkts[pos + 2], mbp2); +#endif + + if (split_packet) { + rte_mbuf_prefetch_part2(rx_pkts[pos]); + rte_mbuf_prefetch_part2(rx_pkts[pos + 1]); + rte_mbuf_prefetch_part2(rx_pkts[pos + 2]); + rte_mbuf_prefetch_part2(rx_pkts[pos + 3]); + } + + /* avoid compiler reorder optimization */ + rte_compiler_barrier(); + + /* pkt 3,4 shift the pktlen field to be 16-bit aligned*/ + const __m128i len3 = _mm_slli_epi32(descs[3], PKTLEN_SHIFT); + const __m128i len2 = _mm_slli_epi32(descs[2], PKTLEN_SHIFT); + + /* merge the now-aligned packet length fields back in */ + descs[3] = _mm_blend_epi16(descs[3], len3, 0x80); + descs[2] = _mm_blend_epi16(descs[2], len2, 0x80); + + /* D.1 pkt 3,4 convert format from desc to pktmbuf */ + pkt_mb4 = _mm_shuffle_epi8(descs[3], shuf_msk); + pkt_mb3 = _mm_shuffle_epi8(descs[2], shuf_msk); + + /* C.1 4=>2 status err info only */ + sterr_tmp2 = _mm_unpackhi_epi32(descs[3], descs[2]); + sterr_tmp1 = _mm_unpackhi_epi32(descs[1], descs[0]); + + desc_to_olflags_v(rxq, descs, &rx_pkts[pos]); + + /* D.2 pkt 3,4 set in_port/nb_seg and remove crc */ + pkt_mb4 = _mm_add_epi16(pkt_mb4, crc_adjust); + pkt_mb3 = _mm_add_epi16(pkt_mb3, crc_adjust); + + /* pkt 1,2 shift the pktlen field to be 16-bit aligned*/ + const __m128i len1 = _mm_slli_epi32(descs[1], PKTLEN_SHIFT); + const __m128i len0 = _mm_slli_epi32(descs[0], PKTLEN_SHIFT); + + /* merge the now-aligned packet length fields back in */ + descs[1] = _mm_blend_epi16(descs[1], len1, 0x80); + descs[0] = _mm_blend_epi16(descs[0], len0, 0x80); + + /* D.1 pkt 1,2 convert format from desc to pktmbuf */ + pkt_mb2 = _mm_shuffle_epi8(descs[1], shuf_msk); + pkt_mb1 = _mm_shuffle_epi8(descs[0], shuf_msk); + + /* C.2 get 4 pkts status err value */ + zero = _mm_xor_si128(dd_check, dd_check); + staterr = _mm_unpacklo_epi32(sterr_tmp1, sterr_tmp2); + + /* D.3 copy final 3,4 data to rx_pkts */ + _mm_storeu_si128( + (void *)&rx_pkts[pos + 3]->rx_descriptor_fields1, + pkt_mb4); + _mm_storeu_si128( + (void *)&rx_pkts[pos + 2]->rx_descriptor_fields1, + pkt_mb3); + + /* D.2 pkt 1,2 remove crc */ + pkt_mb2 = _mm_add_epi16(pkt_mb2, crc_adjust); + pkt_mb1 = _mm_add_epi16(pkt_mb1, crc_adjust); + + /* C* extract and record EOP bit */ + if (split_packet) { + __m128i eop_shuf_mask = _mm_set_epi8( + 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, + 0x04, 0x0C, 0x00, 0x08 + ); + + /* and with mask to extract bits, flipping 1-0 */ + __m128i eop_bits = _mm_andnot_si128(staterr, eop_check); + /* the staterr values are not in order, as the count + * count of dd bits doesn't care. However, for end of + * packet tracking, we do care, so shuffle. This also + * compresses the 32-bit values to 8-bit + */ + eop_bits = _mm_shuffle_epi8(eop_bits, eop_shuf_mask); + /* store the resulting 32-bit value */ + *(int *)split_packet = _mm_cvtsi128_si32(eop_bits); + split_packet += IAVF_VPMD_DESCS_PER_LOOP; + } + + /* C.3 calc available number of desc */ + staterr = _mm_and_si128(staterr, dd_check); + staterr = _mm_packs_epi32(staterr, zero); + + /* D.3 copy final 1,2 data to rx_pkts */ + _mm_storeu_si128( + (void *)&rx_pkts[pos + 1]->rx_descriptor_fields1, + pkt_mb2); + _mm_storeu_si128((void *)&rx_pkts[pos]->rx_descriptor_fields1, + pkt_mb1); + desc_to_ptype_v(descs, &rx_pkts[pos], ptype_tbl); + /* C.4 calc avaialbe number of desc */ + var = __builtin_popcountll(_mm_cvtsi128_si64(staterr)); + nb_pkts_recd += var; + if (likely(var != IAVF_VPMD_DESCS_PER_LOOP)) + break; + } + + /* Update our internal tail pointer */ + rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_pkts_recd); + rxq->rx_tail = (uint16_t)(rxq->rx_tail & (rxq->nb_rx_desc - 1)); + rxq->rxrearm_nb = (uint16_t)(rxq->rxrearm_nb + nb_pkts_recd); + + return nb_pkts_recd; +} + +/* Notice: + * - nb_pkts < IAVF_VPMD_DESCS_PER_LOOP, just return no packet + * - nb_pkts > IAVF_VPMD_RX_MAX_BURST, only scan IAVF_VPMD_RX_MAX_BURST + * numbers of DD bits + */ +static inline uint16_t +_recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq, + struct rte_mbuf **rx_pkts, + uint16_t nb_pkts, uint8_t *split_packet) +{ + volatile union iavf_rx_flex_desc *rxdp; + struct rte_mbuf **sw_ring; + uint16_t nb_pkts_recd; + int pos; + uint64_t var; + const uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl; + __m128i crc_adjust = _mm_set_epi16 + (0, 0, 0, /* ignore non-length fields */ + -rxq->crc_len, /* sub crc on data_len */ + 0, /* ignore high-16bits of pkt_len */ + -rxq->crc_len, /* sub crc on pkt_len */ + 0, 0 /* ignore pkt_type field */ + ); + const __m128i zero = _mm_setzero_si128(); + /* mask to shuffle from desc. to mbuf */ + const __m128i shuf_msk = _mm_set_epi8 + (0xFF, 0xFF, + 0xFF, 0xFF, /* rss hash parsed separately */ + 11, 10, /* octet 10~11, 16 bits vlan_macip */ + 5, 4, /* octet 4~5, 16 bits data_len */ + 0xFF, 0xFF, /* skip high 16 bits pkt_len, zero out */ + 5, 4, /* octet 4~5, low 16 bits pkt_len */ + 0xFF, 0xFF, /* pkt_type set as unknown */ + 0xFF, 0xFF /* pkt_type set as unknown */ + ); + const __m128i eop_shuf_mask = _mm_set_epi8(0xFF, 0xFF, + 0xFF, 0xFF, + 0xFF, 0xFF, + 0xFF, 0xFF, + 0xFF, 0xFF, + 0xFF, 0xFF, + 0x04, 0x0C, + 0x00, 0x08); + + /** + * compile-time check the above crc_adjust layout is correct. + * NOTE: the first field (lowest address) is given last in set_epi16 + * call above. + */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8); + + /* 4 packets DD mask */ + const __m128i dd_check = _mm_set_epi64x(0x0000000100000001LL, + 0x0000000100000001LL); + /* 4 packets EOP mask */ + const __m128i eop_check = _mm_set_epi64x(0x0000000200000002LL, + 0x0000000200000002LL); + + /* nb_pkts shall be less equal than IAVF_VPMD_RX_MAX_BURST */ + nb_pkts = RTE_MIN(nb_pkts, IAVF_VPMD_RX_MAX_BURST); + + /* nb_pkts has to be floor-aligned to IAVF_VPMD_DESCS_PER_LOOP */ + nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, IAVF_VPMD_DESCS_PER_LOOP); + + /* Just the act of getting into the function from the application is + * going to cost about 7 cycles + */ + rxdp = (union iavf_rx_flex_desc *)rxq->rx_ring + rxq->rx_tail; + + rte_prefetch0(rxdp); + + /* See if we need to rearm the RX queue - gives the prefetch a bit + * of time to act + */ + if (rxq->rxrearm_nb > rxq->rx_free_thresh) + iavf_rxq_rearm(rxq); + + /* Before we start moving massive data around, check to see if + * there is actually a packet available + */ + if (!(rxdp->wb.status_error0 & + rte_cpu_to_le_32(1 << IAVF_RX_FLEX_DESC_STATUS0_DD_S))) + return 0; + + /** + * Compile-time verify the shuffle mask + * NOTE: some field positions already verified above, but duplicated + * here for completeness in case of future modifications. + */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, vlan_tci) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 10); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, hash) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 12); + + /* Cache is empty -> need to scan the buffer rings, but first move + * the next 'n' mbufs into the cache + */ + sw_ring = &rxq->sw_ring[rxq->rx_tail]; + + /* A. load 4 packet in one loop + * [A*. mask out 4 unused dirty field in desc] + * B. copy 4 mbuf point from swring to rx_pkts + * C. calc the number of DD bits among the 4 packets + * [C*. extract the end-of-packet bit, if requested] + * D. fill info. from desc to mbuf + */ + + for (pos = 0, nb_pkts_recd = 0; pos < nb_pkts; + pos += IAVF_VPMD_DESCS_PER_LOOP, + rxdp += IAVF_VPMD_DESCS_PER_LOOP) { + __m128i descs[IAVF_VPMD_DESCS_PER_LOOP]; + __m128i pkt_mb0, pkt_mb1, pkt_mb2, pkt_mb3; + __m128i staterr, sterr_tmp1, sterr_tmp2; + /* 2 64 bit or 4 32 bit mbuf pointers in one XMM reg. */ + __m128i mbp1; +#if defined(RTE_ARCH_X86_64) + __m128i mbp2; +#endif + + /* B.1 load 2 (64 bit) or 4 (32 bit) mbuf points */ + mbp1 = _mm_loadu_si128((__m128i *)&sw_ring[pos]); + /* Read desc statuses backwards to avoid race condition */ + /* A.1 load 4 pkts desc */ + descs[3] = _mm_loadu_si128((__m128i *)(rxdp + 3)); + rte_compiler_barrier(); + + /* B.2 copy 2 64 bit or 4 32 bit mbuf point into rx_pkts */ + _mm_storeu_si128((__m128i *)&rx_pkts[pos], mbp1); + +#if defined(RTE_ARCH_X86_64) + /* B.1 load 2 64 bit mbuf points */ + mbp2 = _mm_loadu_si128((__m128i *)&sw_ring[pos + 2]); +#endif + + descs[2] = _mm_loadu_si128((__m128i *)(rxdp + 2)); + rte_compiler_barrier(); + /* B.1 load 2 mbuf point */ + descs[1] = _mm_loadu_si128((__m128i *)(rxdp + 1)); + rte_compiler_barrier(); + descs[0] = _mm_loadu_si128((__m128i *)(rxdp)); + +#if defined(RTE_ARCH_X86_64) + /* B.2 copy 2 mbuf point into rx_pkts */ + _mm_storeu_si128((__m128i *)&rx_pkts[pos + 2], mbp2); +#endif + + if (split_packet) { + rte_mbuf_prefetch_part2(rx_pkts[pos]); + rte_mbuf_prefetch_part2(rx_pkts[pos + 1]); + rte_mbuf_prefetch_part2(rx_pkts[pos + 2]); + rte_mbuf_prefetch_part2(rx_pkts[pos + 3]); + } + + /* avoid compiler reorder optimization */ + rte_compiler_barrier(); + + /* D.1 pkt 3,4 convert format from desc to pktmbuf */ + pkt_mb3 = _mm_shuffle_epi8(descs[3], shuf_msk); + pkt_mb2 = _mm_shuffle_epi8(descs[2], shuf_msk); + + /* D.1 pkt 1,2 convert format from desc to pktmbuf */ + pkt_mb1 = _mm_shuffle_epi8(descs[1], shuf_msk); + pkt_mb0 = _mm_shuffle_epi8(descs[0], shuf_msk); + + /* C.1 4=>2 filter staterr info only */ + sterr_tmp2 = _mm_unpackhi_epi32(descs[3], descs[2]); + /* C.1 4=>2 filter staterr info only */ + sterr_tmp1 = _mm_unpackhi_epi32(descs[1], descs[0]); + + flex_desc_to_olflags_v(rxq, descs, &rx_pkts[pos]); + + /* D.2 pkt 3,4 set in_port/nb_seg and remove crc */ + pkt_mb3 = _mm_add_epi16(pkt_mb3, crc_adjust); + pkt_mb2 = _mm_add_epi16(pkt_mb2, crc_adjust); + + /* D.2 pkt 1,2 set in_port/nb_seg and remove crc */ + pkt_mb1 = _mm_add_epi16(pkt_mb1, crc_adjust); + pkt_mb0 = _mm_add_epi16(pkt_mb0, crc_adjust); + +#ifndef RTE_LIBRTE_IAVF_16BYTE_RX_DESC + /** + * needs to load 2nd 16B of each desc for RSS hash parsing, + * will cause performance drop to get into this context. + */ + if (rxq->vsi->adapter->eth_dev->data->dev_conf.rxmode.offloads & + DEV_RX_OFFLOAD_RSS_HASH) { + /* load bottom half of every 32B desc */ + const __m128i raw_desc_bh3 = + _mm_load_si128 + ((void *)(&rxdp[3].wb.status_error1)); + rte_compiler_barrier(); + const __m128i raw_desc_bh2 = + _mm_load_si128 + ((void *)(&rxdp[2].wb.status_error1)); + rte_compiler_barrier(); + const __m128i raw_desc_bh1 = + _mm_load_si128 + ((void *)(&rxdp[1].wb.status_error1)); + rte_compiler_barrier(); + const __m128i raw_desc_bh0 = + _mm_load_si128 + ((void *)(&rxdp[0].wb.status_error1)); + + /** + * to shift the 32b RSS hash value to the + * highest 32b of each 128b before mask + */ + __m128i rss_hash3 = + _mm_slli_epi64(raw_desc_bh3, 32); + __m128i rss_hash2 = + _mm_slli_epi64(raw_desc_bh2, 32); + __m128i rss_hash1 = + _mm_slli_epi64(raw_desc_bh1, 32); + __m128i rss_hash0 = + _mm_slli_epi64(raw_desc_bh0, 32); + + __m128i rss_hash_msk = + _mm_set_epi32(0xFFFFFFFF, 0, 0, 0); + + rss_hash3 = _mm_and_si128 + (rss_hash3, rss_hash_msk); + rss_hash2 = _mm_and_si128 + (rss_hash2, rss_hash_msk); + rss_hash1 = _mm_and_si128 + (rss_hash1, rss_hash_msk); + rss_hash0 = _mm_and_si128 + (rss_hash0, rss_hash_msk); + + pkt_mb3 = _mm_or_si128(pkt_mb3, rss_hash3); + pkt_mb2 = _mm_or_si128(pkt_mb2, rss_hash2); + pkt_mb1 = _mm_or_si128(pkt_mb1, rss_hash1); + pkt_mb0 = _mm_or_si128(pkt_mb0, rss_hash0); + } /* if() on RSS hash parsing */ +#endif + + /* C.2 get 4 pkts staterr value */ + staterr = _mm_unpacklo_epi32(sterr_tmp1, sterr_tmp2); + + /* D.3 copy final 3,4 data to rx_pkts */ + _mm_storeu_si128 + ((void *)&rx_pkts[pos + 3]->rx_descriptor_fields1, + pkt_mb3); + _mm_storeu_si128 + ((void *)&rx_pkts[pos + 2]->rx_descriptor_fields1, + pkt_mb2); + + /* C* extract and record EOP bit */ + if (split_packet) { + /* and with mask to extract bits, flipping 1-0 */ + __m128i eop_bits = _mm_andnot_si128(staterr, eop_check); + /* the staterr values are not in order, as the count + * count of dd bits doesn't care. However, for end of + * packet tracking, we do care, so shuffle. This also + * compresses the 32-bit values to 8-bit + */ + eop_bits = _mm_shuffle_epi8(eop_bits, eop_shuf_mask); + /* store the resulting 32-bit value */ + *(int *)split_packet = _mm_cvtsi128_si32(eop_bits); + split_packet += IAVF_VPMD_DESCS_PER_LOOP; + } + + /* C.3 calc available number of desc */ + staterr = _mm_and_si128(staterr, dd_check); + staterr = _mm_packs_epi32(staterr, zero); + + /* D.3 copy final 1,2 data to rx_pkts */ + _mm_storeu_si128 + ((void *)&rx_pkts[pos + 1]->rx_descriptor_fields1, + pkt_mb1); + _mm_storeu_si128((void *)&rx_pkts[pos]->rx_descriptor_fields1, + pkt_mb0); + flex_desc_to_ptype_v(descs, &rx_pkts[pos], ptype_tbl); + /* C.4 calc available number of desc */ + var = __builtin_popcountll(_mm_cvtsi128_si64(staterr)); + nb_pkts_recd += var; + if (likely(var != IAVF_VPMD_DESCS_PER_LOOP)) + break; + } + + /* Update our internal tail pointer */ + rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_pkts_recd); + rxq->rx_tail = (uint16_t)(rxq->rx_tail & (rxq->nb_rx_desc - 1)); + rxq->rxrearm_nb = (uint16_t)(rxq->rxrearm_nb + nb_pkts_recd); + + return nb_pkts_recd; +} + +/* Notice: + * - nb_pkts < IAVF_DESCS_PER_LOOP, just return no packet + * - nb_pkts > IAVF_VPMD_RX_MAX_BURST, only scan IAVF_VPMD_RX_MAX_BURST + * numbers of DD bits + */ +uint16_t +iavf_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + return _recv_raw_pkts_vec(rx_queue, rx_pkts, nb_pkts, NULL); +} + +/* Notice: + * - nb_pkts < IAVF_DESCS_PER_LOOP, just return no packet + * - nb_pkts > IAVF_VPMD_RX_MAX_BURST, only scan IAVF_VPMD_RX_MAX_BURST + * numbers of DD bits + */ +uint16_t +iavf_recv_pkts_vec_flex_rxd(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + return _recv_raw_pkts_vec_flex_rxd(rx_queue, rx_pkts, nb_pkts, NULL); +} + +/* vPMD receive routine that reassembles scattered packets + * Notice: + * - nb_pkts < IAVF_VPMD_DESCS_PER_LOOP, just return no packet + * - nb_pkts > VPMD_RX_MAX_BURST, only scan IAVF_VPMD_RX_MAX_BURST + * numbers of DD bits + */ +uint16_t +iavf_recv_scattered_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + struct iavf_rx_queue *rxq = rx_queue; + uint8_t split_flags[IAVF_VPMD_RX_MAX_BURST] = {0}; + unsigned int i = 0; + + /* get some new buffers */ + uint16_t nb_bufs = _recv_raw_pkts_vec(rxq, rx_pkts, nb_pkts, + split_flags); + if (nb_bufs == 0) + return 0; + + /* happy day case, full burst + no packets to be joined */ + const uint64_t *split_fl64 = (uint64_t *)split_flags; + + if (!rxq->pkt_first_seg && + split_fl64[0] == 0 && split_fl64[1] == 0 && + split_fl64[2] == 0 && split_fl64[3] == 0) + return nb_bufs; + + /* reassemble any packets that need reassembly*/ + if (!rxq->pkt_first_seg) { + /* find the first split flag, and only reassemble then*/ + while (i < nb_bufs && !split_flags[i]) + i++; + if (i == nb_bufs) + return nb_bufs; + rxq->pkt_first_seg = rx_pkts[i]; + } + return i + reassemble_packets(rxq, &rx_pkts[i], nb_bufs - i, + &split_flags[i]); +} + +/* vPMD receive routine that reassembles scattered packets for flex RxD + * Notice: + * - nb_pkts < IAVF_VPMD_DESCS_PER_LOOP, just return no packet + * - nb_pkts > VPMD_RX_MAX_BURST, only scan IAVF_VPMD_RX_MAX_BURST + * numbers of DD bits + */ +uint16_t +iavf_recv_scattered_pkts_vec_flex_rxd(void *rx_queue, + struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + struct iavf_rx_queue *rxq = rx_queue; + uint8_t split_flags[IAVF_VPMD_RX_MAX_BURST] = {0}; + unsigned int i = 0; + + /* get some new buffers */ + uint16_t nb_bufs = _recv_raw_pkts_vec_flex_rxd(rxq, rx_pkts, nb_pkts, + split_flags); + if (nb_bufs == 0) + return 0; + + /* happy day case, full burst + no packets to be joined */ + const uint64_t *split_fl64 = (uint64_t *)split_flags; + + if (!rxq->pkt_first_seg && + split_fl64[0] == 0 && split_fl64[1] == 0 && + split_fl64[2] == 0 && split_fl64[3] == 0) + return nb_bufs; + + /* reassemble any packets that need reassembly*/ + if (!rxq->pkt_first_seg) { + /* find the first split flag, and only reassemble then*/ + while (i < nb_bufs && !split_flags[i]) + i++; + if (i == nb_bufs) + return nb_bufs; + rxq->pkt_first_seg = rx_pkts[i]; + } + return i + reassemble_packets(rxq, &rx_pkts[i], nb_bufs - i, + &split_flags[i]); +} + +static inline void +vtx1(volatile struct iavf_tx_desc *txdp, struct rte_mbuf *pkt, uint64_t flags) +{ + uint64_t high_qw = + (IAVF_TX_DESC_DTYPE_DATA | + ((uint64_t)flags << IAVF_TXD_QW1_CMD_SHIFT) | + ((uint64_t)pkt->data_len << + IAVF_TXD_QW1_TX_BUF_SZ_SHIFT)); + + __m128i descriptor = _mm_set_epi64x(high_qw, + pkt->buf_iova + pkt->data_off); + _mm_store_si128((__m128i *)txdp, descriptor); +} + +static inline void +iavf_vtx(volatile struct iavf_tx_desc *txdp, struct rte_mbuf **pkt, + uint16_t nb_pkts, uint64_t flags) +{ + int i; + + for (i = 0; i < nb_pkts; ++i, ++txdp, ++pkt) + vtx1(txdp, *pkt, flags); +} + +uint16_t +iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + struct iavf_tx_queue *txq = (struct iavf_tx_queue *)tx_queue; + volatile struct iavf_tx_desc *txdp; + struct iavf_tx_entry *txep; + uint16_t n, nb_commit, tx_id; + uint64_t flags = IAVF_TX_DESC_CMD_EOP | 0x04; /* bit 2 must be set */ + uint64_t rs = IAVF_TX_DESC_CMD_RS | flags; + int i; + + /* cross rx_thresh boundary is not allowed */ + nb_pkts = RTE_MIN(nb_pkts, txq->rs_thresh); + + if (txq->nb_free < txq->free_thresh) + iavf_tx_free_bufs(txq); + + nb_pkts = (uint16_t)RTE_MIN(txq->nb_free, nb_pkts); + if (unlikely(nb_pkts == 0)) + return 0; + nb_commit = nb_pkts; + + tx_id = txq->tx_tail; + txdp = &txq->tx_ring[tx_id]; + txep = &txq->sw_ring[tx_id]; + + txq->nb_free = (uint16_t)(txq->nb_free - nb_pkts); + + n = (uint16_t)(txq->nb_tx_desc - tx_id); + if (nb_commit >= n) { + tx_backlog_entry(txep, tx_pkts, n); + + for (i = 0; i < n - 1; ++i, ++tx_pkts, ++txdp) + vtx1(txdp, *tx_pkts, flags); + + vtx1(txdp, *tx_pkts++, rs); + + nb_commit = (uint16_t)(nb_commit - n); + + tx_id = 0; + txq->next_rs = (uint16_t)(txq->rs_thresh - 1); + + /* avoid reach the end of ring */ + txdp = &txq->tx_ring[tx_id]; + txep = &txq->sw_ring[tx_id]; + } + + tx_backlog_entry(txep, tx_pkts, nb_commit); + + iavf_vtx(txdp, tx_pkts, nb_commit, flags); + + tx_id = (uint16_t)(tx_id + nb_commit); + if (tx_id > txq->next_rs) { + txq->tx_ring[txq->next_rs].cmd_type_offset_bsz |= + rte_cpu_to_le_64(((uint64_t)IAVF_TX_DESC_CMD_RS) << + IAVF_TXD_QW1_CMD_SHIFT); + txq->next_rs = + (uint16_t)(txq->next_rs + txq->rs_thresh); + } + + txq->tx_tail = tx_id; + + PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_pkts=%u", + txq->port_id, txq->queue_id, tx_id, nb_pkts); + + IAVF_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail); + + return nb_pkts; +} + +uint16_t +iavf_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + uint16_t nb_tx = 0; + struct iavf_tx_queue *txq = (struct iavf_tx_queue *)tx_queue; + + while (nb_pkts) { + uint16_t ret, num; + + num = (uint16_t)RTE_MIN(nb_pkts, txq->rs_thresh); + ret = iavf_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx], num); + nb_tx += ret; + nb_pkts -= ret; + if (ret < num) + break; + } + + return nb_tx; +} + +static void __rte_cold +iavf_rx_queue_release_mbufs_sse(struct iavf_rx_queue *rxq) +{ + _iavf_rx_queue_release_mbufs_vec(rxq); +} + +static void __rte_cold +iavf_tx_queue_release_mbufs_sse(struct iavf_tx_queue *txq) +{ + _iavf_tx_queue_release_mbufs_vec(txq); +} + +static const struct iavf_rxq_ops sse_vec_rxq_ops = { + .release_mbufs = iavf_rx_queue_release_mbufs_sse, +}; + +static const struct iavf_txq_ops sse_vec_txq_ops = { + .release_mbufs = iavf_tx_queue_release_mbufs_sse, +}; + +int __rte_cold +iavf_txq_vec_setup(struct iavf_tx_queue *txq) +{ + txq->ops = &sse_vec_txq_ops; + return 0; +} + +int __rte_cold +iavf_rxq_vec_setup(struct iavf_rx_queue *rxq) +{ + rxq->ops = &sse_vec_rxq_ops; + return iavf_rxq_vec_setup_default(rxq); +} + +int __rte_cold +iavf_rx_vec_dev_check(struct rte_eth_dev *dev) +{ + return iavf_rx_vec_dev_check_default(dev); +} + +int __rte_cold +iavf_tx_vec_dev_check(struct rte_eth_dev *dev) +{ + return iavf_tx_vec_dev_check_default(dev); +} diff --git a/src/spdk/dpdk/drivers/net/iavf/iavf_vchnl.c b/src/spdk/dpdk/drivers/net/iavf/iavf_vchnl.c new file mode 100644 index 000000000..33acea54a --- /dev/null +++ b/src/spdk/dpdk/drivers/net/iavf/iavf_vchnl.c @@ -0,0 +1,1077 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017 Intel Corporation + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "iavf.h" +#include "iavf_rxtx.h" + +#define MAX_TRY_TIMES 200 +#define ASQ_DELAY_MS 10 + +/* Read data in admin queue to get msg from pf driver */ +static enum iavf_status +iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len, + uint8_t *buf) +{ + struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter); + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + struct iavf_arq_event_info event; + enum virtchnl_ops opcode; + int ret; + + event.buf_len = buf_len; + event.msg_buf = buf; + ret = iavf_clean_arq_element(hw, &event, NULL); + /* Can't read any msg from adminQ */ + if (ret) { + PMD_DRV_LOG(DEBUG, "Can't read msg from AQ"); + return ret; + } + + opcode = (enum virtchnl_ops)rte_le_to_cpu_32(event.desc.cookie_high); + vf->cmd_retval = (enum virtchnl_status_code)rte_le_to_cpu_32( + event.desc.cookie_low); + + PMD_DRV_LOG(DEBUG, "AQ from pf carries opcode %u, retval %d", + opcode, vf->cmd_retval); + + if (opcode != vf->pend_cmd) { + PMD_DRV_LOG(WARNING, "command mismatch, expect %u, get %u", + vf->pend_cmd, opcode); + return IAVF_ERR_OPCODE_MISMATCH; + } + + return IAVF_SUCCESS; +} + +static int +iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args) +{ + struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter); + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + enum iavf_status ret; + int err = 0; + int i = 0; + + if (_atomic_set_cmd(vf, args->ops)) + return -1; + + ret = iavf_aq_send_msg_to_pf(hw, args->ops, IAVF_SUCCESS, + args->in_args, args->in_args_size, NULL); + if (ret) { + PMD_DRV_LOG(ERR, "fail to send cmd %d", args->ops); + _clear_cmd(vf); + return err; + } + + switch (args->ops) { + case VIRTCHNL_OP_RESET_VF: + /*no need to wait for response */ + _clear_cmd(vf); + break; + case VIRTCHNL_OP_VERSION: + case VIRTCHNL_OP_GET_VF_RESOURCES: + case VIRTCHNL_OP_GET_SUPPORTED_RXDIDS: + /* for init virtchnl ops, need to poll the response */ + do { + ret = iavf_read_msg_from_pf(adapter, args->out_size, + args->out_buffer); + if (ret == IAVF_SUCCESS) + break; + rte_delay_ms(ASQ_DELAY_MS); + } while (i++ < MAX_TRY_TIMES); + if (i >= MAX_TRY_TIMES || + vf->cmd_retval != VIRTCHNL_STATUS_SUCCESS) { + err = -1; + PMD_DRV_LOG(ERR, "No response or return failure (%d)" + " for cmd %d", vf->cmd_retval, args->ops); + } + _clear_cmd(vf); + break; + + default: + /* For other virtchnl ops in running time, + * wait for the cmd done flag. + */ + do { + if (vf->pend_cmd == VIRTCHNL_OP_UNKNOWN) + break; + rte_delay_ms(ASQ_DELAY_MS); + /* If don't read msg or read sys event, continue */ + } while (i++ < MAX_TRY_TIMES); + /* If there's no response is received, clear command */ + if (i >= MAX_TRY_TIMES || + vf->cmd_retval != VIRTCHNL_STATUS_SUCCESS) { + err = -1; + PMD_DRV_LOG(ERR, "No response or return failure (%d)" + " for cmd %d", vf->cmd_retval, args->ops); + _clear_cmd(vf); + } + break; + } + + return err; +} + +static uint32_t +iavf_convert_link_speed(enum virtchnl_link_speed virt_link_speed) +{ + uint32_t speed; + + switch (virt_link_speed) { + case VIRTCHNL_LINK_SPEED_100MB: + speed = 100; + break; + case VIRTCHNL_LINK_SPEED_1GB: + speed = 1000; + break; + case VIRTCHNL_LINK_SPEED_10GB: + speed = 10000; + break; + case VIRTCHNL_LINK_SPEED_40GB: + speed = 40000; + break; + case VIRTCHNL_LINK_SPEED_20GB: + speed = 20000; + break; + case VIRTCHNL_LINK_SPEED_25GB: + speed = 25000; + break; + case VIRTCHNL_LINK_SPEED_2_5GB: + speed = 2500; + break; + case VIRTCHNL_LINK_SPEED_5GB: + speed = 5000; + break; + default: + speed = 0; + break; + } + + return speed; +} + +static void +iavf_handle_pf_event_msg(struct rte_eth_dev *dev, uint8_t *msg, + uint16_t msglen) +{ + struct virtchnl_pf_event *pf_msg = + (struct virtchnl_pf_event *)msg; + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); + + if (msglen < sizeof(struct virtchnl_pf_event)) { + PMD_DRV_LOG(DEBUG, "Error event"); + return; + } + switch (pf_msg->event) { + case VIRTCHNL_EVENT_RESET_IMPENDING: + PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_RESET_IMPENDING event"); + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET, + NULL); + break; + case VIRTCHNL_EVENT_LINK_CHANGE: + PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_LINK_CHANGE event"); + vf->link_up = pf_msg->event_data.link_event.link_status; + if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_CAP_ADV_LINK_SPEED) { + vf->link_speed = + pf_msg->event_data.link_event_adv.link_speed; + } else { + enum virtchnl_link_speed speed; + speed = pf_msg->event_data.link_event.link_speed; + vf->link_speed = iavf_convert_link_speed(speed); + } + iavf_dev_link_update(dev, 0); + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, + NULL); + break; + case VIRTCHNL_EVENT_PF_DRIVER_CLOSE: + PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_PF_DRIVER_CLOSE event"); + break; + default: + PMD_DRV_LOG(ERR, " unknown event received %u", pf_msg->event); + break; + } +} + +void +iavf_handle_virtchnl_msg(struct rte_eth_dev *dev) +{ + struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); + struct iavf_arq_event_info info; + uint16_t pending, aq_opc; + enum virtchnl_ops msg_opc; + enum iavf_status msg_ret; + int ret; + + info.buf_len = IAVF_AQ_BUF_SZ; + if (!vf->aq_resp) { + PMD_DRV_LOG(ERR, "Buffer for adminq resp should not be NULL"); + return; + } + info.msg_buf = vf->aq_resp; + + pending = 1; + while (pending) { + ret = iavf_clean_arq_element(hw, &info, &pending); + + if (ret != IAVF_SUCCESS) { + PMD_DRV_LOG(INFO, "Failed to read msg from AdminQ," + "ret: %d", ret); + break; + } + aq_opc = rte_le_to_cpu_16(info.desc.opcode); + /* For the message sent from pf to vf, opcode is stored in + * cookie_high of struct iavf_aq_desc, while return error code + * are stored in cookie_low, Which is done by PF driver. + */ + msg_opc = (enum virtchnl_ops)rte_le_to_cpu_32( + info.desc.cookie_high); + msg_ret = (enum iavf_status)rte_le_to_cpu_32( + info.desc.cookie_low); + switch (aq_opc) { + case iavf_aqc_opc_send_msg_to_vf: + if (msg_opc == VIRTCHNL_OP_EVENT) { + iavf_handle_pf_event_msg(dev, info.msg_buf, + info.msg_len); + } else { + /* read message and it's expected one */ + if (msg_opc == vf->pend_cmd) + _notify_cmd(vf, msg_ret); + else + PMD_DRV_LOG(ERR, "command mismatch," + "expect %u, get %u", + vf->pend_cmd, msg_opc); + PMD_DRV_LOG(DEBUG, + "adminq response is received," + " opcode = %d", msg_opc); + } + break; + default: + PMD_DRV_LOG(ERR, "Request %u is not supported yet", + aq_opc); + break; + } + } +} + +int +iavf_enable_vlan_strip(struct iavf_adapter *adapter) +{ + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + struct iavf_cmd_info args; + int ret; + + memset(&args, 0, sizeof(args)); + args.ops = VIRTCHNL_OP_ENABLE_VLAN_STRIPPING; + args.in_args = NULL; + args.in_args_size = 0; + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + ret = iavf_execute_vf_cmd(adapter, &args); + if (ret) + PMD_DRV_LOG(ERR, "Failed to execute command of" + " OP_ENABLE_VLAN_STRIPPING"); + + return ret; +} + +int +iavf_disable_vlan_strip(struct iavf_adapter *adapter) +{ + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + struct iavf_cmd_info args; + int ret; + + memset(&args, 0, sizeof(args)); + args.ops = VIRTCHNL_OP_DISABLE_VLAN_STRIPPING; + args.in_args = NULL; + args.in_args_size = 0; + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + ret = iavf_execute_vf_cmd(adapter, &args); + if (ret) + PMD_DRV_LOG(ERR, "Failed to execute command of" + " OP_DISABLE_VLAN_STRIPPING"); + + return ret; +} + +#define VIRTCHNL_VERSION_MAJOR_START 1 +#define VIRTCHNL_VERSION_MINOR_START 1 + +/* Check API version with sync wait until version read from admin queue */ +int +iavf_check_api_version(struct iavf_adapter *adapter) +{ + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + struct virtchnl_version_info version, *pver; + struct iavf_cmd_info args; + int err; + + version.major = VIRTCHNL_VERSION_MAJOR; + version.minor = VIRTCHNL_VERSION_MINOR; + + args.ops = VIRTCHNL_OP_VERSION; + args.in_args = (uint8_t *)&version; + args.in_args_size = sizeof(version); + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + + err = iavf_execute_vf_cmd(adapter, &args); + if (err) { + PMD_INIT_LOG(ERR, "Fail to execute command of OP_VERSION"); + return err; + } + + pver = (struct virtchnl_version_info *)args.out_buffer; + vf->virtchnl_version = *pver; + + if (vf->virtchnl_version.major < VIRTCHNL_VERSION_MAJOR_START || + (vf->virtchnl_version.major == VIRTCHNL_VERSION_MAJOR_START && + vf->virtchnl_version.minor < VIRTCHNL_VERSION_MINOR_START)) { + PMD_INIT_LOG(ERR, "VIRTCHNL API version should not be lower" + " than (%u.%u) to support Adapative VF", + VIRTCHNL_VERSION_MAJOR_START, + VIRTCHNL_VERSION_MAJOR_START); + return -1; + } else if (vf->virtchnl_version.major > VIRTCHNL_VERSION_MAJOR || + (vf->virtchnl_version.major == VIRTCHNL_VERSION_MAJOR && + vf->virtchnl_version.minor > VIRTCHNL_VERSION_MINOR)) { + PMD_INIT_LOG(ERR, "PF/VF API version mismatch:(%u.%u)-(%u.%u)", + vf->virtchnl_version.major, + vf->virtchnl_version.minor, + VIRTCHNL_VERSION_MAJOR, + VIRTCHNL_VERSION_MINOR); + return -1; + } + + PMD_DRV_LOG(DEBUG, "Peer is supported PF host"); + return 0; +} + +int +iavf_get_vf_resource(struct iavf_adapter *adapter) +{ + struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter); + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + struct iavf_cmd_info args; + uint32_t caps, len; + int err, i; + + args.ops = VIRTCHNL_OP_GET_VF_RESOURCES; + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + + caps = IAVF_BASIC_OFFLOAD_CAPS | VIRTCHNL_VF_CAP_ADV_LINK_SPEED | + VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC | + VIRTCHNL_VF_OFFLOAD_FDIR_PF | + VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF; + + args.in_args = (uint8_t *)∩︀ + args.in_args_size = sizeof(caps); + + err = iavf_execute_vf_cmd(adapter, &args); + + if (err) { + PMD_DRV_LOG(ERR, + "Failed to execute command of OP_GET_VF_RESOURCE"); + return -1; + } + + len = sizeof(struct virtchnl_vf_resource) + + IAVF_MAX_VF_VSI * sizeof(struct virtchnl_vsi_resource); + + rte_memcpy(vf->vf_res, args.out_buffer, + RTE_MIN(args.out_size, len)); + /* parse VF config message back from PF*/ + iavf_vf_parse_hw_config(hw, vf->vf_res); + for (i = 0; i < vf->vf_res->num_vsis; i++) { + if (vf->vf_res->vsi_res[i].vsi_type == VIRTCHNL_VSI_SRIOV) + vf->vsi_res = &vf->vf_res->vsi_res[i]; + } + + if (!vf->vsi_res) { + PMD_INIT_LOG(ERR, "no LAN VSI found"); + return -1; + } + + vf->vsi.vsi_id = vf->vsi_res->vsi_id; + vf->vsi.nb_qps = vf->vsi_res->num_queue_pairs; + vf->vsi.adapter = adapter; + + return 0; +} + +int +iavf_get_supported_rxdid(struct iavf_adapter *adapter) +{ + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + struct iavf_cmd_info args; + int ret; + + args.ops = VIRTCHNL_OP_GET_SUPPORTED_RXDIDS; + args.in_args = NULL; + args.in_args_size = 0; + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + + ret = iavf_execute_vf_cmd(adapter, &args); + if (ret) { + PMD_DRV_LOG(ERR, + "Failed to execute command of OP_GET_SUPPORTED_RXDIDS"); + return ret; + } + + vf->supported_rxdid = + ((struct virtchnl_supported_rxdids *)args.out_buffer)->supported_rxdids; + + return 0; +} + +int +iavf_enable_queues(struct iavf_adapter *adapter) +{ + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + struct virtchnl_queue_select queue_select; + struct iavf_cmd_info args; + int err; + + memset(&queue_select, 0, sizeof(queue_select)); + queue_select.vsi_id = vf->vsi_res->vsi_id; + + queue_select.rx_queues = BIT(adapter->eth_dev->data->nb_rx_queues) - 1; + queue_select.tx_queues = BIT(adapter->eth_dev->data->nb_tx_queues) - 1; + + args.ops = VIRTCHNL_OP_ENABLE_QUEUES; + args.in_args = (u8 *)&queue_select; + args.in_args_size = sizeof(queue_select); + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + err = iavf_execute_vf_cmd(adapter, &args); + if (err) { + PMD_DRV_LOG(ERR, + "Failed to execute command of OP_ENABLE_QUEUES"); + return err; + } + return 0; +} + +int +iavf_disable_queues(struct iavf_adapter *adapter) +{ + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + struct virtchnl_queue_select queue_select; + struct iavf_cmd_info args; + int err; + + memset(&queue_select, 0, sizeof(queue_select)); + queue_select.vsi_id = vf->vsi_res->vsi_id; + + queue_select.rx_queues = BIT(adapter->eth_dev->data->nb_rx_queues) - 1; + queue_select.tx_queues = BIT(adapter->eth_dev->data->nb_tx_queues) - 1; + + args.ops = VIRTCHNL_OP_DISABLE_QUEUES; + args.in_args = (u8 *)&queue_select; + args.in_args_size = sizeof(queue_select); + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + err = iavf_execute_vf_cmd(adapter, &args); + if (err) { + PMD_DRV_LOG(ERR, + "Failed to execute command of OP_DISABLE_QUEUES"); + return err; + } + return 0; +} + +int +iavf_switch_queue(struct iavf_adapter *adapter, uint16_t qid, + bool rx, bool on) +{ + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + struct virtchnl_queue_select queue_select; + struct iavf_cmd_info args; + int err; + + memset(&queue_select, 0, sizeof(queue_select)); + queue_select.vsi_id = vf->vsi_res->vsi_id; + if (rx) + queue_select.rx_queues |= 1 << qid; + else + queue_select.tx_queues |= 1 << qid; + + if (on) + args.ops = VIRTCHNL_OP_ENABLE_QUEUES; + else + args.ops = VIRTCHNL_OP_DISABLE_QUEUES; + args.in_args = (u8 *)&queue_select; + args.in_args_size = sizeof(queue_select); + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + err = iavf_execute_vf_cmd(adapter, &args); + if (err) + PMD_DRV_LOG(ERR, "Failed to execute command of %s", + on ? "OP_ENABLE_QUEUES" : "OP_DISABLE_QUEUES"); + return err; +} + +int +iavf_configure_rss_lut(struct iavf_adapter *adapter) +{ + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + struct virtchnl_rss_lut *rss_lut; + struct iavf_cmd_info args; + int len, err = 0; + + len = sizeof(*rss_lut) + vf->vf_res->rss_lut_size - 1; + rss_lut = rte_zmalloc("rss_lut", len, 0); + if (!rss_lut) + return -ENOMEM; + + rss_lut->vsi_id = vf->vsi_res->vsi_id; + rss_lut->lut_entries = vf->vf_res->rss_lut_size; + rte_memcpy(rss_lut->lut, vf->rss_lut, vf->vf_res->rss_lut_size); + + args.ops = VIRTCHNL_OP_CONFIG_RSS_LUT; + args.in_args = (u8 *)rss_lut; + args.in_args_size = len; + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + + err = iavf_execute_vf_cmd(adapter, &args); + if (err) + PMD_DRV_LOG(ERR, + "Failed to execute command of OP_CONFIG_RSS_LUT"); + + rte_free(rss_lut); + return err; +} + +int +iavf_configure_rss_key(struct iavf_adapter *adapter) +{ + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + struct virtchnl_rss_key *rss_key; + struct iavf_cmd_info args; + int len, err = 0; + + len = sizeof(*rss_key) + vf->vf_res->rss_key_size - 1; + rss_key = rte_zmalloc("rss_key", len, 0); + if (!rss_key) + return -ENOMEM; + + rss_key->vsi_id = vf->vsi_res->vsi_id; + rss_key->key_len = vf->vf_res->rss_key_size; + rte_memcpy(rss_key->key, vf->rss_key, vf->vf_res->rss_key_size); + + args.ops = VIRTCHNL_OP_CONFIG_RSS_KEY; + args.in_args = (u8 *)rss_key; + args.in_args_size = len; + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + + err = iavf_execute_vf_cmd(adapter, &args); + if (err) + PMD_DRV_LOG(ERR, + "Failed to execute command of OP_CONFIG_RSS_KEY"); + + rte_free(rss_key); + return err; +} + +int +iavf_configure_queues(struct iavf_adapter *adapter) +{ + struct iavf_rx_queue **rxq = + (struct iavf_rx_queue **)adapter->eth_dev->data->rx_queues; + struct iavf_tx_queue **txq = + (struct iavf_tx_queue **)adapter->eth_dev->data->tx_queues; + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + struct virtchnl_vsi_queue_config_info *vc_config; + struct virtchnl_queue_pair_info *vc_qp; + struct iavf_cmd_info args; + uint16_t i, size; + int err; + + size = sizeof(*vc_config) + + sizeof(vc_config->qpair[0]) * vf->num_queue_pairs; + vc_config = rte_zmalloc("cfg_queue", size, 0); + if (!vc_config) + return -ENOMEM; + + vc_config->vsi_id = vf->vsi_res->vsi_id; + vc_config->num_queue_pairs = vf->num_queue_pairs; + + for (i = 0, vc_qp = vc_config->qpair; + i < vf->num_queue_pairs; + i++, vc_qp++) { + vc_qp->txq.vsi_id = vf->vsi_res->vsi_id; + vc_qp->txq.queue_id = i; + /* Virtchnnl configure queues by pairs */ + if (i < adapter->eth_dev->data->nb_tx_queues) { + vc_qp->txq.ring_len = txq[i]->nb_tx_desc; + vc_qp->txq.dma_ring_addr = txq[i]->tx_ring_phys_addr; + } + vc_qp->rxq.vsi_id = vf->vsi_res->vsi_id; + vc_qp->rxq.queue_id = i; + vc_qp->rxq.max_pkt_size = vf->max_pkt_len; + /* Virtchnnl configure queues by pairs */ + if (i < adapter->eth_dev->data->nb_rx_queues) { + vc_qp->rxq.ring_len = rxq[i]->nb_rx_desc; + vc_qp->rxq.dma_ring_addr = rxq[i]->rx_ring_phys_addr; + vc_qp->rxq.databuffer_size = rxq[i]->rx_buf_len; + } + +#ifndef RTE_LIBRTE_IAVF_16BYTE_RX_DESC + if (vf->vf_res->vf_cap_flags & + VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC && + vf->supported_rxdid & BIT(IAVF_RXDID_COMMS_OVS_1)) { + vc_qp->rxq.rxdid = IAVF_RXDID_COMMS_OVS_1; + PMD_DRV_LOG(NOTICE, "request RXDID == %d in " + "Queue[%d]", vc_qp->rxq.rxdid, i); + } else { + vc_qp->rxq.rxdid = IAVF_RXDID_LEGACY_1; + PMD_DRV_LOG(NOTICE, "request RXDID == %d in " + "Queue[%d]", vc_qp->rxq.rxdid, i); + } +#else + if (vf->vf_res->vf_cap_flags & + VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC && + vf->supported_rxdid & BIT(IAVF_RXDID_LEGACY_0)) { + vc_qp->rxq.rxdid = IAVF_RXDID_LEGACY_0; + PMD_DRV_LOG(NOTICE, "request RXDID == %d in " + "Queue[%d]", vc_qp->rxq.rxdid, i); + } else { + PMD_DRV_LOG(ERR, "RXDID == 0 is not supported"); + return -1; + } +#endif + } + + memset(&args, 0, sizeof(args)); + args.ops = VIRTCHNL_OP_CONFIG_VSI_QUEUES; + args.in_args = (uint8_t *)vc_config; + args.in_args_size = size; + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + + err = iavf_execute_vf_cmd(adapter, &args); + if (err) + PMD_DRV_LOG(ERR, "Failed to execute command of" + " VIRTCHNL_OP_CONFIG_VSI_QUEUES"); + + rte_free(vc_config); + return err; +} + +int +iavf_config_irq_map(struct iavf_adapter *adapter) +{ + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + struct virtchnl_irq_map_info *map_info; + struct virtchnl_vector_map *vecmap; + struct iavf_cmd_info args; + int len, i, err; + + len = sizeof(struct virtchnl_irq_map_info) + + sizeof(struct virtchnl_vector_map) * vf->nb_msix; + + map_info = rte_zmalloc("map_info", len, 0); + if (!map_info) + return -ENOMEM; + + map_info->num_vectors = vf->nb_msix; + for (i = 0; i < vf->nb_msix; i++) { + vecmap = &map_info->vecmap[i]; + vecmap->vsi_id = vf->vsi_res->vsi_id; + vecmap->rxitr_idx = IAVF_ITR_INDEX_DEFAULT; + vecmap->vector_id = vf->msix_base + i; + vecmap->txq_map = 0; + vecmap->rxq_map = vf->rxq_map[vf->msix_base + i]; + } + + args.ops = VIRTCHNL_OP_CONFIG_IRQ_MAP; + args.in_args = (u8 *)map_info; + args.in_args_size = len; + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + err = iavf_execute_vf_cmd(adapter, &args); + if (err) + PMD_DRV_LOG(ERR, "fail to execute command OP_CONFIG_IRQ_MAP"); + + rte_free(map_info); + return err; +} + +void +iavf_add_del_all_mac_addr(struct iavf_adapter *adapter, bool add) +{ + struct virtchnl_ether_addr_list *list; + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + struct rte_ether_addr *addr; + struct iavf_cmd_info args; + int len, err, i, j; + int next_begin = 0; + int begin = 0; + + do { + j = 0; + len = sizeof(struct virtchnl_ether_addr_list); + for (i = begin; i < IAVF_NUM_MACADDR_MAX; i++, next_begin++) { + addr = &adapter->eth_dev->data->mac_addrs[i]; + if (rte_is_zero_ether_addr(addr)) + continue; + len += sizeof(struct virtchnl_ether_addr); + if (len >= IAVF_AQ_BUF_SZ) { + next_begin = i + 1; + break; + } + } + + list = rte_zmalloc("iavf_del_mac_buffer", len, 0); + if (!list) { + PMD_DRV_LOG(ERR, "fail to allocate memory"); + return; + } + + for (i = begin; i < next_begin; i++) { + addr = &adapter->eth_dev->data->mac_addrs[i]; + if (rte_is_zero_ether_addr(addr)) + continue; + rte_memcpy(list->list[j].addr, addr->addr_bytes, + sizeof(addr->addr_bytes)); + PMD_DRV_LOG(DEBUG, "add/rm mac:%x:%x:%x:%x:%x:%x", + addr->addr_bytes[0], addr->addr_bytes[1], + addr->addr_bytes[2], addr->addr_bytes[3], + addr->addr_bytes[4], addr->addr_bytes[5]); + j++; + } + list->vsi_id = vf->vsi_res->vsi_id; + list->num_elements = j; + args.ops = add ? VIRTCHNL_OP_ADD_ETH_ADDR : + VIRTCHNL_OP_DEL_ETH_ADDR; + args.in_args = (uint8_t *)list; + args.in_args_size = len; + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + err = iavf_execute_vf_cmd(adapter, &args); + if (err) + PMD_DRV_LOG(ERR, "fail to execute command %s", + add ? "OP_ADD_ETHER_ADDRESS" : + "OP_DEL_ETHER_ADDRESS"); + rte_free(list); + begin = next_begin; + } while (begin < IAVF_NUM_MACADDR_MAX); +} + +int +iavf_query_stats(struct iavf_adapter *adapter, + struct virtchnl_eth_stats **pstats) +{ + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + struct virtchnl_queue_select q_stats; + struct iavf_cmd_info args; + int err; + + memset(&q_stats, 0, sizeof(q_stats)); + q_stats.vsi_id = vf->vsi_res->vsi_id; + args.ops = VIRTCHNL_OP_GET_STATS; + args.in_args = (uint8_t *)&q_stats; + args.in_args_size = sizeof(q_stats); + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + + err = iavf_execute_vf_cmd(adapter, &args); + if (err) { + PMD_DRV_LOG(ERR, "fail to execute command OP_GET_STATS"); + *pstats = NULL; + return err; + } + *pstats = (struct virtchnl_eth_stats *)args.out_buffer; + return 0; +} + +int +iavf_config_promisc(struct iavf_adapter *adapter, + bool enable_unicast, + bool enable_multicast) +{ + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + struct virtchnl_promisc_info promisc; + struct iavf_cmd_info args; + int err; + + promisc.flags = 0; + promisc.vsi_id = vf->vsi_res->vsi_id; + + if (enable_unicast) + promisc.flags |= FLAG_VF_UNICAST_PROMISC; + + if (enable_multicast) + promisc.flags |= FLAG_VF_MULTICAST_PROMISC; + + args.ops = VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE; + args.in_args = (uint8_t *)&promisc; + args.in_args_size = sizeof(promisc); + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + + err = iavf_execute_vf_cmd(adapter, &args); + + if (err) + PMD_DRV_LOG(ERR, + "fail to execute command CONFIG_PROMISCUOUS_MODE"); + return err; +} + +int +iavf_add_del_eth_addr(struct iavf_adapter *adapter, struct rte_ether_addr *addr, + bool add) +{ + struct virtchnl_ether_addr_list *list; + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + uint8_t cmd_buffer[sizeof(struct virtchnl_ether_addr_list) + + sizeof(struct virtchnl_ether_addr)]; + struct iavf_cmd_info args; + int err; + + list = (struct virtchnl_ether_addr_list *)cmd_buffer; + list->vsi_id = vf->vsi_res->vsi_id; + list->num_elements = 1; + rte_memcpy(list->list[0].addr, addr->addr_bytes, + sizeof(addr->addr_bytes)); + + args.ops = add ? VIRTCHNL_OP_ADD_ETH_ADDR : VIRTCHNL_OP_DEL_ETH_ADDR; + args.in_args = cmd_buffer; + args.in_args_size = sizeof(cmd_buffer); + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + err = iavf_execute_vf_cmd(adapter, &args); + if (err) + PMD_DRV_LOG(ERR, "fail to execute command %s", + add ? "OP_ADD_ETH_ADDR" : "OP_DEL_ETH_ADDR"); + return err; +} + +int +iavf_add_del_vlan(struct iavf_adapter *adapter, uint16_t vlanid, bool add) +{ + struct virtchnl_vlan_filter_list *vlan_list; + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + uint8_t cmd_buffer[sizeof(struct virtchnl_vlan_filter_list) + + sizeof(uint16_t)]; + struct iavf_cmd_info args; + int err; + + vlan_list = (struct virtchnl_vlan_filter_list *)cmd_buffer; + vlan_list->vsi_id = vf->vsi_res->vsi_id; + vlan_list->num_elements = 1; + vlan_list->vlan_id[0] = vlanid; + + args.ops = add ? VIRTCHNL_OP_ADD_VLAN : VIRTCHNL_OP_DEL_VLAN; + args.in_args = cmd_buffer; + args.in_args_size = sizeof(cmd_buffer); + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + err = iavf_execute_vf_cmd(adapter, &args); + if (err) + PMD_DRV_LOG(ERR, "fail to execute command %s", + add ? "OP_ADD_VLAN" : "OP_DEL_VLAN"); + + return err; +} + +int +iavf_fdir_add(struct iavf_adapter *adapter, + struct iavf_fdir_conf *filter) +{ + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + struct virtchnl_fdir_add *fdir_ret; + + struct iavf_cmd_info args; + int err; + + filter->add_fltr.vsi_id = vf->vsi_res->vsi_id; + filter->add_fltr.validate_only = 0; + + args.ops = VIRTCHNL_OP_ADD_FDIR_FILTER; + args.in_args = (uint8_t *)(&filter->add_fltr); + args.in_args_size = sizeof(*(&filter->add_fltr)); + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + + err = iavf_execute_vf_cmd(adapter, &args); + if (err) { + PMD_DRV_LOG(ERR, "fail to execute command OP_ADD_FDIR_FILTER"); + return err; + } + + fdir_ret = (struct virtchnl_fdir_add *)args.out_buffer; + filter->flow_id = fdir_ret->flow_id; + + if (fdir_ret->status == VIRTCHNL_FDIR_SUCCESS) { + PMD_DRV_LOG(INFO, + "Succeed in adding rule request by PF"); + } else if (fdir_ret->status == VIRTCHNL_FDIR_FAILURE_RULE_NORESOURCE) { + PMD_DRV_LOG(ERR, + "Failed to add rule request due to no hw resource"); + return -1; + } else if (fdir_ret->status == VIRTCHNL_FDIR_FAILURE_RULE_EXIST) { + PMD_DRV_LOG(ERR, + "Failed to add rule request due to the rule is already existed"); + return -1; + } else if (fdir_ret->status == VIRTCHNL_FDIR_FAILURE_RULE_CONFLICT) { + PMD_DRV_LOG(ERR, + "Failed to add rule request due to the rule is conflict with existing rule"); + return -1; + } else if (fdir_ret->status == VIRTCHNL_FDIR_FAILURE_RULE_INVALID) { + PMD_DRV_LOG(ERR, + "Failed to add rule request due to the hw doesn't support"); + return -1; + } else if (fdir_ret->status == VIRTCHNL_FDIR_FAILURE_RULE_TIMEOUT) { + PMD_DRV_LOG(ERR, + "Failed to add rule request due to time out for programming"); + return -1; + } else { + PMD_DRV_LOG(ERR, + "Failed to add rule request due to other reasons"); + return -1; + } + + return 0; +}; + +int +iavf_fdir_del(struct iavf_adapter *adapter, + struct iavf_fdir_conf *filter) +{ + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + struct virtchnl_fdir_del *fdir_ret; + + struct iavf_cmd_info args; + int err; + + filter->del_fltr.vsi_id = vf->vsi_res->vsi_id; + filter->del_fltr.flow_id = filter->flow_id; + + args.ops = VIRTCHNL_OP_DEL_FDIR_FILTER; + args.in_args = (uint8_t *)(&filter->del_fltr); + args.in_args_size = sizeof(filter->del_fltr); + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + + err = iavf_execute_vf_cmd(adapter, &args); + if (err) { + PMD_DRV_LOG(ERR, "fail to execute command OP_DEL_FDIR_FILTER"); + return err; + } + + fdir_ret = (struct virtchnl_fdir_del *)args.out_buffer; + + if (fdir_ret->status == VIRTCHNL_FDIR_SUCCESS) { + PMD_DRV_LOG(INFO, + "Succeed in deleting rule request by PF"); + } else if (fdir_ret->status == VIRTCHNL_FDIR_FAILURE_RULE_NONEXIST) { + PMD_DRV_LOG(ERR, + "Failed to delete rule request due to this rule doesn't exist"); + return -1; + } else if (fdir_ret->status == VIRTCHNL_FDIR_FAILURE_RULE_TIMEOUT) { + PMD_DRV_LOG(ERR, + "Failed to delete rule request due to time out for programming"); + return -1; + } else { + PMD_DRV_LOG(ERR, + "Failed to delete rule request due to other reasons"); + return -1; + } + + return 0; +}; + +int +iavf_fdir_check(struct iavf_adapter *adapter, + struct iavf_fdir_conf *filter) +{ + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + struct virtchnl_fdir_add *fdir_ret; + + struct iavf_cmd_info args; + int err; + + filter->add_fltr.vsi_id = vf->vsi_res->vsi_id; + filter->add_fltr.validate_only = 1; + + args.ops = VIRTCHNL_OP_ADD_FDIR_FILTER; + args.in_args = (uint8_t *)(&filter->add_fltr); + args.in_args_size = sizeof(*(&filter->add_fltr)); + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + + err = iavf_execute_vf_cmd(adapter, &args); + if (err) { + PMD_DRV_LOG(ERR, "fail to check flow direcotor rule"); + return err; + } + + fdir_ret = (struct virtchnl_fdir_add *)args.out_buffer; + + if (fdir_ret->status == VIRTCHNL_FDIR_SUCCESS) { + PMD_DRV_LOG(INFO, + "Succeed in checking rule request by PF"); + } else if (fdir_ret->status == VIRTCHNL_FDIR_FAILURE_RULE_INVALID) { + PMD_DRV_LOG(ERR, + "Failed to check rule request due to parameters validation" + " or HW doesn't support"); + return -1; + } else { + PMD_DRV_LOG(ERR, + "Failed to check rule request due to other reasons"); + return -1; + } + + return 0; +} + +int +iavf_add_del_rss_cfg(struct iavf_adapter *adapter, + struct virtchnl_rss_cfg *rss_cfg, bool add) +{ + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); + struct iavf_cmd_info args; + int err; + + memset(&args, 0, sizeof(args)); + args.ops = add ? VIRTCHNL_OP_ADD_RSS_CFG : + VIRTCHNL_OP_DEL_RSS_CFG; + args.in_args = (u8 *)rss_cfg; + args.in_args_size = sizeof(*rss_cfg); + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + + err = iavf_execute_vf_cmd(adapter, &args); + if (err) + PMD_DRV_LOG(ERR, + "Failed to execute command of %s", + add ? "OP_ADD_RSS_CFG" : + "OP_DEL_RSS_INPUT_CFG"); + + return err; +} diff --git a/src/spdk/dpdk/drivers/net/iavf/meson.build b/src/spdk/dpdk/drivers/net/iavf/meson.build new file mode 100644 index 000000000..a3fad363d --- /dev/null +++ b/src/spdk/dpdk/drivers/net/iavf/meson.build @@ -0,0 +1,37 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2018 Luca Boccassi + +cflags += ['-Wno-strict-aliasing'] + +includes += include_directories('../../common/iavf') +deps += ['common_iavf'] + +sources = files( + 'iavf_ethdev.c', + 'iavf_rxtx.c', + 'iavf_vchnl.c', + 'iavf_generic_flow.c', + 'iavf_fdir.c', + 'iavf_hash.c', +) + +if arch_subdir == 'x86' + sources += files('iavf_rxtx_vec_sse.c') + + # compile AVX2 version if either: + # a. we have AVX supported in minimum instruction set baseline + # b. it's not minimum instruction set, but supported by compiler + if dpdk_conf.has('RTE_MACHINE_CPUFLAG_AVX2') + cflags += ['-DCC_AVX2_SUPPORT'] + sources += files('iavf_rxtx_vec_avx2.c') + elif cc.has_argument('-mavx2') + cflags += ['-DCC_AVX2_SUPPORT'] + iavf_avx2_lib = static_library('iavf_avx2_lib', + 'iavf_rxtx_vec_avx2.c', + dependencies: [static_rte_ethdev, + static_rte_kvargs, static_rte_hash], + include_directories: includes, + c_args: [cflags, '-mavx2']) + objs += iavf_avx2_lib.extract_objects('iavf_rxtx_vec_avx2.c') + endif +endif diff --git a/src/spdk/dpdk/drivers/net/iavf/rte_pmd_iavf_version.map b/src/spdk/dpdk/drivers/net/iavf/rte_pmd_iavf_version.map new file mode 100644 index 000000000..f9f17e4f6 --- /dev/null +++ b/src/spdk/dpdk/drivers/net/iavf/rte_pmd_iavf_version.map @@ -0,0 +1,3 @@ +DPDK_20.0 { + local: *; +}; -- cgit v1.2.3