Adding upstream version 14.2.21.upstream/14.2.21upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

30 files changed, 23123 insertions, 0 deletions

diff --git a/src/spdk/dpdk/drivers/net/mlx5/Makefile b/src/spdk/dpdk/drivers/net/mlx5/Makefile
new file mode 100644
index 00000000..2e70dec5
--- /dev/null
+++ b/src/spdk/dpdk/drivers/net/mlx5/Makefile
@@ -0,0 +1,422 @@
+#   SPDX-License-Identifier: BSD-3-Clause
+#   Copyright 2015 6WIND S.A.
+#   Copyright 2015 Mellanox Technologies, Ltd
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+# Library name.
+LIB = librte_pmd_mlx5.a
+LIB_GLUE = $(LIB_GLUE_BASE).$(LIB_GLUE_VERSION)
+LIB_GLUE_BASE = librte_pmd_mlx5_glue.so
+LIB_GLUE_VERSION = 18.05.0
+
+# Sources.
+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5.c
+ifneq ($(CONFIG_RTE_LIBRTE_MLX5_DLOPEN_DEPS),y)
+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_glue.c
+endif
+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rxq.c
+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_txq.c
+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rxtx.c
+ifneq ($(filter y,$(CONFIG_RTE_ARCH_X86_64) \
+		  $(CONFIG_RTE_ARCH_ARM64)),)
+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rxtx_vec.c
+endif
+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_trigger.c
+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_ethdev.c
+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_mac.c
+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rxmode.c
+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_vlan.c
+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_stats.c
+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rss.c
+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_mr.c
+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow.c
+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_socket.c
+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_nl.c
+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_nl_flow.c
+
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLOPEN_DEPS),y)
+INSTALL-$(CONFIG_RTE_LIBRTE_MLX5_PMD)-lib += $(LIB_GLUE)
+endif
+
+# Basic CFLAGS.
+CFLAGS += -O3
+CFLAGS += -std=c11 -Wall -Wextra
+CFLAGS += -g
+CFLAGS += -I.
+CFLAGS += -D_BSD_SOURCE
+CFLAGS += -D_DEFAULT_SOURCE
+CFLAGS += -D_XOPEN_SOURCE=600
+CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -Wno-strict-prototypes
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLOPEN_DEPS),y)
+CFLAGS += -DMLX5_GLUE='"$(LIB_GLUE)"'
+CFLAGS += -DMLX5_GLUE_VERSION='"$(LIB_GLUE_VERSION)"'
+CFLAGS_mlx5_glue.o += -fPIC
+LDLIBS += -ldl
+else
+LDLIBS += -libverbs -lmlx5
+endif
+LDLIBS += -lmnl
+LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
+LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
+LDLIBS += -lrte_bus_pci
+
+# A few warnings cannot be avoided in external headers.
+CFLAGS += -Wno-error=cast-qual
+
+EXPORT_MAP := rte_pmd_mlx5_version.map
+LIBABIVER := 1
+
+# memseg walk is not part of stable API
+CFLAGS += -DALLOW_EXPERIMENTAL_API
+
+# DEBUG which is usually provided on the command-line may enable
+# CONFIG_RTE_LIBRTE_MLX5_DEBUG.
+ifeq ($(DEBUG),1)
+CONFIG_RTE_LIBRTE_MLX5_DEBUG := y
+endif
+
+# User-defined CFLAGS.
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DEBUG),y)
+CFLAGS += -pedantic -UNDEBUG -DPEDANTIC
+else
+CFLAGS += -DNDEBUG -UPEDANTIC
+endif
+
+include $(RTE_SDK)/mk/rte.lib.mk
+
+# Generate and clean-up mlx5_autoconf.h.
+
+export CC CFLAGS CPPFLAGS EXTRA_CFLAGS EXTRA_CPPFLAGS
+export AUTO_CONFIG_CFLAGS = -Wno-error
+
+ifndef V
+AUTOCONF_OUTPUT := >/dev/null
+endif
+
+mlx5_autoconf.h.new: FORCE
+
+mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
+	$Q $(RM) -f -- '$@'
+	$Q sh -- '$<' '$@' \
+		HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT \
+		infiniband/mlx5dv.h \
+		enum MLX5DV_CQE_RES_FORMAT_CSUM_STRIDX \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_IBV_DEVICE_TUNNEL_SUPPORT \
+		infiniband/mlx5dv.h \
+		enum MLX5DV_CONTEXT_MASK_TUNNEL_OFFLOADS \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_IBV_DEVICE_MPLS_SUPPORT \
+		infiniband/verbs.h \
+		enum IBV_FLOW_SPEC_MPLS \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_IBV_WQ_FLAG_RX_END_PADDING \
+		infiniband/verbs.h \
+		enum IBV_WQ_FLAG_RX_END_PADDING \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_IBV_MLX5_MOD_SWP \
+		infiniband/mlx5dv.h \
+		type 'struct mlx5dv_sw_parsing_caps' \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_IBV_MLX5_MOD_MPW \
+		infiniband/mlx5dv.h \
+		enum MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_IBV_MLX5_MOD_CQE_128B_COMP \
+		infiniband/mlx5dv.h \
+		enum MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_ETHTOOL_LINK_MODE_25G \
+		/usr/include/linux/ethtool.h \
+		enum ETHTOOL_LINK_MODE_25000baseCR_Full_BIT \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_ETHTOOL_LINK_MODE_50G \
+		/usr/include/linux/ethtool.h \
+		enum ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_ETHTOOL_LINK_MODE_100G \
+		/usr/include/linux/ethtool.h \
+		enum ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \
+		infiniband/verbs.h \
+		type 'struct ibv_counter_set_init_attr' \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_RDMA_NL_NLDEV \
+		rdma/rdma_netlink.h \
+		enum RDMA_NL_NLDEV \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_RDMA_NLDEV_CMD_GET \
+		rdma/rdma_netlink.h \
+		enum RDMA_NLDEV_CMD_GET \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_RDMA_NLDEV_CMD_PORT_GET \
+		rdma/rdma_netlink.h \
+		enum RDMA_NLDEV_CMD_PORT_GET \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_RDMA_NLDEV_ATTR_DEV_INDEX \
+		rdma/rdma_netlink.h \
+		enum RDMA_NLDEV_ATTR_DEV_INDEX \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_RDMA_NLDEV_ATTR_DEV_NAME \
+		rdma/rdma_netlink.h \
+		enum RDMA_NLDEV_ATTR_DEV_NAME \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_RDMA_NLDEV_ATTR_PORT_INDEX \
+		rdma/rdma_netlink.h \
+		enum RDMA_NLDEV_ATTR_PORT_INDEX \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_RDMA_NLDEV_ATTR_NDEV_INDEX \
+		rdma/rdma_netlink.h \
+		enum RDMA_NLDEV_ATTR_NDEV_INDEX \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_IFLA_PHYS_SWITCH_ID \
+		linux/if_link.h \
+		enum IFLA_PHYS_SWITCH_ID \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_IFLA_PHYS_PORT_NAME \
+		linux/if_link.h \
+		enum IFLA_PHYS_PORT_NAME \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_ACT \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_ACT \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_FLAGS \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_FLAGS \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_ETH_TYPE \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_ETH_TYPE \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_ETH_DST \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_ETH_DST \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_ETH_DST_MASK \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_ETH_DST_MASK \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_ETH_SRC \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_ETH_SRC \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_ETH_SRC_MASK \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_ETH_SRC_MASK \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_IP_PROTO \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_IP_PROTO \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_IPV4_SRC \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_IPV4_SRC \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_IPV4_SRC_MASK \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_IPV4_SRC_MASK \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_IPV4_DST \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_IPV4_DST \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_IPV4_DST_MASK \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_IPV4_DST_MASK \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_IPV6_SRC \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_IPV6_SRC \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_IPV6_SRC_MASK \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_IPV6_SRC_MASK \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_IPV6_DST \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_IPV6_DST \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_IPV6_DST_MASK \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_IPV6_DST_MASK \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_TCP_SRC \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_TCP_SRC \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_TCP_SRC_MASK \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_TCP_SRC_MASK \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_TCP_DST \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_TCP_DST \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_TCP_DST_MASK \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_TCP_DST_MASK \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_UDP_SRC \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_UDP_SRC \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_UDP_SRC_MASK \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_UDP_SRC_MASK \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_UDP_DST \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_UDP_DST \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_UDP_DST_MASK \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_UDP_DST_MASK \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_VLAN_ID \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_VLAN_ID \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_VLAN_PRIO \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_VLAN_PRIO \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_VLAN_ETH_TYPE \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_VLAN_ETH_TYPE \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TC_ACT_VLAN \
+		linux/tc_act/tc_vlan.h \
+		enum TCA_VLAN_PUSH_VLAN_PRIORITY \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_SUPPORTED_40000baseKR4_Full \
+		/usr/include/linux/ethtool.h \
+		define SUPPORTED_40000baseKR4_Full \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_SUPPORTED_40000baseCR4_Full \
+		/usr/include/linux/ethtool.h \
+		define SUPPORTED_40000baseCR4_Full \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_SUPPORTED_40000baseSR4_Full \
+		/usr/include/linux/ethtool.h \
+		define SUPPORTED_40000baseSR4_Full \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_SUPPORTED_40000baseLR4_Full \
+		/usr/include/linux/ethtool.h \
+		define SUPPORTED_40000baseLR4_Full \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_SUPPORTED_56000baseKR4_Full \
+		/usr/include/linux/ethtool.h \
+		define SUPPORTED_56000baseKR4_Full \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_SUPPORTED_56000baseCR4_Full \
+		/usr/include/linux/ethtool.h \
+		define SUPPORTED_56000baseCR4_Full \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_SUPPORTED_56000baseSR4_Full \
+		/usr/include/linux/ethtool.h \
+		define SUPPORTED_56000baseSR4_Full \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_SUPPORTED_56000baseLR4_Full \
+		/usr/include/linux/ethtool.h \
+		define SUPPORTED_56000baseLR4_Full \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_STATIC_ASSERT \
+		/usr/include/assert.h \
+		define static_assert \
+		$(AUTOCONF_OUTPUT)
+
+# Create mlx5_autoconf.h or update it in case it differs from the new one.
+
+mlx5_autoconf.h: mlx5_autoconf.h.new
+	$Q [ -f '$@' ] && \
+		cmp '$<' '$@' $(AUTOCONF_OUTPUT) || \
+		mv '$<' '$@'
+
+$(SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD):.c=.o): mlx5_autoconf.h
+
+# Generate dependency plug-in for rdma-core when the PMD must not be linked
+# directly, so that applications do not inherit this dependency.
+
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLOPEN_DEPS),y)
+
+$(LIB): $(LIB_GLUE)
+
+ifeq ($(LINK_USING_CC),1)
+GLUE_LDFLAGS := $(call linkerprefix,$(LDFLAGS))
+else
+GLUE_LDFLAGS := $(LDFLAGS)
+endif
+$(LIB_GLUE): mlx5_glue.o
+	$Q $(LD) $(GLUE_LDFLAGS) $(EXTRA_LDFLAGS) \
+		-Wl,-h,$(LIB_GLUE) \
+		-shared -o $@ $< -libverbs -lmlx5
+
+mlx5_glue.o: mlx5_autoconf.h
+
+endif
+
+clean_mlx5: FORCE
+	$Q rm -f -- mlx5_autoconf.h mlx5_autoconf.h.new
+	$Q rm -f -- mlx5_glue.o $(LIB_GLUE_BASE)*
+
+clean: clean_mlx5
diff --git a/src/spdk/dpdk/drivers/net/mlx5/mlx5.c b/src/spdk/dpdk/drivers/net/mlx5/mlx5.c
new file mode 100644
index 00000000..ec63bc6e
--- /dev/null
+++ b/src/spdk/dpdk/drivers/net/mlx5/mlx5.c
@@ -0,0 +1,1690 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2015 6WIND S.A.
+ * Copyright 2015 Mellanox Technologies, Ltd
+ */
+
+#include <stddef.h>
+#include <unistd.h>
+#include <string.h>
+#include <assert.h>
+#include <dlfcn.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <net/if.h>
+#include <sys/mman.h>
+#include <linux/netlink.h>
+#include <linux/rtnetlink.h>
+
+/* Verbs header. */
+/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <infiniband/verbs.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+#include <rte_malloc.h>
+#include <rte_ethdev_driver.h>
+#include <rte_ethdev_pci.h>
+#include <rte_pci.h>
+#include <rte_bus_pci.h>
+#include <rte_common.h>
+#include <rte_config.h>
+#include <rte_eal_memconfig.h>
+#include <rte_kvargs.h>
+#include <rte_rwlock.h>
+#include <rte_spinlock.h>
+#include <rte_string_fns.h>
+
+#include "mlx5.h"
+#include "mlx5_utils.h"
+#include "mlx5_rxtx.h"
+#include "mlx5_autoconf.h"
+#include "mlx5_defs.h"
+#include "mlx5_glue.h"
+#include "mlx5_mr.h"
+
+/* Device parameter to enable RX completion queue compression. */
+#define MLX5_RXQ_CQE_COMP_EN "rxq_cqe_comp_en"
+
+/* Device parameter to enable Multi-Packet Rx queue. */
+#define MLX5_RX_MPRQ_EN "mprq_en"
+
+/* Device parameter to configure log 2 of the number of strides for MPRQ. */
+#define MLX5_RX_MPRQ_LOG_STRIDE_NUM "mprq_log_stride_num"
+
+/* Device parameter to limit the size of memcpy'd packet for MPRQ. */
+#define MLX5_RX_MPRQ_MAX_MEMCPY_LEN "mprq_max_memcpy_len"
+
+/* Device parameter to set the minimum number of Rx queues to enable MPRQ. */
+#define MLX5_RXQS_MIN_MPRQ "rxqs_min_mprq"
+
+/* Device parameter to configure inline send. */
+#define MLX5_TXQ_INLINE "txq_inline"
+
+/*
+ * Device parameter to configure the number of TX queues threshold for
+ * enabling inline send.
+ */
+#define MLX5_TXQS_MIN_INLINE "txqs_min_inline"
+
+/* Device parameter to enable multi-packet send WQEs. */
+#define MLX5_TXQ_MPW_EN "txq_mpw_en"
+
+/* Device parameter to include 2 dsegs in the title WQEBB. */
+#define MLX5_TXQ_MPW_HDR_DSEG_EN "txq_mpw_hdr_dseg_en"
+
+/* Device parameter to limit the size of inlining packet. */
+#define MLX5_TXQ_MAX_INLINE_LEN "txq_max_inline_len"
+
+/* Device parameter to enable hardware Tx vector. */
+#define MLX5_TX_VEC_EN "tx_vec_en"
+
+/* Device parameter to enable hardware Rx vector. */
+#define MLX5_RX_VEC_EN "rx_vec_en"
+
+/* Allow L3 VXLAN flow creation. */
+#define MLX5_L3_VXLAN_EN "l3_vxlan_en"
+
+/* Activate Netlink support in VF mode. */
+#define MLX5_VF_NL_EN "vf_nl_en"
+
+/* Select port representors to instantiate. */
+#define MLX5_REPRESENTOR "representor"
+
+#ifndef HAVE_IBV_MLX5_MOD_MPW
+#define MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED (1 << 2)
+#define MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW (1 << 3)
+#endif
+
+#ifndef HAVE_IBV_MLX5_MOD_CQE_128B_COMP
+#define MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP (1 << 4)
+#endif
+
+static const char *MZ_MLX5_PMD_SHARED_DATA = "mlx5_pmd_shared_data";
+
+/* Shared memory between primary and secondary processes. */
+struct mlx5_shared_data *mlx5_shared_data;
+
+/* Spinlock for mlx5_shared_data allocation. */
+static rte_spinlock_t mlx5_shared_data_lock = RTE_SPINLOCK_INITIALIZER;
+
+/** Driver-specific log messages type. */
+int mlx5_logtype;
+
+/**
+ * Prepare shared data between primary and secondary process.
+ */
+static void
+mlx5_prepare_shared_data(void)
+{
+	const struct rte_memzone *mz;
+
+	rte_spinlock_lock(&mlx5_shared_data_lock);
+	if (mlx5_shared_data == NULL) {
+		if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+			/* Allocate shared memory. */
+			mz = rte_memzone_reserve(MZ_MLX5_PMD_SHARED_DATA,
+						 sizeof(*mlx5_shared_data),
+						 SOCKET_ID_ANY, 0);
+		} else {
+			/* Lookup allocated shared memory. */
+			mz = rte_memzone_lookup(MZ_MLX5_PMD_SHARED_DATA);
+		}
+		if (mz == NULL)
+			rte_panic("Cannot allocate mlx5 shared data\n");
+		mlx5_shared_data = mz->addr;
+		/* Initialize shared data. */
+		if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+			LIST_INIT(&mlx5_shared_data->mem_event_cb_list);
+			rte_rwlock_init(&mlx5_shared_data->mem_event_rwlock);
+		}
+		rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",
+						mlx5_mr_mem_event_cb, NULL);
+	}
+	rte_spinlock_unlock(&mlx5_shared_data_lock);
+}
+
+/**
+ * Retrieve integer value from environment variable.
+ *
+ * @param[in] name
+ *   Environment variable name.
+ *
+ * @return
+ *   Integer value, 0 if the variable is not set.
+ */
+int
+mlx5_getenv_int(const char *name)
+{
+	const char *val = getenv(name);
+
+	if (val == NULL)
+		return 0;
+	return atoi(val);
+}
+
+/**
+ * Verbs callback to allocate a memory. This function should allocate the space
+ * according to the size provided residing inside a huge page.
+ * Please note that all allocation must respect the alignment from libmlx5
+ * (i.e. currently sysconf(_SC_PAGESIZE)).
+ *
+ * @param[in] size
+ *   The size in bytes of the memory to allocate.
+ * @param[in] data
+ *   A pointer to the callback data.
+ *
+ * @return
+ *   Allocated buffer, NULL otherwise and rte_errno is set.
+ */
+static void *
+mlx5_alloc_verbs_buf(size_t size, void *data)
+{
+	struct priv *priv = data;
+	void *ret;
+	size_t alignment = sysconf(_SC_PAGESIZE);
+	unsigned int socket = SOCKET_ID_ANY;
+
+	if (priv->verbs_alloc_ctx.type == MLX5_VERBS_ALLOC_TYPE_TX_QUEUE) {
+		const struct mlx5_txq_ctrl *ctrl = priv->verbs_alloc_ctx.obj;
+
+		socket = ctrl->socket;
+	} else if (priv->verbs_alloc_ctx.type ==
+		   MLX5_VERBS_ALLOC_TYPE_RX_QUEUE) {
+		const struct mlx5_rxq_ctrl *ctrl = priv->verbs_alloc_ctx.obj;
+
+		socket = ctrl->socket;
+	}
+	assert(data != NULL);
+	ret = rte_malloc_socket(__func__, size, alignment, socket);
+	if (!ret && size)
+		rte_errno = ENOMEM;
+	return ret;
+}
+
+/**
+ * Verbs callback to free a memory.
+ *
+ * @param[in] ptr
+ *   A pointer to the memory to free.
+ * @param[in] data
+ *   A pointer to the callback data.
+ */
+static void
+mlx5_free_verbs_buf(void *ptr, void *data __rte_unused)
+{
+	assert(data != NULL);
+	rte_free(ptr);
+}
+
+/**
+ * DPDK callback to close the device.
+ *
+ * Destroy all queues and objects, free memory.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ */
+static void
+mlx5_dev_close(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	unsigned int i;
+	int ret;
+
+	DRV_LOG(DEBUG, "port %u closing device \"%s\"",
+		dev->data->port_id,
+		((priv->ctx != NULL) ? priv->ctx->device->name : ""));
+	/* In case mlx5_dev_stop() has not been called. */
+	mlx5_dev_interrupt_handler_uninstall(dev);
+	mlx5_traffic_disable(dev);
+	mlx5_flow_flush(dev, NULL);
+	/* Prevent crashes when queues are still in use. */
+	dev->rx_pkt_burst = removed_rx_burst;
+	dev->tx_pkt_burst = removed_tx_burst;
+	if (priv->rxqs != NULL) {
+		/* XXX race condition if mlx5_rx_burst() is still running. */
+		usleep(1000);
+		for (i = 0; (i != priv->rxqs_n); ++i)
+			mlx5_rxq_release(dev, i);
+		priv->rxqs_n = 0;
+		priv->rxqs = NULL;
+	}
+	if (priv->txqs != NULL) {
+		/* XXX race condition if mlx5_tx_burst() is still running. */
+		usleep(1000);
+		for (i = 0; (i != priv->txqs_n); ++i)
+			mlx5_txq_release(dev, i);
+		priv->txqs_n = 0;
+		priv->txqs = NULL;
+	}
+	mlx5_mprq_free_mp(dev);
+	mlx5_mr_release(dev);
+	if (priv->pd != NULL) {
+		assert(priv->ctx != NULL);
+		claim_zero(mlx5_glue->dealloc_pd(priv->pd));
+		claim_zero(mlx5_glue->close_device(priv->ctx));
+	} else
+		assert(priv->ctx == NULL);
+	if (priv->rss_conf.rss_key != NULL)
+		rte_free(priv->rss_conf.rss_key);
+	if (priv->reta_idx != NULL)
+		rte_free(priv->reta_idx);
+	if (priv->primary_socket)
+		mlx5_socket_uninit(dev);
+	if (priv->config.vf)
+		mlx5_nl_mac_addr_flush(dev);
+	if (priv->nl_socket_route >= 0)
+		close(priv->nl_socket_route);
+	if (priv->nl_socket_rdma >= 0)
+		close(priv->nl_socket_rdma);
+	if (priv->mnl_socket)
+		mlx5_nl_flow_socket_destroy(priv->mnl_socket);
+	ret = mlx5_hrxq_ibv_verify(dev);
+	if (ret)
+		DRV_LOG(WARNING, "port %u some hash Rx queue still remain",
+			dev->data->port_id);
+	ret = mlx5_ind_table_ibv_verify(dev);
+	if (ret)
+		DRV_LOG(WARNING, "port %u some indirection table still remain",
+			dev->data->port_id);
+	ret = mlx5_rxq_ibv_verify(dev);
+	if (ret)
+		DRV_LOG(WARNING, "port %u some Verbs Rx queue still remain",
+			dev->data->port_id);
+	ret = mlx5_rxq_verify(dev);
+	if (ret)
+		DRV_LOG(WARNING, "port %u some Rx queues still remain",
+			dev->data->port_id);
+	ret = mlx5_txq_ibv_verify(dev);
+	if (ret)
+		DRV_LOG(WARNING, "port %u some Verbs Tx queue still remain",
+			dev->data->port_id);
+	ret = mlx5_txq_verify(dev);
+	if (ret)
+		DRV_LOG(WARNING, "port %u some Tx queues still remain",
+			dev->data->port_id);
+	ret = mlx5_flow_verify(dev);
+	if (ret)
+		DRV_LOG(WARNING, "port %u some flows still remain",
+			dev->data->port_id);
+	if (priv->domain_id != RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID) {
+		unsigned int c = 0;
+		unsigned int i = mlx5_dev_to_port_id(dev->device, NULL, 0);
+		uint16_t port_id[i];
+
+		i = RTE_MIN(mlx5_dev_to_port_id(dev->device, port_id, i), i);
+		while (i--) {
+			struct priv *opriv =
+				rte_eth_devices[port_id[i]].data->dev_private;
+
+			if (!opriv ||
+			    opriv->domain_id != priv->domain_id ||
+			    &rte_eth_devices[port_id[i]] == dev)
+				continue;
+			++c;
+		}
+		if (!c)
+			claim_zero(rte_eth_switch_domain_free(priv->domain_id));
+	}
+	memset(priv, 0, sizeof(*priv));
+	priv->domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID;
+}
+
+const struct eth_dev_ops mlx5_dev_ops = {
+	.dev_configure = mlx5_dev_configure,
+	.dev_start = mlx5_dev_start,
+	.dev_stop = mlx5_dev_stop,
+	.dev_set_link_down = mlx5_set_link_down,
+	.dev_set_link_up = mlx5_set_link_up,
+	.dev_close = mlx5_dev_close,
+	.promiscuous_enable = mlx5_promiscuous_enable,
+	.promiscuous_disable = mlx5_promiscuous_disable,
+	.allmulticast_enable = mlx5_allmulticast_enable,
+	.allmulticast_disable = mlx5_allmulticast_disable,
+	.link_update = mlx5_link_update,
+	.stats_get = mlx5_stats_get,
+	.stats_reset = mlx5_stats_reset,
+	.xstats_get = mlx5_xstats_get,
+	.xstats_reset = mlx5_xstats_reset,
+	.xstats_get_names = mlx5_xstats_get_names,
+	.dev_infos_get = mlx5_dev_infos_get,
+	.dev_supported_ptypes_get = mlx5_dev_supported_ptypes_get,
+	.vlan_filter_set = mlx5_vlan_filter_set,
+	.rx_queue_setup = mlx5_rx_queue_setup,
+	.tx_queue_setup = mlx5_tx_queue_setup,
+	.rx_queue_release = mlx5_rx_queue_release,
+	.tx_queue_release = mlx5_tx_queue_release,
+	.flow_ctrl_get = mlx5_dev_get_flow_ctrl,
+	.flow_ctrl_set = mlx5_dev_set_flow_ctrl,
+	.mac_addr_remove = mlx5_mac_addr_remove,
+	.mac_addr_add = mlx5_mac_addr_add,
+	.mac_addr_set = mlx5_mac_addr_set,
+	.set_mc_addr_list = mlx5_set_mc_addr_list,
+	.mtu_set = mlx5_dev_set_mtu,
+	.vlan_strip_queue_set = mlx5_vlan_strip_queue_set,
+	.vlan_offload_set = mlx5_vlan_offload_set,
+	.reta_update = mlx5_dev_rss_reta_update,
+	.reta_query = mlx5_dev_rss_reta_query,
+	.rss_hash_update = mlx5_rss_hash_update,
+	.rss_hash_conf_get = mlx5_rss_hash_conf_get,
+	.filter_ctrl = mlx5_dev_filter_ctrl,
+	.rx_descriptor_status = mlx5_rx_descriptor_status,
+	.tx_descriptor_status = mlx5_tx_descriptor_status,
+	.rx_queue_intr_enable = mlx5_rx_intr_enable,
+	.rx_queue_intr_disable = mlx5_rx_intr_disable,
+	.is_removed = mlx5_is_removed,
+};
+
+static const struct eth_dev_ops mlx5_dev_sec_ops = {
+	.stats_get = mlx5_stats_get,
+	.stats_reset = mlx5_stats_reset,
+	.xstats_get = mlx5_xstats_get,
+	.xstats_reset = mlx5_xstats_reset,
+	.xstats_get_names = mlx5_xstats_get_names,
+	.dev_infos_get = mlx5_dev_infos_get,
+	.rx_descriptor_status = mlx5_rx_descriptor_status,
+	.tx_descriptor_status = mlx5_tx_descriptor_status,
+};
+
+/* Available operators in flow isolated mode. */
+const struct eth_dev_ops mlx5_dev_ops_isolate = {
+	.dev_configure = mlx5_dev_configure,
+	.dev_start = mlx5_dev_start,
+	.dev_stop = mlx5_dev_stop,
+	.dev_set_link_down = mlx5_set_link_down,
+	.dev_set_link_up = mlx5_set_link_up,
+	.dev_close = mlx5_dev_close,
+	.promiscuous_enable = mlx5_promiscuous_enable,
+	.promiscuous_disable = mlx5_promiscuous_disable,
+	.allmulticast_enable = mlx5_allmulticast_enable,
+	.allmulticast_disable = mlx5_allmulticast_disable,
+	.link_update = mlx5_link_update,
+	.stats_get = mlx5_stats_get,
+	.stats_reset = mlx5_stats_reset,
+	.xstats_get = mlx5_xstats_get,
+	.xstats_reset = mlx5_xstats_reset,
+	.xstats_get_names = mlx5_xstats_get_names,
+	.dev_infos_get = mlx5_dev_infos_get,
+	.dev_supported_ptypes_get = mlx5_dev_supported_ptypes_get,
+	.vlan_filter_set = mlx5_vlan_filter_set,
+	.rx_queue_setup = mlx5_rx_queue_setup,
+	.tx_queue_setup = mlx5_tx_queue_setup,
+	.rx_queue_release = mlx5_rx_queue_release,
+	.tx_queue_release = mlx5_tx_queue_release,
+	.flow_ctrl_get = mlx5_dev_get_flow_ctrl,
+	.flow_ctrl_set = mlx5_dev_set_flow_ctrl,
+	.mac_addr_remove = mlx5_mac_addr_remove,
+	.mac_addr_add = mlx5_mac_addr_add,
+	.mac_addr_set = mlx5_mac_addr_set,
+	.set_mc_addr_list = mlx5_set_mc_addr_list,
+	.mtu_set = mlx5_dev_set_mtu,
+	.vlan_strip_queue_set = mlx5_vlan_strip_queue_set,
+	.vlan_offload_set = mlx5_vlan_offload_set,
+	.filter_ctrl = mlx5_dev_filter_ctrl,
+	.rx_descriptor_status = mlx5_rx_descriptor_status,
+	.tx_descriptor_status = mlx5_tx_descriptor_status,
+	.rx_queue_intr_enable = mlx5_rx_intr_enable,
+	.rx_queue_intr_disable = mlx5_rx_intr_disable,
+	.is_removed = mlx5_is_removed,
+};
+
+/**
+ * Verify and store value for device argument.
+ *
+ * @param[in] key
+ *   Key argument to verify.
+ * @param[in] val
+ *   Value associated with key.
+ * @param opaque
+ *   User data.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_args_check(const char *key, const char *val, void *opaque)
+{
+	struct mlx5_dev_config *config = opaque;
+	unsigned long tmp;
+
+	/* No-op, port representors are processed in mlx5_dev_spawn(). */
+	if (!strcmp(MLX5_REPRESENTOR, key))
+		return 0;
+	errno = 0;
+	tmp = strtoul(val, NULL, 0);
+	if (errno) {
+		rte_errno = errno;
+		DRV_LOG(WARNING, "%s: \"%s\" is not a valid integer", key, val);
+		return -rte_errno;
+	}
+	if (strcmp(MLX5_RXQ_CQE_COMP_EN, key) == 0) {
+		config->cqe_comp = !!tmp;
+	} else if (strcmp(MLX5_RX_MPRQ_EN, key) == 0) {
+		config->mprq.enabled = !!tmp;
+	} else if (strcmp(MLX5_RX_MPRQ_LOG_STRIDE_NUM, key) == 0) {
+		config->mprq.stride_num_n = tmp;
+	} else if (strcmp(MLX5_RX_MPRQ_MAX_MEMCPY_LEN, key) == 0) {
+		config->mprq.max_memcpy_len = tmp;
+	} else if (strcmp(MLX5_RXQS_MIN_MPRQ, key) == 0) {
+		config->mprq.min_rxqs_num = tmp;
+	} else if (strcmp(MLX5_TXQ_INLINE, key) == 0) {
+		config->txq_inline = tmp;
+	} else if (strcmp(MLX5_TXQS_MIN_INLINE, key) == 0) {
+		config->txqs_inline = tmp;
+	} else if (strcmp(MLX5_TXQ_MPW_EN, key) == 0) {
+		config->mps = !!tmp ? config->mps : 0;
+	} else if (strcmp(MLX5_TXQ_MPW_HDR_DSEG_EN, key) == 0) {
+		config->mpw_hdr_dseg = !!tmp;
+	} else if (strcmp(MLX5_TXQ_MAX_INLINE_LEN, key) == 0) {
+		config->inline_max_packet_sz = tmp;
+	} else if (strcmp(MLX5_TX_VEC_EN, key) == 0) {
+		config->tx_vec_en = !!tmp;
+	} else if (strcmp(MLX5_RX_VEC_EN, key) == 0) {
+		config->rx_vec_en = !!tmp;
+	} else if (strcmp(MLX5_L3_VXLAN_EN, key) == 0) {
+		config->l3_vxlan_en = !!tmp;
+	} else if (strcmp(MLX5_VF_NL_EN, key) == 0) {
+		config->vf_nl_en = !!tmp;
+	} else {
+		DRV_LOG(WARNING, "%s: unknown parameter", key);
+		rte_errno = EINVAL;
+		return -rte_errno;
+	}
+	return 0;
+}
+
+/**
+ * Parse device parameters.
+ *
+ * @param config
+ *   Pointer to device configuration structure.
+ * @param devargs
+ *   Device arguments structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_args(struct mlx5_dev_config *config, struct rte_devargs *devargs)
+{
+	const char **params = (const char *[]){
+		MLX5_RXQ_CQE_COMP_EN,
+		MLX5_RX_MPRQ_EN,
+		MLX5_RX_MPRQ_LOG_STRIDE_NUM,
+		MLX5_RX_MPRQ_MAX_MEMCPY_LEN,
+		MLX5_RXQS_MIN_MPRQ,
+		MLX5_TXQ_INLINE,
+		MLX5_TXQS_MIN_INLINE,
+		MLX5_TXQ_MPW_EN,
+		MLX5_TXQ_MPW_HDR_DSEG_EN,
+		MLX5_TXQ_MAX_INLINE_LEN,
+		MLX5_TX_VEC_EN,
+		MLX5_RX_VEC_EN,
+		MLX5_L3_VXLAN_EN,
+		MLX5_VF_NL_EN,
+		MLX5_REPRESENTOR,
+		NULL,
+	};
+	struct rte_kvargs *kvlist;
+	int ret = 0;
+	int i;
+
+	if (devargs == NULL)
+		return 0;
+	/* Following UGLY cast is done to pass checkpatch. */
+	kvlist = rte_kvargs_parse(devargs->args, params);
+	if (kvlist == NULL)
+		return 0;
+	/* Process parameters. */
+	for (i = 0; (params[i] != NULL); ++i) {
+		if (rte_kvargs_count(kvlist, params[i])) {
+			ret = rte_kvargs_process(kvlist, params[i],
+						 mlx5_args_check, config);
+			if (ret) {
+				rte_errno = EINVAL;
+				rte_kvargs_free(kvlist);
+				return -rte_errno;
+			}
+		}
+	}
+	rte_kvargs_free(kvlist);
+	return 0;
+}
+
+static struct rte_pci_driver mlx5_driver;
+
+/*
+ * Reserved UAR address space for TXQ UAR(hw doorbell) mapping, process
+ * local resource used by both primary and secondary to avoid duplicate
+ * reservation.
+ * The space has to be available on both primary and secondary process,
+ * TXQ UAR maps to this area using fixed mmap w/o double check.
+ */
+static void *uar_base;
+
+static int
+find_lower_va_bound(const struct rte_memseg_list *msl __rte_unused,
+		const struct rte_memseg *ms, void *arg)
+{
+	void **addr = arg;
+
+	if (*addr == NULL)
+		*addr = ms->addr;
+	else
+		*addr = RTE_MIN(*addr, ms->addr);
+
+	return 0;
+}
+
+/**
+ * Reserve UAR address space for primary process.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_uar_init_primary(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	void *addr = (void *)0;
+
+	if (uar_base) { /* UAR address space mapped. */
+		priv->uar_base = uar_base;
+		return 0;
+	}
+	/* find out lower bound of hugepage segments */
+	rte_memseg_walk(find_lower_va_bound, &addr);
+
+	/* keep distance to hugepages to minimize potential conflicts. */
+	addr = RTE_PTR_SUB(addr, (uintptr_t)(MLX5_UAR_OFFSET + MLX5_UAR_SIZE));
+	/* anonymous mmap, no real memory consumption. */
+	addr = mmap(addr, MLX5_UAR_SIZE,
+		    PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+	if (addr == MAP_FAILED) {
+		DRV_LOG(ERR,
+			"port %u failed to reserve UAR address space, please"
+			" adjust MLX5_UAR_SIZE or try --base-virtaddr",
+			dev->data->port_id);
+		rte_errno = ENOMEM;
+		return -rte_errno;
+	}
+	/* Accept either same addr or a new addr returned from mmap if target
+	 * range occupied.
+	 */
+	DRV_LOG(INFO, "port %u reserved UAR address space: %p",
+		dev->data->port_id, addr);
+	priv->uar_base = addr; /* for primary and secondary UAR re-mmap. */
+	uar_base = addr; /* process local, don't reserve again. */
+	return 0;
+}
+
+/**
+ * Reserve UAR address space for secondary process, align with
+ * primary process.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_uar_init_secondary(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	void *addr;
+
+	assert(priv->uar_base);
+	if (uar_base) { /* already reserved. */
+		assert(uar_base == priv->uar_base);
+		return 0;
+	}
+	/* anonymous mmap, no real memory consumption. */
+	addr = mmap(priv->uar_base, MLX5_UAR_SIZE,
+		    PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+	if (addr == MAP_FAILED) {
+		DRV_LOG(ERR, "port %u UAR mmap failed: %p size: %llu",
+			dev->data->port_id, priv->uar_base, MLX5_UAR_SIZE);
+		rte_errno = ENXIO;
+		return -rte_errno;
+	}
+	if (priv->uar_base != addr) {
+		DRV_LOG(ERR,
+			"port %u UAR address %p size %llu occupied, please"
+			" adjust MLX5_UAR_OFFSET or try EAL parameter"
+			" --base-virtaddr",
+			dev->data->port_id, priv->uar_base, MLX5_UAR_SIZE);
+		rte_errno = ENXIO;
+		return -rte_errno;
+	}
+	uar_base = addr; /* process local, don't reserve again */
+	DRV_LOG(INFO, "port %u reserved UAR address space: %p",
+		dev->data->port_id, addr);
+	return 0;
+}
+
+/**
+ * Spawn an Ethernet device from Verbs information.
+ *
+ * @param dpdk_dev
+ *   Backing DPDK device.
+ * @param ibv_dev
+ *   Verbs device.
+ * @param vf
+ *   If nonzero, enable VF-specific features.
+ * @param[in] switch_info
+ *   Switch properties of Ethernet device.
+ *
+ * @return
+ *   A valid Ethernet device object on success, NULL otherwise and rte_errno
+ *   is set. The following error is defined:
+ *
+ *   EBUSY: device is not supposed to be spawned.
+ */
+static struct rte_eth_dev *
+mlx5_dev_spawn(struct rte_device *dpdk_dev,
+	       struct ibv_device *ibv_dev,
+	       int vf,
+	       const struct mlx5_switch_info *switch_info)
+{
+	struct ibv_context *ctx;
+	struct ibv_device_attr_ex attr;
+	struct ibv_port_attr port_attr;
+	struct ibv_pd *pd = NULL;
+	struct mlx5dv_context dv_attr = { .comp_mask = 0 };
+	struct mlx5_dev_config config = {
+		.vf = !!vf,
+		.tx_vec_en = 1,
+		.rx_vec_en = 1,
+		.mpw_hdr_dseg = 0,
+		.txq_inline = MLX5_ARG_UNSET,
+		.txqs_inline = MLX5_ARG_UNSET,
+		.inline_max_packet_sz = MLX5_ARG_UNSET,
+		.vf_nl_en = 1,
+		.mprq = {
+			.enabled = 0,
+			.stride_num_n = MLX5_MPRQ_STRIDE_NUM_N,
+			.max_memcpy_len = MLX5_MPRQ_MEMCPY_DEFAULT_LEN,
+			.min_rxqs_num = MLX5_MPRQ_MIN_RXQS,
+		},
+	};
+	struct rte_eth_dev *eth_dev = NULL;
+	struct priv *priv = NULL;
+	int err = 0;
+	unsigned int mps;
+	unsigned int cqe_comp;
+	unsigned int tunnel_en = 0;
+	unsigned int mpls_en = 0;
+	unsigned int swp = 0;
+	unsigned int mprq = 0;
+	unsigned int mprq_min_stride_size_n = 0;
+	unsigned int mprq_max_stride_size_n = 0;
+	unsigned int mprq_min_stride_num_n = 0;
+	unsigned int mprq_max_stride_num_n = 0;
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+	struct ibv_counter_set_description cs_desc = { .counter_type = 0 };
+#endif
+	struct ether_addr mac;
+	char name[RTE_ETH_NAME_MAX_LEN];
+	int own_domain_id = 0;
+	unsigned int i;
+
+	/* Determine if this port representor is supposed to be spawned. */
+	if (switch_info->representor && dpdk_dev->devargs) {
+		struct rte_eth_devargs eth_da;
+
+		err = rte_eth_devargs_parse(dpdk_dev->devargs->args, &eth_da);
+		if (err) {
+			rte_errno = -err;
+			DRV_LOG(ERR, "failed to process device arguments: %s",
+				strerror(rte_errno));
+			return NULL;
+		}
+		for (i = 0; i < eth_da.nb_representor_ports; ++i)
+			if (eth_da.representor_ports[i] ==
+			    (uint16_t)switch_info->port_name)
+				break;
+		if (i == eth_da.nb_representor_ports) {
+			rte_errno = EBUSY;
+			return NULL;
+		}
+	}
+	/* Prepare shared data between primary and secondary process. */
+	mlx5_prepare_shared_data();
+	errno = 0;
+	ctx = mlx5_glue->open_device(ibv_dev);
+	if (!ctx) {
+		rte_errno = errno ? errno : ENODEV;
+		return NULL;
+	}
+#ifdef HAVE_IBV_MLX5_MOD_SWP
+	dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_SWP;
+#endif
+	/*
+	 * Multi-packet send is supported by ConnectX-4 Lx PF as well
+	 * as all ConnectX-5 devices.
+	 */
+#ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
+	dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_TUNNEL_OFFLOADS;
+#endif
+#ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
+	dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_STRIDING_RQ;
+#endif
+	mlx5_glue->dv_query_device(ctx, &dv_attr);
+	if (dv_attr.flags & MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED) {
+		if (dv_attr.flags & MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW) {
+			DRV_LOG(DEBUG, "enhanced MPW is supported");
+			mps = MLX5_MPW_ENHANCED;
+		} else {
+			DRV_LOG(DEBUG, "MPW is supported");
+			mps = MLX5_MPW;
+		}
+	} else {
+		DRV_LOG(DEBUG, "MPW isn't supported");
+		mps = MLX5_MPW_DISABLED;
+	}
+	config.mps = mps;
+#ifdef HAVE_IBV_MLX5_MOD_SWP
+	if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_SWP)
+		swp = dv_attr.sw_parsing_caps.sw_parsing_offloads;
+	DRV_LOG(DEBUG, "SWP support: %u", swp);
+#endif
+	config.swp = !!swp;
+#ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
+	if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_STRIDING_RQ) {
+		struct mlx5dv_striding_rq_caps mprq_caps =
+			dv_attr.striding_rq_caps;
+
+		DRV_LOG(DEBUG, "\tmin_single_stride_log_num_of_bytes: %d",
+			mprq_caps.min_single_stride_log_num_of_bytes);
+		DRV_LOG(DEBUG, "\tmax_single_stride_log_num_of_bytes: %d",
+			mprq_caps.max_single_stride_log_num_of_bytes);
+		DRV_LOG(DEBUG, "\tmin_single_wqe_log_num_of_strides: %d",
+			mprq_caps.min_single_wqe_log_num_of_strides);
+		DRV_LOG(DEBUG, "\tmax_single_wqe_log_num_of_strides: %d",
+			mprq_caps.max_single_wqe_log_num_of_strides);
+		DRV_LOG(DEBUG, "\tsupported_qpts: %d",
+			mprq_caps.supported_qpts);
+		DRV_LOG(DEBUG, "device supports Multi-Packet RQ");
+		mprq = 1;
+		mprq_min_stride_size_n =
+			mprq_caps.min_single_stride_log_num_of_bytes;
+		mprq_max_stride_size_n =
+			mprq_caps.max_single_stride_log_num_of_bytes;
+		mprq_min_stride_num_n =
+			mprq_caps.min_single_wqe_log_num_of_strides;
+		mprq_max_stride_num_n =
+			mprq_caps.max_single_wqe_log_num_of_strides;
+		config.mprq.stride_num_n = RTE_MAX(MLX5_MPRQ_STRIDE_NUM_N,
+						   mprq_min_stride_num_n);
+	}
+#endif
+	if (RTE_CACHE_LINE_SIZE == 128 &&
+	    !(dv_attr.flags & MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP))
+		cqe_comp = 0;
+	else
+		cqe_comp = 1;
+	config.cqe_comp = cqe_comp;
+#ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
+	if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_TUNNEL_OFFLOADS) {
+		tunnel_en = ((dv_attr.tunnel_offloads_caps &
+			      MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_VXLAN) &&
+			     (dv_attr.tunnel_offloads_caps &
+			      MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_GRE));
+	}
+	DRV_LOG(DEBUG, "tunnel offloading is %ssupported",
+		tunnel_en ? "" : "not ");
+#else
+	DRV_LOG(WARNING,
+		"tunnel offloading disabled due to old OFED/rdma-core version");
+#endif
+	config.tunnel_en = tunnel_en;
+#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
+	mpls_en = ((dv_attr.tunnel_offloads_caps &
+		    MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_CW_MPLS_OVER_GRE) &&
+		   (dv_attr.tunnel_offloads_caps &
+		    MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_CW_MPLS_OVER_UDP));
+	DRV_LOG(DEBUG, "MPLS over GRE/UDP tunnel offloading is %ssupported",
+		mpls_en ? "" : "not ");
+#else
+	DRV_LOG(WARNING, "MPLS over GRE/UDP tunnel offloading disabled due to"
+		" old OFED/rdma-core version or firmware configuration");
+#endif
+	config.mpls_en = mpls_en;
+	err = mlx5_glue->query_device_ex(ctx, NULL, &attr);
+	if (err) {
+		DEBUG("ibv_query_device_ex() failed");
+		goto error;
+	}
+	if (!switch_info->representor)
+		rte_strlcpy(name, dpdk_dev->name, sizeof(name));
+	else
+		snprintf(name, sizeof(name), "%s_representor_%u",
+			 dpdk_dev->name, switch_info->port_name);
+	DRV_LOG(DEBUG, "naming Ethernet device \"%s\"", name);
+	if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
+		eth_dev = rte_eth_dev_attach_secondary(name);
+		if (eth_dev == NULL) {
+			DRV_LOG(ERR, "can not attach rte ethdev");
+			rte_errno = ENOMEM;
+			err = rte_errno;
+			goto error;
+		}
+		eth_dev->device = dpdk_dev;
+		eth_dev->dev_ops = &mlx5_dev_sec_ops;
+		err = mlx5_uar_init_secondary(eth_dev);
+		if (err) {
+			err = rte_errno;
+			goto error;
+		}
+		/* Receive command fd from primary process */
+		err = mlx5_socket_connect(eth_dev);
+		if (err < 0) {
+			err = rte_errno;
+			goto error;
+		}
+		/* Remap UAR for Tx queues. */
+		err = mlx5_tx_uar_remap(eth_dev, err);
+		if (err) {
+			err = rte_errno;
+			goto error;
+		}
+		/*
+		 * Ethdev pointer is still required as input since
+		 * the primary device is not accessible from the
+		 * secondary process.
+		 */
+		eth_dev->rx_pkt_burst = mlx5_select_rx_function(eth_dev);
+		eth_dev->tx_pkt_burst = mlx5_select_tx_function(eth_dev);
+		claim_zero(mlx5_glue->close_device(ctx));
+		return eth_dev;
+	}
+	/* Check port status. */
+	err = mlx5_glue->query_port(ctx, 1, &port_attr);
+	if (err) {
+		DRV_LOG(ERR, "port query failed: %s", strerror(err));
+		goto error;
+	}
+	if (port_attr.link_layer != IBV_LINK_LAYER_ETHERNET) {
+		DRV_LOG(ERR, "port is not configured in Ethernet mode");
+		err = EINVAL;
+		goto error;
+	}
+	if (port_attr.state != IBV_PORT_ACTIVE)
+		DRV_LOG(DEBUG, "port is not active: \"%s\" (%d)",
+			mlx5_glue->port_state_str(port_attr.state),
+			port_attr.state);
+	/* Allocate protection domain. */
+	pd = mlx5_glue->alloc_pd(ctx);
+	if (pd == NULL) {
+		DRV_LOG(ERR, "PD allocation failure");
+		err = ENOMEM;
+		goto error;
+	}
+	priv = rte_zmalloc("ethdev private structure",
+			   sizeof(*priv),
+			   RTE_CACHE_LINE_SIZE);
+	if (priv == NULL) {
+		DRV_LOG(ERR, "priv allocation failure");
+		err = ENOMEM;
+		goto error;
+	}
+	priv->ctx = ctx;
+	strncpy(priv->ibdev_name, priv->ctx->device->name,
+		sizeof(priv->ibdev_name));
+	strncpy(priv->ibdev_path, priv->ctx->device->ibdev_path,
+		sizeof(priv->ibdev_path));
+	priv->device_attr = attr;
+	priv->pd = pd;
+	priv->mtu = ETHER_MTU;
+#ifndef RTE_ARCH_64
+	/* Initialize UAR access locks for 32bit implementations. */
+	rte_spinlock_init(&priv->uar_lock_cq);
+	for (i = 0; i < MLX5_UAR_PAGE_NUM_MAX; i++)
+		rte_spinlock_init(&priv->uar_lock[i]);
+#endif
+	/* Some internal functions rely on Netlink sockets, open them now. */
+	priv->nl_socket_rdma = mlx5_nl_init(NETLINK_RDMA);
+	priv->nl_socket_route =	mlx5_nl_init(NETLINK_ROUTE);
+	priv->nl_sn = 0;
+	priv->representor = !!switch_info->representor;
+	priv->domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID;
+	priv->representor_id =
+		switch_info->representor ? switch_info->port_name : -1;
+	/*
+	 * Look for sibling devices in order to reuse their switch domain
+	 * if any, otherwise allocate one.
+	 */
+	i = mlx5_dev_to_port_id(dpdk_dev, NULL, 0);
+	if (i > 0) {
+		uint16_t port_id[i];
+
+		i = RTE_MIN(mlx5_dev_to_port_id(dpdk_dev, port_id, i), i);
+		while (i--) {
+			const struct priv *opriv =
+				rte_eth_devices[port_id[i]].data->dev_private;
+
+			if (!opriv ||
+			    opriv->domain_id ==
+			    RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID)
+				continue;
+			priv->domain_id = opriv->domain_id;
+			break;
+		}
+	}
+	if (priv->domain_id == RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID) {
+		err = rte_eth_switch_domain_alloc(&priv->domain_id);
+		if (err) {
+			err = rte_errno;
+			DRV_LOG(ERR, "unable to allocate switch domain: %s",
+				strerror(rte_errno));
+			goto error;
+		}
+		own_domain_id = 1;
+	}
+	err = mlx5_args(&config, dpdk_dev->devargs);
+	if (err) {
+		err = rte_errno;
+		DRV_LOG(ERR, "failed to process device arguments: %s",
+			strerror(rte_errno));
+		goto error;
+	}
+	config.hw_csum = !!(attr.device_cap_flags_ex & IBV_DEVICE_RAW_IP_CSUM);
+	DRV_LOG(DEBUG, "checksum offloading is %ssupported",
+		(config.hw_csum ? "" : "not "));
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+	config.flow_counter_en = !!attr.max_counter_sets;
+	mlx5_glue->describe_counter_set(ctx, 0, &cs_desc);
+	DRV_LOG(DEBUG, "counter type = %d, num of cs = %ld, attributes = %d",
+		cs_desc.counter_type, cs_desc.num_of_cs,
+		cs_desc.attributes);
+#endif
+	config.ind_table_max_size =
+		attr.rss_caps.max_rwq_indirection_table_size;
+	/*
+	 * Remove this check once DPDK supports larger/variable
+	 * indirection tables.
+	 */
+	if (config.ind_table_max_size > (unsigned int)ETH_RSS_RETA_SIZE_512)
+		config.ind_table_max_size = ETH_RSS_RETA_SIZE_512;
+	DRV_LOG(DEBUG, "maximum Rx indirection table size is %u",
+		config.ind_table_max_size);
+	config.hw_vlan_strip = !!(attr.raw_packet_caps &
+				  IBV_RAW_PACKET_CAP_CVLAN_STRIPPING);
+	DRV_LOG(DEBUG, "VLAN stripping is %ssupported",
+		(config.hw_vlan_strip ? "" : "not "));
+	config.hw_fcs_strip = !!(attr.raw_packet_caps &
+				 IBV_RAW_PACKET_CAP_SCATTER_FCS);
+	DRV_LOG(DEBUG, "FCS stripping configuration is %ssupported",
+		(config.hw_fcs_strip ? "" : "not "));
+#ifdef HAVE_IBV_WQ_FLAG_RX_END_PADDING
+	config.hw_padding = !!attr.rx_pad_end_addr_align;
+#endif
+	DRV_LOG(DEBUG, "hardware Rx end alignment padding is %ssupported",
+		(config.hw_padding ? "" : "not "));
+	config.tso = (attr.tso_caps.max_tso > 0 &&
+		      (attr.tso_caps.supported_qpts &
+		       (1 << IBV_QPT_RAW_PACKET)));
+	if (config.tso)
+		config.tso_max_payload_sz = attr.tso_caps.max_tso;
+	if (config.mps && !mps) {
+		DRV_LOG(ERR,
+			"multi-packet send not supported on this device"
+			" (" MLX5_TXQ_MPW_EN ")");
+		err = ENOTSUP;
+		goto error;
+	}
+	DRV_LOG(INFO, "%sMPS is %s",
+		config.mps == MLX5_MPW_ENHANCED ? "enhanced " : "",
+		config.mps != MLX5_MPW_DISABLED ? "enabled" : "disabled");
+	if (config.cqe_comp && !cqe_comp) {
+		DRV_LOG(WARNING, "Rx CQE compression isn't supported");
+		config.cqe_comp = 0;
+	}
+	if (config.mprq.enabled && mprq) {
+		if (config.mprq.stride_num_n > mprq_max_stride_num_n ||
+		    config.mprq.stride_num_n < mprq_min_stride_num_n) {
+			config.mprq.stride_num_n =
+				RTE_MAX(MLX5_MPRQ_STRIDE_NUM_N,
+					mprq_min_stride_num_n);
+			DRV_LOG(WARNING,
+				"the number of strides"
+				" for Multi-Packet RQ is out of range,"
+				" setting default value (%u)",
+				1 << config.mprq.stride_num_n);
+		}
+		config.mprq.min_stride_size_n = mprq_min_stride_size_n;
+		config.mprq.max_stride_size_n = mprq_max_stride_size_n;
+	} else if (config.mprq.enabled && !mprq) {
+		DRV_LOG(WARNING, "Multi-Packet RQ isn't supported");
+		config.mprq.enabled = 0;
+	}
+	eth_dev = rte_eth_dev_allocate(name);
+	if (eth_dev == NULL) {
+		DRV_LOG(ERR, "can not allocate rte ethdev");
+		err = ENOMEM;
+		goto error;
+	}
+	if (priv->representor)
+		eth_dev->data->dev_flags |= RTE_ETH_DEV_REPRESENTOR;
+	eth_dev->data->dev_private = priv;
+	priv->dev_data = eth_dev->data;
+	eth_dev->data->mac_addrs = priv->mac;
+	eth_dev->device = dpdk_dev;
+	eth_dev->device->driver = &mlx5_driver.driver;
+	err = mlx5_uar_init_primary(eth_dev);
+	if (err) {
+		err = rte_errno;
+		goto error;
+	}
+	/* Configure the first MAC address by default. */
+	if (mlx5_get_mac(eth_dev, &mac.addr_bytes)) {
+		DRV_LOG(ERR,
+			"port %u cannot get MAC address, is mlx5_en"
+			" loaded? (errno: %s)",
+			eth_dev->data->port_id, strerror(rte_errno));
+		err = ENODEV;
+		goto error;
+	}
+	DRV_LOG(INFO,
+		"port %u MAC address is %02x:%02x:%02x:%02x:%02x:%02x",
+		eth_dev->data->port_id,
+		mac.addr_bytes[0], mac.addr_bytes[1],
+		mac.addr_bytes[2], mac.addr_bytes[3],
+		mac.addr_bytes[4], mac.addr_bytes[5]);
+#ifndef NDEBUG
+	{
+		char ifname[IF_NAMESIZE];
+
+		if (mlx5_get_ifname(eth_dev, &ifname) == 0)
+			DRV_LOG(DEBUG, "port %u ifname is \"%s\"",
+				eth_dev->data->port_id, ifname);
+		else
+			DRV_LOG(DEBUG, "port %u ifname is unknown",
+				eth_dev->data->port_id);
+	}
+#endif
+	/* Get actual MTU if possible. */
+	err = mlx5_get_mtu(eth_dev, &priv->mtu);
+	if (err) {
+		err = rte_errno;
+		goto error;
+	}
+	DRV_LOG(DEBUG, "port %u MTU is %u", eth_dev->data->port_id,
+		priv->mtu);
+	/* Initialize burst functions to prevent crashes before link-up. */
+	eth_dev->rx_pkt_burst = removed_rx_burst;
+	eth_dev->tx_pkt_burst = removed_tx_burst;
+	eth_dev->dev_ops = &mlx5_dev_ops;
+	/* Register MAC address. */
+	claim_zero(mlx5_mac_addr_add(eth_dev, &mac, 0, 0));
+	if (vf && config.vf_nl_en)
+		mlx5_nl_mac_addr_sync(eth_dev);
+	priv->mnl_socket = mlx5_nl_flow_socket_create();
+	if (!priv->mnl_socket) {
+		err = -rte_errno;
+		DRV_LOG(WARNING,
+			"flow rules relying on switch offloads will not be"
+			" supported: cannot open libmnl socket: %s",
+			strerror(rte_errno));
+	} else {
+		struct rte_flow_error error;
+		unsigned int ifindex = mlx5_ifindex(eth_dev);
+
+		if (!ifindex) {
+			err = -rte_errno;
+			error.message =
+				"cannot retrieve network interface index";
+		} else {
+			err = mlx5_nl_flow_init(priv->mnl_socket, ifindex,
+						&error);
+		}
+		if (err) {
+			DRV_LOG(WARNING,
+				"flow rules relying on switch offloads will"
+				" not be supported: %s: %s",
+				error.message, strerror(rte_errno));
+			mlx5_nl_flow_socket_destroy(priv->mnl_socket);
+			priv->mnl_socket = NULL;
+		}
+	}
+	TAILQ_INIT(&priv->flows);
+	TAILQ_INIT(&priv->ctrl_flows);
+	/* Hint libmlx5 to use PMD allocator for data plane resources */
+	struct mlx5dv_ctx_allocators alctr = {
+		.alloc = &mlx5_alloc_verbs_buf,
+		.free = &mlx5_free_verbs_buf,
+		.data = priv,
+	};
+	mlx5_glue->dv_set_context_attr(ctx, MLX5DV_CTX_ATTR_BUF_ALLOCATORS,
+				       (void *)((uintptr_t)&alctr));
+	/* Bring Ethernet device up. */
+	DRV_LOG(DEBUG, "port %u forcing Ethernet interface up",
+		eth_dev->data->port_id);
+	mlx5_set_link_up(eth_dev);
+	/*
+	 * Even though the interrupt handler is not installed yet,
+	 * interrupts will still trigger on the asyn_fd from
+	 * Verbs context returned by ibv_open_device().
+	 */
+	mlx5_link_update(eth_dev, 0);
+	/* Store device configuration on private structure. */
+	priv->config = config;
+	/* Supported Verbs flow priority number detection. */
+	err = mlx5_flow_discover_priorities(eth_dev);
+	if (err < 0)
+		goto error;
+	priv->config.flow_prio = err;
+	/*
+	 * Once the device is added to the list of memory event
+	 * callback, its global MR cache table cannot be expanded
+	 * on the fly because of deadlock. If it overflows, lookup
+	 * should be done by searching MR list linearly, which is slow.
+	 */
+	err = mlx5_mr_btree_init(&priv->mr.cache,
+				 MLX5_MR_BTREE_CACHE_N * 2,
+				 eth_dev->device->numa_node);
+	if (err) {
+		err = rte_errno;
+		goto error;
+	}
+	/* Add device to memory callback list. */
+	rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock);
+	LIST_INSERT_HEAD(&mlx5_shared_data->mem_event_cb_list,
+			 priv, mem_event_cb);
+	rte_rwlock_write_unlock(&mlx5_shared_data->mem_event_rwlock);
+	return eth_dev;
+error:
+	if (priv) {
+		if (priv->nl_socket_route >= 0)
+			close(priv->nl_socket_route);
+		if (priv->nl_socket_rdma >= 0)
+			close(priv->nl_socket_rdma);
+		if (priv->mnl_socket)
+			mlx5_nl_flow_socket_destroy(priv->mnl_socket);
+		if (own_domain_id)
+			claim_zero(rte_eth_switch_domain_free(priv->domain_id));
+		rte_free(priv);
+	}
+	if (pd)
+		claim_zero(mlx5_glue->dealloc_pd(pd));
+	if (eth_dev)
+		rte_eth_dev_release_port(eth_dev);
+	if (ctx)
+		claim_zero(mlx5_glue->close_device(ctx));
+	assert(err > 0);
+	rte_errno = err;
+	return NULL;
+}
+
+/** Data associated with devices to spawn. */
+struct mlx5_dev_spawn_data {
+	unsigned int ifindex; /**< Network interface index. */
+	struct mlx5_switch_info info; /**< Switch information. */
+	struct ibv_device *ibv_dev; /**< Associated IB device. */
+	struct rte_eth_dev *eth_dev; /**< Associated Ethernet device. */
+};
+
+/**
+ * Comparison callback to sort device data.
+ *
+ * This is meant to be used with qsort().
+ *
+ * @param a[in]
+ *   Pointer to pointer to first data object.
+ * @param b[in]
+ *   Pointer to pointer to second data object.
+ *
+ * @return
+ *   0 if both objects are equal, less than 0 if the first argument is less
+ *   than the second, greater than 0 otherwise.
+ */
+static int
+mlx5_dev_spawn_data_cmp(const void *a, const void *b)
+{
+	const struct mlx5_switch_info *si_a =
+		&((const struct mlx5_dev_spawn_data *)a)->info;
+	const struct mlx5_switch_info *si_b =
+		&((const struct mlx5_dev_spawn_data *)b)->info;
+	int ret;
+
+	/* Master device first. */
+	ret = si_b->master - si_a->master;
+	if (ret)
+		return ret;
+	/* Then representor devices. */
+	ret = si_b->representor - si_a->representor;
+	if (ret)
+		return ret;
+	/* Unidentified devices come last in no specific order. */
+	if (!si_a->representor)
+		return 0;
+	/* Order representors by name. */
+	return si_a->port_name - si_b->port_name;
+}
+
+/**
+ * DPDK callback to register a PCI device.
+ *
+ * This function spawns Ethernet devices out of a given PCI device.
+ *
+ * @param[in] pci_drv
+ *   PCI driver structure (mlx5_driver).
+ * @param[in] pci_dev
+ *   PCI device information.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
+	       struct rte_pci_device *pci_dev)
+{
+	struct ibv_device **ibv_list;
+	unsigned int n = 0;
+	int vf;
+	int ret;
+
+	assert(pci_drv == &mlx5_driver);
+	errno = 0;
+	ibv_list = mlx5_glue->get_device_list(&ret);
+	if (!ibv_list) {
+		rte_errno = errno ? errno : ENOSYS;
+		DRV_LOG(ERR, "cannot list devices, is ib_uverbs loaded?");
+		return -rte_errno;
+	}
+
+	struct ibv_device *ibv_match[ret + 1];
+
+	while (ret-- > 0) {
+		struct rte_pci_addr pci_addr;
+
+		DRV_LOG(DEBUG, "checking device \"%s\"", ibv_list[ret]->name);
+		if (mlx5_ibv_device_to_pci_addr(ibv_list[ret], &pci_addr))
+			continue;
+		if (pci_dev->addr.domain != pci_addr.domain ||
+		    pci_dev->addr.bus != pci_addr.bus ||
+		    pci_dev->addr.devid != pci_addr.devid ||
+		    pci_dev->addr.function != pci_addr.function)
+			continue;
+		DRV_LOG(INFO, "PCI information matches for device \"%s\"",
+			ibv_list[ret]->name);
+		ibv_match[n++] = ibv_list[ret];
+	}
+	ibv_match[n] = NULL;
+
+	struct mlx5_dev_spawn_data list[n];
+	int nl_route = n ? mlx5_nl_init(NETLINK_ROUTE) : -1;
+	int nl_rdma = n ? mlx5_nl_init(NETLINK_RDMA) : -1;
+	unsigned int i;
+	unsigned int u;
+
+	/*
+	 * The existence of several matching entries (n > 1) means port
+	 * representors have been instantiated. No existing Verbs call nor
+	 * /sys entries can tell them apart, this can only be done through
+	 * Netlink calls assuming kernel drivers are recent enough to
+	 * support them.
+	 *
+	 * In the event of identification failure through Netlink, try again
+	 * through sysfs, then either:
+	 *
+	 * 1. No device matches (n == 0), complain and bail out.
+	 * 2. A single IB device matches (n == 1) and is not a representor,
+	 *    assume no switch support.
+	 * 3. Otherwise no safe assumptions can be made; complain louder and
+	 *    bail out.
+	 */
+	for (i = 0; i != n; ++i) {
+		list[i].ibv_dev = ibv_match[i];
+		list[i].eth_dev = NULL;
+		if (nl_rdma < 0)
+			list[i].ifindex = 0;
+		else
+			list[i].ifindex = mlx5_nl_ifindex
+				(nl_rdma, list[i].ibv_dev->name);
+		if (nl_route < 0 ||
+		    !list[i].ifindex ||
+		    mlx5_nl_switch_info(nl_route, list[i].ifindex,
+					&list[i].info) ||
+		    ((!list[i].info.representor && !list[i].info.master) &&
+		     mlx5_sysfs_switch_info(list[i].ifindex, &list[i].info))) {
+			list[i].ifindex = 0;
+			memset(&list[i].info, 0, sizeof(list[i].info));
+			continue;
+		}
+	}
+	if (nl_rdma >= 0)
+		close(nl_rdma);
+	if (nl_route >= 0)
+		close(nl_route);
+	/* Count unidentified devices. */
+	for (u = 0, i = 0; i != n; ++i)
+		if (!list[i].info.master && !list[i].info.representor)
+			++u;
+	if (u) {
+		if (n == 1 && u == 1) {
+			/* Case #2. */
+			DRV_LOG(INFO, "no switch support detected");
+		} else {
+			/* Case #3. */
+			DRV_LOG(ERR,
+				"unable to tell which of the matching devices"
+				" is the master (lack of kernel support?)");
+			n = 0;
+		}
+	}
+	/*
+	 * Sort list to probe devices in natural order for users convenience
+	 * (i.e. master first, then representors from lowest to highest ID).
+	 */
+	if (n)
+		qsort(list, n, sizeof(*list), mlx5_dev_spawn_data_cmp);
+	switch (pci_dev->id.device_id) {
+	case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF:
+	case PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF:
+	case PCI_DEVICE_ID_MELLANOX_CONNECTX5VF:
+	case PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF:
+		vf = 1;
+		break;
+	default:
+		vf = 0;
+	}
+	for (i = 0; i != n; ++i) {
+		uint32_t restore;
+
+		list[i].eth_dev = mlx5_dev_spawn
+			(&pci_dev->device, list[i].ibv_dev, vf, &list[i].info);
+		if (!list[i].eth_dev) {
+			if (rte_errno != EBUSY)
+				break;
+			/* Device is disabled, ignore it. */
+			continue;
+		}
+		restore = list[i].eth_dev->data->dev_flags;
+		rte_eth_copy_pci_info(list[i].eth_dev, pci_dev);
+		/* Restore non-PCI flags cleared by the above call. */
+		list[i].eth_dev->data->dev_flags |= restore;
+		rte_eth_dev_probing_finish(list[i].eth_dev);
+	}
+	mlx5_glue->free_device_list(ibv_list);
+	if (!n) {
+		DRV_LOG(WARNING,
+			"no Verbs device matches PCI device " PCI_PRI_FMT ","
+			" are kernel drivers loaded?",
+			pci_dev->addr.domain, pci_dev->addr.bus,
+			pci_dev->addr.devid, pci_dev->addr.function);
+		rte_errno = ENOENT;
+		ret = -rte_errno;
+	} else if (i != n) {
+		DRV_LOG(ERR,
+			"probe of PCI device " PCI_PRI_FMT " aborted after"
+			" encountering an error: %s",
+			pci_dev->addr.domain, pci_dev->addr.bus,
+			pci_dev->addr.devid, pci_dev->addr.function,
+			strerror(rte_errno));
+		ret = -rte_errno;
+		/* Roll back. */
+		while (i--) {
+			if (!list[i].eth_dev)
+				continue;
+			mlx5_dev_close(list[i].eth_dev);
+			if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+				rte_free(list[i].eth_dev->data->dev_private);
+			claim_zero(rte_eth_dev_release_port(list[i].eth_dev));
+		}
+		/* Restore original error. */
+		rte_errno = -ret;
+	} else {
+		ret = 0;
+	}
+	return ret;
+}
+
+static const struct rte_pci_id mlx5_pci_id_map[] = {
+	{
+		RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
+			       PCI_DEVICE_ID_MELLANOX_CONNECTX4)
+	},
+	{
+		RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
+			       PCI_DEVICE_ID_MELLANOX_CONNECTX4VF)
+	},
+	{
+		RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
+			       PCI_DEVICE_ID_MELLANOX_CONNECTX4LX)
+	},
+	{
+		RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
+			       PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF)
+	},
+	{
+		RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
+			       PCI_DEVICE_ID_MELLANOX_CONNECTX5)
+	},
+	{
+		RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
+			       PCI_DEVICE_ID_MELLANOX_CONNECTX5VF)
+	},
+	{
+		RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
+			       PCI_DEVICE_ID_MELLANOX_CONNECTX5EX)
+	},
+	{
+		RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
+			       PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF)
+	},
+	{
+		RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
+			       PCI_DEVICE_ID_MELLANOX_CONNECTX5BF)
+	},
+	{
+		.vendor_id = 0
+	}
+};
+
+static struct rte_pci_driver mlx5_driver = {
+	.driver = {
+		.name = MLX5_DRIVER_NAME
+	},
+	.id_table = mlx5_pci_id_map,
+	.probe = mlx5_pci_probe,
+	.drv_flags = RTE_PCI_DRV_INTR_LSC | RTE_PCI_DRV_INTR_RMV,
+};
+
+#ifdef RTE_LIBRTE_MLX5_DLOPEN_DEPS
+
+/**
+ * Suffix RTE_EAL_PMD_PATH with "-glue".
+ *
+ * This function performs a sanity check on RTE_EAL_PMD_PATH before
+ * suffixing its last component.
+ *
+ * @param buf[out]
+ *   Output buffer, should be large enough otherwise NULL is returned.
+ * @param size
+ *   Size of @p out.
+ *
+ * @return
+ *   Pointer to @p buf or @p NULL in case suffix cannot be appended.
+ */
+static char *
+mlx5_glue_path(char *buf, size_t size)
+{
+	static const char *const bad[] = { "/", ".", "..", NULL };
+	const char *path = RTE_EAL_PMD_PATH;
+	size_t len = strlen(path);
+	size_t off;
+	int i;
+
+	while (len && path[len - 1] == '/')
+		--len;
+	for (off = len; off && path[off - 1] != '/'; --off)
+		;
+	for (i = 0; bad[i]; ++i)
+		if (!strncmp(path + off, bad[i], (int)(len - off)))
+			goto error;
+	i = snprintf(buf, size, "%.*s-glue", (int)len, path);
+	if (i == -1 || (size_t)i >= size)
+		goto error;
+	return buf;
+error:
+	DRV_LOG(ERR,
+		"unable to append \"-glue\" to last component of"
+		" RTE_EAL_PMD_PATH (\"" RTE_EAL_PMD_PATH "\"),"
+		" please re-configure DPDK");
+	return NULL;
+}
+
+/**
+ * Initialization routine for run-time dependency on rdma-core.
+ */
+static int
+mlx5_glue_init(void)
+{
+	char glue_path[sizeof(RTE_EAL_PMD_PATH) - 1 + sizeof("-glue")];
+	const char *path[] = {
+		/*
+		 * A basic security check is necessary before trusting
+		 * MLX5_GLUE_PATH, which may override RTE_EAL_PMD_PATH.
+		 */
+		(geteuid() == getuid() && getegid() == getgid() ?
+		 getenv("MLX5_GLUE_PATH") : NULL),
+		/*
+		 * When RTE_EAL_PMD_PATH is set, use its glue-suffixed
+		 * variant, otherwise let dlopen() look up libraries on its
+		 * own.
+		 */
+		(*RTE_EAL_PMD_PATH ?
+		 mlx5_glue_path(glue_path, sizeof(glue_path)) : ""),
+	};
+	unsigned int i = 0;
+	void *handle = NULL;
+	void **sym;
+	const char *dlmsg;
+
+	while (!handle && i != RTE_DIM(path)) {
+		const char *end;
+		size_t len;
+		int ret;
+
+		if (!path[i]) {
+			++i;
+			continue;
+		}
+		end = strpbrk(path[i], ":;");
+		if (!end)
+			end = path[i] + strlen(path[i]);
+		len = end - path[i];
+		ret = 0;
+		do {
+			char name[ret + 1];
+
+			ret = snprintf(name, sizeof(name), "%.*s%s" MLX5_GLUE,
+				       (int)len, path[i],
+				       (!len || *(end - 1) == '/') ? "" : "/");
+			if (ret == -1)
+				break;
+			if (sizeof(name) != (size_t)ret + 1)
+				continue;
+			DRV_LOG(DEBUG, "looking for rdma-core glue as \"%s\"",
+				name);
+			handle = dlopen(name, RTLD_LAZY);
+			break;
+		} while (1);
+		path[i] = end + 1;
+		if (!*end)
+			++i;
+	}
+	if (!handle) {
+		rte_errno = EINVAL;
+		dlmsg = dlerror();
+		if (dlmsg)
+			DRV_LOG(WARNING, "cannot load glue library: %s", dlmsg);
+		goto glue_error;
+	}
+	sym = dlsym(handle, "mlx5_glue");
+	if (!sym || !*sym) {
+		rte_errno = EINVAL;
+		dlmsg = dlerror();
+		if (dlmsg)
+			DRV_LOG(ERR, "cannot resolve glue symbol: %s", dlmsg);
+		goto glue_error;
+	}
+	mlx5_glue = *sym;
+	return 0;
+glue_error:
+	if (handle)
+		dlclose(handle);
+	DRV_LOG(WARNING,
+		"cannot initialize PMD due to missing run-time dependency on"
+		" rdma-core libraries (libibverbs, libmlx5)");
+	return -rte_errno;
+}
+
+#endif
+
+/**
+ * Driver initialization routine.
+ */
+RTE_INIT(rte_mlx5_pmd_init)
+{
+	/* Initialize driver log type. */
+	mlx5_logtype = rte_log_register("pmd.net.mlx5");
+	if (mlx5_logtype >= 0)
+		rte_log_set_level(mlx5_logtype, RTE_LOG_NOTICE);
+
+	/* Build the static tables for Verbs conversion. */
+	mlx5_set_ptype_table();
+	mlx5_set_cksum_table();
+	mlx5_set_swp_types_table();
+	/*
+	 * RDMAV_HUGEPAGES_SAFE tells ibv_fork_init() we intend to use
+	 * huge pages. Calling ibv_fork_init() during init allows
+	 * applications to use fork() safely for purposes other than
+	 * using this PMD, which is not supported in forked processes.
+	 */
+	setenv("RDMAV_HUGEPAGES_SAFE", "1", 1);
+	/* Match the size of Rx completion entry to the size of a cacheline. */
+	if (RTE_CACHE_LINE_SIZE == 128)
+		setenv("MLX5_CQE_SIZE", "128", 0);
+	/*
+	 * MLX5_DEVICE_FATAL_CLEANUP tells ibv_destroy functions to
+	 * cleanup all the Verbs resources even when the device was removed.
+	 */
+	setenv("MLX5_DEVICE_FATAL_CLEANUP", "1", 1);
+#ifdef RTE_LIBRTE_MLX5_DLOPEN_DEPS
+	if (mlx5_glue_init())
+		return;
+	assert(mlx5_glue);
+#endif
+#ifndef NDEBUG
+	/* Glue structure must not contain any NULL pointers. */
+	{
+		unsigned int i;
+
+		for (i = 0; i != sizeof(*mlx5_glue) / sizeof(void *); ++i)
+			assert(((const void *const *)mlx5_glue)[i]);
+	}
+#endif
+	if (strcmp(mlx5_glue->version, MLX5_GLUE_VERSION)) {
+		DRV_LOG(ERR,
+			"rdma-core glue \"%s\" mismatch: \"%s\" is required",
+			mlx5_glue->version, MLX5_GLUE_VERSION);
+		return;
+	}
+	mlx5_glue->fork_init();
+	rte_pci_register(&mlx5_driver);
+}
+
+RTE_PMD_EXPORT_NAME(net_mlx5, __COUNTER__);
+RTE_PMD_REGISTER_PCI_TABLE(net_mlx5, mlx5_pci_id_map);
+RTE_PMD_REGISTER_KMOD_DEP(net_mlx5, "* ib_uverbs & mlx5_core & mlx5_ib");
diff --git a/src/spdk/dpdk/drivers/net/mlx5/mlx5.h b/src/spdk/dpdk/drivers/net/mlx5/mlx5.h
new file mode 100644
index 00000000..a3a34cff
--- /dev/null
+++ b/src/spdk/dpdk/drivers/net/mlx5/mlx5.h
@@ -0,0 +1,418 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2015 6WIND S.A.
+ * Copyright 2015 Mellanox Technologies, Ltd
+ */
+
+#ifndef RTE_PMD_MLX5_H_
+#define RTE_PMD_MLX5_H_
+
+#include <stddef.h>
+#include <stdint.h>
+#include <limits.h>
+#include <net/if.h>
+#include <netinet/in.h>
+#include <sys/queue.h>
+
+/* Verbs header. */
+/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <infiniband/verbs.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+#include <rte_pci.h>
+#include <rte_ether.h>
+#include <rte_ethdev_driver.h>
+#include <rte_rwlock.h>
+#include <rte_interrupts.h>
+#include <rte_errno.h>
+#include <rte_flow.h>
+
+#include "mlx5_utils.h"
+#include "mlx5_mr.h"
+#include "mlx5_rxtx.h"
+#include "mlx5_autoconf.h"
+#include "mlx5_defs.h"
+
+enum {
+	PCI_VENDOR_ID_MELLANOX = 0x15b3,
+};
+
+enum {
+	PCI_DEVICE_ID_MELLANOX_CONNECTX4 = 0x1013,
+	PCI_DEVICE_ID_MELLANOX_CONNECTX4VF = 0x1014,
+	PCI_DEVICE_ID_MELLANOX_CONNECTX4LX = 0x1015,
+	PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF = 0x1016,
+	PCI_DEVICE_ID_MELLANOX_CONNECTX5 = 0x1017,
+	PCI_DEVICE_ID_MELLANOX_CONNECTX5VF = 0x1018,
+	PCI_DEVICE_ID_MELLANOX_CONNECTX5EX = 0x1019,
+	PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF = 0x101a,
+	PCI_DEVICE_ID_MELLANOX_CONNECTX5BF = 0xa2d2,
+};
+
+/** Switch information returned by mlx5_nl_switch_info(). */
+struct mlx5_switch_info {
+	uint32_t master:1; /**< Master device. */
+	uint32_t representor:1; /**< Representor device. */
+	int32_t port_name; /**< Representor port name. */
+	uint64_t switch_id; /**< Switch identifier. */
+};
+
+LIST_HEAD(mlx5_dev_list, priv);
+
+/* Shared memory between primary and secondary processes. */
+struct mlx5_shared_data {
+	struct mlx5_dev_list mem_event_cb_list;
+	rte_rwlock_t mem_event_rwlock;
+};
+
+extern struct mlx5_shared_data *mlx5_shared_data;
+
+struct mlx5_xstats_ctrl {
+	/* Number of device stats. */
+	uint16_t stats_n;
+	/* Index in the device counters table. */
+	uint16_t dev_table_idx[MLX5_MAX_XSTATS];
+	uint64_t base[MLX5_MAX_XSTATS];
+};
+
+/* Flow list . */
+TAILQ_HEAD(mlx5_flows, rte_flow);
+
+/* Default PMD specific parameter value. */
+#define MLX5_ARG_UNSET (-1)
+
+/*
+ * Device configuration structure.
+ *
+ * Merged configuration from:
+ *
+ *  - Device capabilities,
+ *  - User device parameters disabled features.
+ */
+struct mlx5_dev_config {
+	unsigned int hw_csum:1; /* Checksum offload is supported. */
+	unsigned int hw_vlan_strip:1; /* VLAN stripping is supported. */
+	unsigned int hw_fcs_strip:1; /* FCS stripping is supported. */
+	unsigned int hw_padding:1; /* End alignment padding is supported. */
+	unsigned int vf:1; /* This is a VF. */
+	unsigned int mps:2; /* Multi-packet send supported mode. */
+	unsigned int tunnel_en:1;
+	/* Whether tunnel stateless offloads are supported. */
+	unsigned int mpls_en:1; /* MPLS over GRE/UDP is enabled. */
+	unsigned int flow_counter_en:1; /* Whether flow counter is supported. */
+	unsigned int cqe_comp:1; /* CQE compression is enabled. */
+	unsigned int tso:1; /* Whether TSO is supported. */
+	unsigned int tx_vec_en:1; /* Tx vector is enabled. */
+	unsigned int rx_vec_en:1; /* Rx vector is enabled. */
+	unsigned int mpw_hdr_dseg:1; /* Enable DSEGs in the title WQEBB. */
+	unsigned int l3_vxlan_en:1; /* Enable L3 VXLAN flow creation. */
+	unsigned int vf_nl_en:1; /* Enable Netlink requests in VF mode. */
+	unsigned int swp:1; /* Tx generic tunnel checksum and TSO offload. */
+	struct {
+		unsigned int enabled:1; /* Whether MPRQ is enabled. */
+		unsigned int stride_num_n; /* Number of strides. */
+		unsigned int min_stride_size_n; /* Min size of a stride. */
+		unsigned int max_stride_size_n; /* Max size of a stride. */
+		unsigned int max_memcpy_len;
+		/* Maximum packet size to memcpy Rx packets. */
+		unsigned int min_rxqs_num;
+		/* Rx queue count threshold to enable MPRQ. */
+	} mprq; /* Configurations for Multi-Packet RQ. */
+	unsigned int flow_prio; /* Number of flow priorities. */
+	unsigned int tso_max_payload_sz; /* Maximum TCP payload for TSO. */
+	unsigned int ind_table_max_size; /* Maximum indirection table size. */
+	int txq_inline; /* Maximum packet size for inlining. */
+	int txqs_inline; /* Queue number threshold for inlining. */
+	int inline_max_packet_sz; /* Max packet size for inlining. */
+};
+
+/**
+ * Type of objet being allocated.
+ */
+enum mlx5_verbs_alloc_type {
+	MLX5_VERBS_ALLOC_TYPE_NONE,
+	MLX5_VERBS_ALLOC_TYPE_TX_QUEUE,
+	MLX5_VERBS_ALLOC_TYPE_RX_QUEUE,
+};
+
+/**
+ * Verbs allocator needs a context to know in the callback which kind of
+ * resources it is allocating.
+ */
+struct mlx5_verbs_alloc_ctx {
+	enum mlx5_verbs_alloc_type type; /* Kind of object being allocated. */
+	const void *obj; /* Pointer to the DPDK object. */
+};
+
+LIST_HEAD(mlx5_mr_list, mlx5_mr);
+
+/* Flow drop context necessary due to Verbs API. */
+struct mlx5_drop {
+	struct mlx5_hrxq *hrxq; /* Hash Rx queue queue. */
+	struct mlx5_rxq_ibv *rxq; /* Verbs Rx queue. */
+};
+
+/** DPDK port to network interface index (ifindex) conversion. */
+struct mlx5_nl_flow_ptoi {
+	uint16_t port_id; /**< DPDK port ID. */
+	unsigned int ifindex; /**< Network interface index. */
+};
+
+struct mnl_socket;
+
+struct priv {
+	LIST_ENTRY(priv) mem_event_cb; /* Called by memory event callback. */
+	struct rte_eth_dev_data *dev_data;  /* Pointer to device data. */
+	struct ibv_context *ctx; /* Verbs context. */
+	struct ibv_device_attr_ex device_attr; /* Device properties. */
+	struct ibv_pd *pd; /* Protection Domain. */
+	char ibdev_name[IBV_SYSFS_NAME_MAX]; /* IB device name. */
+	char ibdev_path[IBV_SYSFS_PATH_MAX]; /* IB device path for secondary */
+	struct ether_addr mac[MLX5_MAX_MAC_ADDRESSES]; /* MAC addresses. */
+	BITFIELD_DECLARE(mac_own, uint64_t, MLX5_MAX_MAC_ADDRESSES);
+	/* Bit-field of MAC addresses owned by the PMD. */
+	uint16_t vlan_filter[MLX5_MAX_VLAN_IDS]; /* VLAN filters table. */
+	unsigned int vlan_filter_n; /* Number of configured VLAN filters. */
+	/* Device properties. */
+	uint16_t mtu; /* Configured MTU. */
+	unsigned int isolated:1; /* Whether isolated mode is enabled. */
+	unsigned int representor:1; /* Device is a port representor. */
+	uint16_t domain_id; /* Switch domain identifier. */
+	int32_t representor_id; /* Port representor identifier. */
+	/* RX/TX queues. */
+	unsigned int rxqs_n; /* RX queues array size. */
+	unsigned int txqs_n; /* TX queues array size. */
+	struct mlx5_rxq_data *(*rxqs)[]; /* RX queues. */
+	struct mlx5_txq_data *(*txqs)[]; /* TX queues. */
+	struct rte_mempool *mprq_mp; /* Mempool for Multi-Packet RQ. */
+	struct rte_eth_rss_conf rss_conf; /* RSS configuration. */
+	struct rte_intr_handle intr_handle; /* Interrupt handler. */
+	unsigned int (*reta_idx)[]; /* RETA index table. */
+	unsigned int reta_idx_n; /* RETA index size. */
+	struct mlx5_drop drop_queue; /* Flow drop queues. */
+	struct mlx5_flows flows; /* RTE Flow rules. */
+	struct mlx5_flows ctrl_flows; /* Control flow rules. */
+	LIST_HEAD(counters, mlx5_flow_counter) flow_counters;
+	/* Flow counters. */
+	struct {
+		uint32_t dev_gen; /* Generation number to flush local caches. */
+		rte_rwlock_t rwlock; /* MR Lock. */
+		struct mlx5_mr_btree cache; /* Global MR cache table. */
+		struct mlx5_mr_list mr_list; /* Registered MR list. */
+		struct mlx5_mr_list mr_free_list; /* Freed MR list. */
+	} mr;
+	LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
+	LIST_HEAD(rxqibv, mlx5_rxq_ibv) rxqsibv; /* Verbs Rx queues. */
+	LIST_HEAD(hrxq, mlx5_hrxq) hrxqs; /* Verbs Hash Rx queues. */
+	LIST_HEAD(txq, mlx5_txq_ctrl) txqsctrl; /* DPDK Tx queues. */
+	LIST_HEAD(txqibv, mlx5_txq_ibv) txqsibv; /* Verbs Tx queues. */
+	/* Verbs Indirection tables. */
+	LIST_HEAD(ind_tables, mlx5_ind_table_ibv) ind_tbls;
+	uint32_t link_speed_capa; /* Link speed capabilities. */
+	struct mlx5_xstats_ctrl xstats_ctrl; /* Extended stats control. */
+	int primary_socket; /* Unix socket for primary process. */
+	void *uar_base; /* Reserved address space for UAR mapping */
+	struct rte_intr_handle intr_handle_socket; /* Interrupt handler. */
+	struct mlx5_dev_config config; /* Device configuration. */
+	struct mlx5_verbs_alloc_ctx verbs_alloc_ctx;
+	/* Context for Verbs allocator. */
+	int nl_socket_rdma; /* Netlink socket (NETLINK_RDMA). */
+	int nl_socket_route; /* Netlink socket (NETLINK_ROUTE). */
+	uint32_t nl_sn; /* Netlink message sequence number. */
+#ifndef RTE_ARCH_64
+	rte_spinlock_t uar_lock_cq; /* CQs share a common distinct UAR */
+	rte_spinlock_t uar_lock[MLX5_UAR_PAGE_NUM_MAX];
+	/* UAR same-page access control required in 32bit implementations. */
+#endif
+	struct mnl_socket *mnl_socket; /* Libmnl socket. */
+};
+
+#define PORT_ID(priv) ((priv)->dev_data->port_id)
+#define ETH_DEV(priv) (&rte_eth_devices[PORT_ID(priv)])
+
+/* mlx5.c */
+
+int mlx5_getenv_int(const char *);
+
+/* mlx5_ethdev.c */
+
+int mlx5_get_master_ifname(const struct rte_eth_dev *dev,
+			   char (*ifname)[IF_NAMESIZE]);
+int mlx5_get_ifname(const struct rte_eth_dev *dev, char (*ifname)[IF_NAMESIZE]);
+unsigned int mlx5_ifindex(const struct rte_eth_dev *dev);
+int mlx5_ifreq(const struct rte_eth_dev *dev, int req, struct ifreq *ifr,
+	       int master);
+int mlx5_get_mtu(struct rte_eth_dev *dev, uint16_t *mtu);
+int mlx5_set_flags(struct rte_eth_dev *dev, unsigned int keep,
+		   unsigned int flags);
+int mlx5_dev_configure(struct rte_eth_dev *dev);
+void mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info);
+const uint32_t *mlx5_dev_supported_ptypes_get(struct rte_eth_dev *dev);
+int mlx5_link_update(struct rte_eth_dev *dev, int wait_to_complete);
+int mlx5_force_link_status_change(struct rte_eth_dev *dev, int status);
+int mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu);
+int mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev,
+			   struct rte_eth_fc_conf *fc_conf);
+int mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev,
+			   struct rte_eth_fc_conf *fc_conf);
+int mlx5_ibv_device_to_pci_addr(const struct ibv_device *device,
+				struct rte_pci_addr *pci_addr);
+void mlx5_dev_link_status_handler(void *arg);
+void mlx5_dev_interrupt_handler(void *arg);
+void mlx5_dev_interrupt_handler_uninstall(struct rte_eth_dev *dev);
+void mlx5_dev_interrupt_handler_install(struct rte_eth_dev *dev);
+int mlx5_set_link_down(struct rte_eth_dev *dev);
+int mlx5_set_link_up(struct rte_eth_dev *dev);
+int mlx5_is_removed(struct rte_eth_dev *dev);
+eth_tx_burst_t mlx5_select_tx_function(struct rte_eth_dev *dev);
+eth_rx_burst_t mlx5_select_rx_function(struct rte_eth_dev *dev);
+unsigned int mlx5_dev_to_port_id(const struct rte_device *dev,
+				 uint16_t *port_list,
+				 unsigned int port_list_n);
+int mlx5_sysfs_switch_info(unsigned int ifindex,
+			   struct mlx5_switch_info *info);
+
+/* mlx5_mac.c */
+
+int mlx5_get_mac(struct rte_eth_dev *dev, uint8_t (*mac)[ETHER_ADDR_LEN]);
+void mlx5_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index);
+int mlx5_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac,
+		      uint32_t index, uint32_t vmdq);
+int mlx5_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr);
+int mlx5_set_mc_addr_list(struct rte_eth_dev *dev,
+			  struct ether_addr *mc_addr_set, uint32_t nb_mc_addr);
+
+/* mlx5_rss.c */
+
+int mlx5_rss_hash_update(struct rte_eth_dev *dev,
+			 struct rte_eth_rss_conf *rss_conf);
+int mlx5_rss_hash_conf_get(struct rte_eth_dev *dev,
+			   struct rte_eth_rss_conf *rss_conf);
+int mlx5_rss_reta_index_resize(struct rte_eth_dev *dev, unsigned int reta_size);
+int mlx5_dev_rss_reta_query(struct rte_eth_dev *dev,
+			    struct rte_eth_rss_reta_entry64 *reta_conf,
+			    uint16_t reta_size);
+int mlx5_dev_rss_reta_update(struct rte_eth_dev *dev,
+			     struct rte_eth_rss_reta_entry64 *reta_conf,
+			     uint16_t reta_size);
+
+/* mlx5_rxmode.c */
+
+void mlx5_promiscuous_enable(struct rte_eth_dev *dev);
+void mlx5_promiscuous_disable(struct rte_eth_dev *dev);
+void mlx5_allmulticast_enable(struct rte_eth_dev *dev);
+void mlx5_allmulticast_disable(struct rte_eth_dev *dev);
+
+/* mlx5_stats.c */
+
+void mlx5_xstats_init(struct rte_eth_dev *dev);
+int mlx5_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats);
+void mlx5_stats_reset(struct rte_eth_dev *dev);
+int mlx5_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *stats,
+		    unsigned int n);
+void mlx5_xstats_reset(struct rte_eth_dev *dev);
+int mlx5_xstats_get_names(struct rte_eth_dev *dev __rte_unused,
+			  struct rte_eth_xstat_name *xstats_names,
+			  unsigned int n);
+
+/* mlx5_vlan.c */
+
+int mlx5_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
+void mlx5_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on);
+int mlx5_vlan_offload_set(struct rte_eth_dev *dev, int mask);
+
+/* mlx5_trigger.c */
+
+int mlx5_dev_start(struct rte_eth_dev *dev);
+void mlx5_dev_stop(struct rte_eth_dev *dev);
+int mlx5_traffic_enable(struct rte_eth_dev *dev);
+void mlx5_traffic_disable(struct rte_eth_dev *dev);
+int mlx5_traffic_restart(struct rte_eth_dev *dev);
+
+/* mlx5_flow.c */
+
+int mlx5_flow_discover_priorities(struct rte_eth_dev *dev);
+void mlx5_flow_print(struct rte_flow *flow);
+int mlx5_flow_validate(struct rte_eth_dev *dev,
+		       const struct rte_flow_attr *attr,
+		       const struct rte_flow_item items[],
+		       const struct rte_flow_action actions[],
+		       struct rte_flow_error *error);
+struct rte_flow *mlx5_flow_create(struct rte_eth_dev *dev,
+				  const struct rte_flow_attr *attr,
+				  const struct rte_flow_item items[],
+				  const struct rte_flow_action actions[],
+				  struct rte_flow_error *error);
+int mlx5_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
+		      struct rte_flow_error *error);
+void mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list);
+int mlx5_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error);
+int mlx5_flow_query(struct rte_eth_dev *dev, struct rte_flow *flow,
+		    const struct rte_flow_action *action, void *data,
+		    struct rte_flow_error *error);
+int mlx5_flow_isolate(struct rte_eth_dev *dev, int enable,
+		      struct rte_flow_error *error);
+int mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
+			 enum rte_filter_type filter_type,
+			 enum rte_filter_op filter_op,
+			 void *arg);
+int mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list);
+void mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list);
+int mlx5_flow_verify(struct rte_eth_dev *dev);
+int mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
+			struct rte_flow_item_eth *eth_spec,
+			struct rte_flow_item_eth *eth_mask,
+			struct rte_flow_item_vlan *vlan_spec,
+			struct rte_flow_item_vlan *vlan_mask);
+int mlx5_ctrl_flow(struct rte_eth_dev *dev,
+		   struct rte_flow_item_eth *eth_spec,
+		   struct rte_flow_item_eth *eth_mask);
+int mlx5_flow_create_drop_queue(struct rte_eth_dev *dev);
+void mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev);
+
+/* mlx5_socket.c */
+
+int mlx5_socket_init(struct rte_eth_dev *priv);
+void mlx5_socket_uninit(struct rte_eth_dev *priv);
+void mlx5_socket_handle(struct rte_eth_dev *priv);
+int mlx5_socket_connect(struct rte_eth_dev *priv);
+
+/* mlx5_nl.c */
+
+int mlx5_nl_init(int protocol);
+int mlx5_nl_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac,
+			 uint32_t index);
+int mlx5_nl_mac_addr_remove(struct rte_eth_dev *dev, struct ether_addr *mac,
+			    uint32_t index);
+void mlx5_nl_mac_addr_sync(struct rte_eth_dev *dev);
+void mlx5_nl_mac_addr_flush(struct rte_eth_dev *dev);
+int mlx5_nl_promisc(struct rte_eth_dev *dev, int enable);
+int mlx5_nl_allmulti(struct rte_eth_dev *dev, int enable);
+unsigned int mlx5_nl_ifindex(int nl, const char *name);
+int mlx5_nl_switch_info(int nl, unsigned int ifindex,
+			struct mlx5_switch_info *info);
+
+/* mlx5_nl_flow.c */
+
+int mlx5_nl_flow_transpose(void *buf,
+			   size_t size,
+			   const struct mlx5_nl_flow_ptoi *ptoi,
+			   const struct rte_flow_attr *attr,
+			   const struct rte_flow_item *pattern,
+			   const struct rte_flow_action *actions,
+			   struct rte_flow_error *error);
+void mlx5_nl_flow_brand(void *buf, uint32_t handle);
+int mlx5_nl_flow_create(struct mnl_socket *nl, void *buf,
+			struct rte_flow_error *error);
+int mlx5_nl_flow_destroy(struct mnl_socket *nl, void *buf,
+			 struct rte_flow_error *error);
+int mlx5_nl_flow_init(struct mnl_socket *nl, unsigned int ifindex,
+		      struct rte_flow_error *error);
+struct mnl_socket *mlx5_nl_flow_socket_create(void);
+void mlx5_nl_flow_socket_destroy(struct mnl_socket *nl);
+
+#endif /* RTE_PMD_MLX5_H_ */
diff --git a/src/spdk/dpdk/drivers/net/mlx5/mlx5_defs.h b/src/spdk/dpdk/drivers/net/mlx5/mlx5_defs.h
new file mode 100644
index 00000000..f2a16795
--- /dev/null
+++ b/src/spdk/dpdk/drivers/net/mlx5/mlx5_defs.h
@@ -0,0 +1,136 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2015 6WIND S.A.
+ * Copyright 2015 Mellanox Technologies, Ltd
+ */
+
+#ifndef RTE_PMD_MLX5_DEFS_H_
+#define RTE_PMD_MLX5_DEFS_H_
+
+#include <rte_ethdev_driver.h>
+
+#include "mlx5_autoconf.h"
+
+/* Reported driver name. */
+#define MLX5_DRIVER_NAME "net_mlx5"
+
+/* Maximum number of simultaneous unicast MAC addresses. */
+#define MLX5_MAX_UC_MAC_ADDRESSES 128
+/* Maximum number of simultaneous Multicast MAC addresses. */
+#define MLX5_MAX_MC_MAC_ADDRESSES 128
+/* Maximum number of simultaneous MAC addresses. */
+#define MLX5_MAX_MAC_ADDRESSES \
+	(MLX5_MAX_UC_MAC_ADDRESSES + MLX5_MAX_MC_MAC_ADDRESSES)
+
+/* Maximum number of simultaneous VLAN filters. */
+#define MLX5_MAX_VLAN_IDS 128
+
+/*
+ * Request TX completion every time descriptors reach this threshold since
+ * the previous request. Must be a power of two for performance reasons.
+ */
+#define MLX5_TX_COMP_THRESH 32
+
+/*
+ * Request TX completion every time the total number of WQEBBs used for inlining
+ * packets exceeds the size of WQ divided by this divisor. Better to be power of
+ * two for performance.
+ */
+#define MLX5_TX_COMP_THRESH_INLINE_DIV (1 << 3)
+
+/* Size of per-queue MR cache array for linear search. */
+#define MLX5_MR_CACHE_N 8
+
+/* Size of MR cache table for binary search. */
+#define MLX5_MR_BTREE_CACHE_N 256
+
+/*
+ * If defined, only use software counters. The PMD will never ask the hardware
+ * for these, and many of them won't be available.
+ */
+#ifndef MLX5_PMD_SOFT_COUNTERS
+#define MLX5_PMD_SOFT_COUNTERS 1
+#endif
+
+/* Alarm timeout. */
+#define MLX5_ALARM_TIMEOUT_US 100000
+
+/* Maximum number of extended statistics counters. */
+#define MLX5_MAX_XSTATS 32
+
+/* Maximum Packet headers size (L2+L3+L4) for TSO. */
+#define MLX5_MAX_TSO_HEADER 192
+
+/* Default minimum number of Tx queues for vectorized Tx. */
+#define MLX5_VPMD_MIN_TXQS 4
+
+/* Threshold of buffer replenishment for vectorized Rx. */
+#define MLX5_VPMD_RXQ_RPLNSH_THRESH(n) \
+	(RTE_MIN(MLX5_VPMD_RX_MAX_BURST, (unsigned int)(n) >> 2))
+
+/* Maximum size of burst for vectorized Rx. */
+#define MLX5_VPMD_RX_MAX_BURST 64U
+
+/*
+ * Maximum size of burst for vectorized Tx. This is related to the maximum size
+ * of Enhanced MPW (eMPW) WQE as vectorized Tx is supported with eMPW.
+ * Careful when changing, large value can cause WQE DS to overlap.
+ */
+#define MLX5_VPMD_TX_MAX_BURST        32U
+
+/* Number of packets vectorized Rx can simultaneously process in a loop. */
+#define MLX5_VPMD_DESCS_PER_LOOP      4
+
+/* Supported RSS */
+#define MLX5_RSS_HF_MASK (~(ETH_RSS_IP | ETH_RSS_UDP | ETH_RSS_TCP))
+
+/* Timeout in seconds to get a valid link status. */
+#define MLX5_LINK_STATUS_TIMEOUT 10
+
+/* Reserved address space for UAR mapping. */
+#define MLX5_UAR_SIZE (1ULL << (sizeof(uintptr_t) * 4))
+
+/* Offset of reserved UAR address space to hugepage memory. Offset is used here
+ * to minimize possibility of address next to hugepage being used by other code
+ * in either primary or secondary process, failing to map TX UAR would make TX
+ * packets invisible to HW.
+ */
+#define MLX5_UAR_OFFSET (1ULL << (sizeof(uintptr_t) * 4))
+
+/* Maximum number of UAR pages used by a port,
+ * These are the size and mask for an array of mutexes used to synchronize
+ * the access to port's UARs on platforms that do not support 64 bit writes.
+ * In such systems it is possible to issue the 64 bits DoorBells through two
+ * consecutive writes, each write 32 bits. The access to a UAR page (which can
+ * be accessible by all threads in the process) must be synchronized
+ * (for example, using a semaphore). Such a synchronization is not required
+ * when ringing DoorBells on different UAR pages.
+ * A port with 512 Tx queues uses 8, 4kBytes, UAR pages which are shared
+ * among the ports.
+ */
+#define MLX5_UAR_PAGE_NUM_MAX 64
+#define MLX5_UAR_PAGE_NUM_MASK ((MLX5_UAR_PAGE_NUM_MAX) - 1)
+
+/* Log 2 of the default number of strides per WQE for Multi-Packet RQ. */
+#define MLX5_MPRQ_STRIDE_NUM_N 6U
+
+/* Two-byte shift is disabled for Multi-Packet RQ. */
+#define MLX5_MPRQ_TWO_BYTE_SHIFT 0
+
+/*
+ * Minimum size of packet to be memcpy'd instead of being attached as an
+ * external buffer.
+ */
+#define MLX5_MPRQ_MEMCPY_DEFAULT_LEN 128
+
+/* Minimum number Rx queues to enable Multi-Packet RQ. */
+#define MLX5_MPRQ_MIN_RXQS 12
+
+/* Cache size of mempool for Multi-Packet RQ. */
+#define MLX5_MPRQ_MP_CACHE_SZ 32U
+
+/* Definition of static_assert found in /usr/include/assert.h */
+#ifndef HAVE_STATIC_ASSERT
+#define static_assert _Static_assert
+#endif
+
+#endif /* RTE_PMD_MLX5_DEFS_H_ */
diff --git a/src/spdk/dpdk/drivers/net/mlx5/mlx5_ethdev.c b/src/spdk/dpdk/drivers/net/mlx5/mlx5_ethdev.c
new file mode 100644
index 00000000..34c5b95e
--- /dev/null
+++ b/src/spdk/dpdk/drivers/net/mlx5/mlx5_ethdev.c
@@ -0,0 +1,1372 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2015 6WIND S.A.
+ * Copyright 2015 Mellanox Technologies, Ltd
+ */
+
+#define _GNU_SOURCE
+
+#include <stddef.h>
+#include <assert.h>
+#include <inttypes.h>
+#include <unistd.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <dirent.h>
+#include <net/if.h>
+#include <sys/ioctl.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <linux/ethtool.h>
+#include <linux/sockios.h>
+#include <fcntl.h>
+#include <stdalign.h>
+#include <sys/un.h>
+#include <time.h>
+
+#include <rte_atomic.h>
+#include <rte_ethdev_driver.h>
+#include <rte_bus_pci.h>
+#include <rte_mbuf.h>
+#include <rte_common.h>
+#include <rte_interrupts.h>
+#include <rte_malloc.h>
+#include <rte_string_fns.h>
+#include <rte_rwlock.h>
+
+#include "mlx5.h"
+#include "mlx5_glue.h"
+#include "mlx5_rxtx.h"
+#include "mlx5_utils.h"
+
+/* Supported speed values found in /usr/include/linux/ethtool.h */
+#ifndef HAVE_SUPPORTED_40000baseKR4_Full
+#define SUPPORTED_40000baseKR4_Full (1 << 23)
+#endif
+#ifndef HAVE_SUPPORTED_40000baseCR4_Full
+#define SUPPORTED_40000baseCR4_Full (1 << 24)
+#endif
+#ifndef HAVE_SUPPORTED_40000baseSR4_Full
+#define SUPPORTED_40000baseSR4_Full (1 << 25)
+#endif
+#ifndef HAVE_SUPPORTED_40000baseLR4_Full
+#define SUPPORTED_40000baseLR4_Full (1 << 26)
+#endif
+#ifndef HAVE_SUPPORTED_56000baseKR4_Full
+#define SUPPORTED_56000baseKR4_Full (1 << 27)
+#endif
+#ifndef HAVE_SUPPORTED_56000baseCR4_Full
+#define SUPPORTED_56000baseCR4_Full (1 << 28)
+#endif
+#ifndef HAVE_SUPPORTED_56000baseSR4_Full
+#define SUPPORTED_56000baseSR4_Full (1 << 29)
+#endif
+#ifndef HAVE_SUPPORTED_56000baseLR4_Full
+#define SUPPORTED_56000baseLR4_Full (1 << 30)
+#endif
+
+/* Add defines in case the running kernel is not the same as user headers. */
+#ifndef ETHTOOL_GLINKSETTINGS
+struct ethtool_link_settings {
+	uint32_t cmd;
+	uint32_t speed;
+	uint8_t duplex;
+	uint8_t port;
+	uint8_t phy_address;
+	uint8_t autoneg;
+	uint8_t mdio_support;
+	uint8_t eth_to_mdix;
+	uint8_t eth_tp_mdix_ctrl;
+	int8_t link_mode_masks_nwords;
+	uint32_t reserved[8];
+	uint32_t link_mode_masks[];
+};
+
+#define ETHTOOL_GLINKSETTINGS 0x0000004c
+#define ETHTOOL_LINK_MODE_1000baseT_Full_BIT 5
+#define ETHTOOL_LINK_MODE_Autoneg_BIT 6
+#define ETHTOOL_LINK_MODE_1000baseKX_Full_BIT 17
+#define ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT 18
+#define ETHTOOL_LINK_MODE_10000baseKR_Full_BIT 19
+#define ETHTOOL_LINK_MODE_10000baseR_FEC_BIT 20
+#define ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT 21
+#define ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT 22
+#define ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT 23
+#define ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT 24
+#define ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT 25
+#define ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT 26
+#define ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT 27
+#define ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT 28
+#define ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT 29
+#define ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT 30
+#endif
+#ifndef HAVE_ETHTOOL_LINK_MODE_25G
+#define ETHTOOL_LINK_MODE_25000baseCR_Full_BIT 31
+#define ETHTOOL_LINK_MODE_25000baseKR_Full_BIT 32
+#define ETHTOOL_LINK_MODE_25000baseSR_Full_BIT 33
+#endif
+#ifndef HAVE_ETHTOOL_LINK_MODE_50G
+#define ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT 34
+#define ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT 35
+#endif
+#ifndef HAVE_ETHTOOL_LINK_MODE_100G
+#define ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT 36
+#define ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT 37
+#define ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT 38
+#define ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT 39
+#endif
+
+/**
+ * Get master interface name from private structure.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ * @param[out] ifname
+ *   Interface name output buffer.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_get_master_ifname(const struct rte_eth_dev *dev,
+		       char (*ifname)[IF_NAMESIZE])
+{
+	struct priv *priv = dev->data->dev_private;
+	DIR *dir;
+	struct dirent *dent;
+	unsigned int dev_type = 0;
+	unsigned int dev_port_prev = ~0u;
+	char match[IF_NAMESIZE] = "";
+
+	{
+		MKSTR(path, "%s/device/net", priv->ibdev_path);
+
+		dir = opendir(path);
+		if (dir == NULL) {
+			rte_errno = errno;
+			return -rte_errno;
+		}
+	}
+	while ((dent = readdir(dir)) != NULL) {
+		char *name = dent->d_name;
+		FILE *file;
+		unsigned int dev_port;
+		int r;
+
+		if ((name[0] == '.') &&
+		    ((name[1] == '\0') ||
+		     ((name[1] == '.') && (name[2] == '\0'))))
+			continue;
+
+		MKSTR(path, "%s/device/net/%s/%s",
+		      priv->ibdev_path, name,
+		      (dev_type ? "dev_id" : "dev_port"));
+
+		file = fopen(path, "rb");
+		if (file == NULL) {
+			if (errno != ENOENT)
+				continue;
+			/*
+			 * Switch to dev_id when dev_port does not exist as
+			 * is the case with Linux kernel versions < 3.15.
+			 */
+try_dev_id:
+			match[0] = '\0';
+			if (dev_type)
+				break;
+			dev_type = 1;
+			dev_port_prev = ~0u;
+			rewinddir(dir);
+			continue;
+		}
+		r = fscanf(file, (dev_type ? "%x" : "%u"), &dev_port);
+		fclose(file);
+		if (r != 1)
+			continue;
+		/*
+		 * Switch to dev_id when dev_port returns the same value for
+		 * all ports. May happen when using a MOFED release older than
+		 * 3.0 with a Linux kernel >= 3.15.
+		 */
+		if (dev_port == dev_port_prev)
+			goto try_dev_id;
+		dev_port_prev = dev_port;
+		if (dev_port == 0)
+			strlcpy(match, name, sizeof(match));
+	}
+	closedir(dir);
+	if (match[0] == '\0') {
+		rte_errno = ENOENT;
+		return -rte_errno;
+	}
+	strncpy(*ifname, match, sizeof(*ifname));
+	return 0;
+}
+
+/**
+ * Get interface name from private structure.
+ *
+ * This is a port representor-aware version of mlx5_get_master_ifname().
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ * @param[out] ifname
+ *   Interface name output buffer.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_get_ifname(const struct rte_eth_dev *dev, char (*ifname)[IF_NAMESIZE])
+{
+	struct priv *priv = dev->data->dev_private;
+	unsigned int ifindex =
+		priv->nl_socket_rdma >= 0 ?
+		mlx5_nl_ifindex(priv->nl_socket_rdma, priv->ibdev_name) : 0;
+
+	if (!ifindex) {
+		if (!priv->representor)
+			return mlx5_get_master_ifname(dev, ifname);
+		rte_errno = ENXIO;
+		return -rte_errno;
+	}
+	if (if_indextoname(ifindex, &(*ifname)[0]))
+		return 0;
+	rte_errno = errno;
+	return -rte_errno;
+}
+
+/**
+ * Get the interface index from device name.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   Nonzero interface index on success, zero otherwise and rte_errno is set.
+ */
+unsigned int
+mlx5_ifindex(const struct rte_eth_dev *dev)
+{
+	char ifname[IF_NAMESIZE];
+	unsigned int ifindex;
+
+	if (mlx5_get_ifname(dev, &ifname))
+		return 0;
+	ifindex = if_nametoindex(ifname);
+	if (!ifindex)
+		rte_errno = errno;
+	return ifindex;
+}
+
+/**
+ * Perform ifreq ioctl() on associated Ethernet device.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ * @param req
+ *   Request number to pass to ioctl().
+ * @param[out] ifr
+ *   Interface request structure output buffer.
+ * @param master
+ *   When device is a port representor, perform request on master device
+ *   instead.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_ifreq(const struct rte_eth_dev *dev, int req, struct ifreq *ifr,
+	   int master)
+{
+	int sock = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP);
+	int ret = 0;
+
+	if (sock == -1) {
+		rte_errno = errno;
+		return -rte_errno;
+	}
+	if (master)
+		ret = mlx5_get_master_ifname(dev, &ifr->ifr_name);
+	else
+		ret = mlx5_get_ifname(dev, &ifr->ifr_name);
+	if (ret)
+		goto error;
+	ret = ioctl(sock, req, ifr);
+	if (ret == -1) {
+		rte_errno = errno;
+		goto error;
+	}
+	close(sock);
+	return 0;
+error:
+	close(sock);
+	return -rte_errno;
+}
+
+/**
+ * Get device MTU.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param[out] mtu
+ *   MTU value output buffer.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_get_mtu(struct rte_eth_dev *dev, uint16_t *mtu)
+{
+	struct ifreq request;
+	int ret = mlx5_ifreq(dev, SIOCGIFMTU, &request, 0);
+
+	if (ret)
+		return ret;
+	*mtu = request.ifr_mtu;
+	return 0;
+}
+
+/**
+ * Set device MTU.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param mtu
+ *   MTU value to set.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
+{
+	struct ifreq request = { .ifr_mtu = mtu, };
+
+	return mlx5_ifreq(dev, SIOCSIFMTU, &request, 0);
+}
+
+/**
+ * Set device flags.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param keep
+ *   Bitmask for flags that must remain untouched.
+ * @param flags
+ *   Bitmask for flags to modify.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_set_flags(struct rte_eth_dev *dev, unsigned int keep, unsigned int flags)
+{
+	struct ifreq request;
+	int ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &request, 0);
+
+	if (ret)
+		return ret;
+	request.ifr_flags &= keep;
+	request.ifr_flags |= flags & ~keep;
+	return mlx5_ifreq(dev, SIOCSIFFLAGS, &request, 0);
+}
+
+/**
+ * DPDK callback for Ethernet device configuration.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_dev_configure(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	unsigned int rxqs_n = dev->data->nb_rx_queues;
+	unsigned int txqs_n = dev->data->nb_tx_queues;
+	unsigned int i;
+	unsigned int j;
+	unsigned int reta_idx_n;
+	const uint8_t use_app_rss_key =
+		!!dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key;
+	int ret = 0;
+
+	if (use_app_rss_key &&
+	    (dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len !=
+	     MLX5_RSS_HASH_KEY_LEN)) {
+		DRV_LOG(ERR, "port %u RSS key len must be %s Bytes long",
+			dev->data->port_id, RTE_STR(MLX5_RSS_HASH_KEY_LEN));
+		rte_errno = EINVAL;
+		return -rte_errno;
+	}
+	priv->rss_conf.rss_key =
+		rte_realloc(priv->rss_conf.rss_key,
+			    MLX5_RSS_HASH_KEY_LEN, 0);
+	if (!priv->rss_conf.rss_key) {
+		DRV_LOG(ERR, "port %u cannot allocate RSS hash key memory (%u)",
+			dev->data->port_id, rxqs_n);
+		rte_errno = ENOMEM;
+		return -rte_errno;
+	}
+	memcpy(priv->rss_conf.rss_key,
+	       use_app_rss_key ?
+	       dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key :
+	       rss_hash_default_key,
+	       MLX5_RSS_HASH_KEY_LEN);
+	priv->rss_conf.rss_key_len = MLX5_RSS_HASH_KEY_LEN;
+	priv->rss_conf.rss_hf = dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf;
+	priv->rxqs = (void *)dev->data->rx_queues;
+	priv->txqs = (void *)dev->data->tx_queues;
+	if (txqs_n != priv->txqs_n) {
+		DRV_LOG(INFO, "port %u Tx queues number update: %u -> %u",
+			dev->data->port_id, priv->txqs_n, txqs_n);
+		priv->txqs_n = txqs_n;
+	}
+	if (rxqs_n > priv->config.ind_table_max_size) {
+		DRV_LOG(ERR, "port %u cannot handle this many Rx queues (%u)",
+			dev->data->port_id, rxqs_n);
+		rte_errno = EINVAL;
+		return -rte_errno;
+	}
+	if (rxqs_n == priv->rxqs_n)
+		return 0;
+	DRV_LOG(INFO, "port %u Rx queues number update: %u -> %u",
+		dev->data->port_id, priv->rxqs_n, rxqs_n);
+	priv->rxqs_n = rxqs_n;
+	/* If the requested number of RX queues is not a power of two, use the
+	 * maximum indirection table size for better balancing.
+	 * The result is always rounded to the next power of two. */
+	reta_idx_n = (1 << log2above((rxqs_n & (rxqs_n - 1)) ?
+				     priv->config.ind_table_max_size :
+				     rxqs_n));
+	ret = mlx5_rss_reta_index_resize(dev, reta_idx_n);
+	if (ret)
+		return ret;
+	/* When the number of RX queues is not a power of two, the remaining
+	 * table entries are padded with reused WQs and hashes are not spread
+	 * uniformly. */
+	for (i = 0, j = 0; (i != reta_idx_n); ++i) {
+		(*priv->reta_idx)[i] = j;
+		if (++j == rxqs_n)
+			j = 0;
+	}
+	return 0;
+}
+
+/**
+ * Sets default tuning parameters.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param[out] info
+ *   Info structure output buffer.
+ */
+static void
+mlx5_set_default_params(struct rte_eth_dev *dev, struct rte_eth_dev_info *info)
+{
+	struct priv *priv = dev->data->dev_private;
+
+	/* Minimum CPU utilization. */
+	info->default_rxportconf.ring_size = 256;
+	info->default_txportconf.ring_size = 256;
+	info->default_rxportconf.burst_size = 64;
+	info->default_txportconf.burst_size = 64;
+	if (priv->link_speed_capa & ETH_LINK_SPEED_100G) {
+		info->default_rxportconf.nb_queues = 16;
+		info->default_txportconf.nb_queues = 16;
+		if (dev->data->nb_rx_queues > 2 ||
+		    dev->data->nb_tx_queues > 2) {
+			/* Max Throughput. */
+			info->default_rxportconf.ring_size = 2048;
+			info->default_txportconf.ring_size = 2048;
+		}
+	} else {
+		info->default_rxportconf.nb_queues = 8;
+		info->default_txportconf.nb_queues = 8;
+		if (dev->data->nb_rx_queues > 2 ||
+		    dev->data->nb_tx_queues > 2) {
+			/* Max Throughput. */
+			info->default_rxportconf.ring_size = 4096;
+			info->default_txportconf.ring_size = 4096;
+		}
+	}
+}
+
+/**
+ * DPDK callback to get information about the device.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param[out] info
+ *   Info structure output buffer.
+ */
+void
+mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_dev_config *config = &priv->config;
+	unsigned int max;
+	char ifname[IF_NAMESIZE];
+
+	/* FIXME: we should ask the device for these values. */
+	info->min_rx_bufsize = 32;
+	info->max_rx_pktlen = 65536;
+	/*
+	 * Since we need one CQ per QP, the limit is the minimum number
+	 * between the two values.
+	 */
+	max = RTE_MIN(priv->device_attr.orig_attr.max_cq,
+		      priv->device_attr.orig_attr.max_qp);
+	/* If max >= 65535 then max = 0, max_rx_queues is uint16_t. */
+	if (max >= 65535)
+		max = 65535;
+	info->max_rx_queues = max;
+	info->max_tx_queues = max;
+	info->max_mac_addrs = MLX5_MAX_UC_MAC_ADDRESSES;
+	info->rx_queue_offload_capa = mlx5_get_rx_queue_offloads(dev);
+	info->rx_offload_capa = (mlx5_get_rx_port_offloads() |
+				 info->rx_queue_offload_capa);
+	info->tx_offload_capa = mlx5_get_tx_port_offloads(dev);
+	if (mlx5_get_ifname(dev, &ifname) == 0)
+		info->if_index = if_nametoindex(ifname);
+	info->reta_size = priv->reta_idx_n ?
+		priv->reta_idx_n : config->ind_table_max_size;
+	info->hash_key_size = MLX5_RSS_HASH_KEY_LEN;
+	info->speed_capa = priv->link_speed_capa;
+	info->flow_type_rss_offloads = ~MLX5_RSS_HF_MASK;
+	mlx5_set_default_params(dev, info);
+	info->switch_info.name = dev->data->name;
+	info->switch_info.domain_id = priv->domain_id;
+	info->switch_info.port_id = priv->representor_id;
+	if (priv->representor) {
+		unsigned int i = mlx5_dev_to_port_id(dev->device, NULL, 0);
+		uint16_t port_id[i];
+
+		i = RTE_MIN(mlx5_dev_to_port_id(dev->device, port_id, i), i);
+		while (i--) {
+			struct priv *opriv =
+				rte_eth_devices[port_id[i]].data->dev_private;
+
+			if (!opriv ||
+			    opriv->representor ||
+			    opriv->domain_id != priv->domain_id)
+				continue;
+			/*
+			 * Override switch name with that of the master
+			 * device.
+			 */
+			info->switch_info.name = opriv->dev_data->name;
+			break;
+		}
+	}
+}
+
+/**
+ * Get supported packet types.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ *
+ * @return
+ *   A pointer to the supported Packet types array.
+ */
+const uint32_t *
+mlx5_dev_supported_ptypes_get(struct rte_eth_dev *dev)
+{
+	static const uint32_t ptypes[] = {
+		/* refers to rxq_cq_to_pkt_type() */
+		RTE_PTYPE_L2_ETHER,
+		RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
+		RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
+		RTE_PTYPE_L4_NONFRAG,
+		RTE_PTYPE_L4_FRAG,
+		RTE_PTYPE_L4_TCP,
+		RTE_PTYPE_L4_UDP,
+		RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN,
+		RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN,
+		RTE_PTYPE_INNER_L4_NONFRAG,
+		RTE_PTYPE_INNER_L4_FRAG,
+		RTE_PTYPE_INNER_L4_TCP,
+		RTE_PTYPE_INNER_L4_UDP,
+		RTE_PTYPE_UNKNOWN
+	};
+
+	if (dev->rx_pkt_burst == mlx5_rx_burst ||
+	    dev->rx_pkt_burst == mlx5_rx_burst_mprq ||
+	    dev->rx_pkt_burst == mlx5_rx_burst_vec)
+		return ptypes;
+	return NULL;
+}
+
+/**
+ * DPDK callback to retrieve physical link information.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param[out] link
+ *   Storage for current link status.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev,
+			       struct rte_eth_link *link)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct ethtool_cmd edata = {
+		.cmd = ETHTOOL_GSET /* Deprecated since Linux v4.5. */
+	};
+	struct ifreq ifr;
+	struct rte_eth_link dev_link;
+	int link_speed = 0;
+	int ret;
+
+	ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr, 1);
+	if (ret) {
+		DRV_LOG(WARNING, "port %u ioctl(SIOCGIFFLAGS) failed: %s",
+			dev->data->port_id, strerror(rte_errno));
+		return ret;
+	}
+	memset(&dev_link, 0, sizeof(dev_link));
+	dev_link.link_status = ((ifr.ifr_flags & IFF_UP) &&
+				(ifr.ifr_flags & IFF_RUNNING));
+	ifr.ifr_data = (void *)&edata;
+	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr, 1);
+	if (ret) {
+		DRV_LOG(WARNING,
+			"port %u ioctl(SIOCETHTOOL, ETHTOOL_GSET) failed: %s",
+			dev->data->port_id, strerror(rte_errno));
+		return ret;
+	}
+	link_speed = ethtool_cmd_speed(&edata);
+	if (link_speed == -1)
+		dev_link.link_speed = ETH_SPEED_NUM_NONE;
+	else
+		dev_link.link_speed = link_speed;
+	priv->link_speed_capa = 0;
+	if (edata.supported & SUPPORTED_Autoneg)
+		priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG;
+	if (edata.supported & (SUPPORTED_1000baseT_Full |
+			       SUPPORTED_1000baseKX_Full))
+		priv->link_speed_capa |= ETH_LINK_SPEED_1G;
+	if (edata.supported & SUPPORTED_10000baseKR_Full)
+		priv->link_speed_capa |= ETH_LINK_SPEED_10G;
+	if (edata.supported & (SUPPORTED_40000baseKR4_Full |
+			       SUPPORTED_40000baseCR4_Full |
+			       SUPPORTED_40000baseSR4_Full |
+			       SUPPORTED_40000baseLR4_Full))
+		priv->link_speed_capa |= ETH_LINK_SPEED_40G;
+	dev_link.link_duplex = ((edata.duplex == DUPLEX_HALF) ?
+				ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX);
+	dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds &
+			ETH_LINK_SPEED_FIXED);
+	if ((dev_link.link_speed && !dev_link.link_status) ||
+	    (!dev_link.link_speed && dev_link.link_status)) {
+		rte_errno = EAGAIN;
+		return -rte_errno;
+	}
+	*link = dev_link;
+	return 0;
+}
+
+/**
+ * Retrieve physical link information (unlocked version using new ioctl).
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param[out] link
+ *   Storage for current link status.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev,
+			     struct rte_eth_link *link)
+
+{
+	struct priv *priv = dev->data->dev_private;
+	struct ethtool_link_settings gcmd = { .cmd = ETHTOOL_GLINKSETTINGS };
+	struct ifreq ifr;
+	struct rte_eth_link dev_link;
+	uint64_t sc;
+	int ret;
+
+	ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr, 1);
+	if (ret) {
+		DRV_LOG(WARNING, "port %u ioctl(SIOCGIFFLAGS) failed: %s",
+			dev->data->port_id, strerror(rte_errno));
+		return ret;
+	}
+	memset(&dev_link, 0, sizeof(dev_link));
+	dev_link.link_status = ((ifr.ifr_flags & IFF_UP) &&
+				(ifr.ifr_flags & IFF_RUNNING));
+	ifr.ifr_data = (void *)&gcmd;
+	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr, 1);
+	if (ret) {
+		DRV_LOG(DEBUG,
+			"port %u ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS)"
+			" failed: %s",
+			dev->data->port_id, strerror(rte_errno));
+		return ret;
+	}
+	gcmd.link_mode_masks_nwords = -gcmd.link_mode_masks_nwords;
+
+	alignas(struct ethtool_link_settings)
+	uint8_t data[offsetof(struct ethtool_link_settings, link_mode_masks) +
+		     sizeof(uint32_t) * gcmd.link_mode_masks_nwords * 3];
+	struct ethtool_link_settings *ecmd = (void *)data;
+
+	*ecmd = gcmd;
+	ifr.ifr_data = (void *)ecmd;
+	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr, 1);
+	if (ret) {
+		DRV_LOG(DEBUG,
+			"port %u ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS)"
+			" failed: %s",
+			dev->data->port_id, strerror(rte_errno));
+		return ret;
+	}
+	dev_link.link_speed = ecmd->speed;
+	sc = ecmd->link_mode_masks[0] |
+		((uint64_t)ecmd->link_mode_masks[1] << 32);
+	priv->link_speed_capa = 0;
+	if (sc & MLX5_BITSHIFT(ETHTOOL_LINK_MODE_Autoneg_BIT))
+		priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG;
+	if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_1000baseT_Full_BIT) |
+		  MLX5_BITSHIFT(ETHTOOL_LINK_MODE_1000baseKX_Full_BIT)))
+		priv->link_speed_capa |= ETH_LINK_SPEED_1G;
+	if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT) |
+		  MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseKR_Full_BIT) |
+		  MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseR_FEC_BIT)))
+		priv->link_speed_capa |= ETH_LINK_SPEED_10G;
+	if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT) |
+		  MLX5_BITSHIFT(ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT)))
+		priv->link_speed_capa |= ETH_LINK_SPEED_20G;
+	if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT) |
+		  MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT) |
+		  MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT) |
+		  MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT)))
+		priv->link_speed_capa |= ETH_LINK_SPEED_40G;
+	if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT) |
+		  MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT) |
+		  MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT) |
+		  MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT)))
+		priv->link_speed_capa |= ETH_LINK_SPEED_56G;
+	if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseCR_Full_BIT) |
+		  MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseKR_Full_BIT) |
+		  MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseSR_Full_BIT)))
+		priv->link_speed_capa |= ETH_LINK_SPEED_25G;
+	if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT) |
+		  MLX5_BITSHIFT(ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT)))
+		priv->link_speed_capa |= ETH_LINK_SPEED_50G;
+	if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT) |
+		  MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT) |
+		  MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT) |
+		  MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT)))
+		priv->link_speed_capa |= ETH_LINK_SPEED_100G;
+	dev_link.link_duplex = ((ecmd->duplex == DUPLEX_HALF) ?
+				ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX);
+	dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds &
+				  ETH_LINK_SPEED_FIXED);
+	if ((dev_link.link_speed && !dev_link.link_status) ||
+	    (!dev_link.link_speed && dev_link.link_status)) {
+		rte_errno = EAGAIN;
+		return -rte_errno;
+	}
+	*link = dev_link;
+	return 0;
+}
+
+/**
+ * DPDK callback to retrieve physical link information.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param wait_to_complete
+ *   Wait for request completion.
+ *
+ * @return
+ *   0 if link status was not updated, positive if it was, a negative errno
+ *   value otherwise and rte_errno is set.
+ */
+int
+mlx5_link_update(struct rte_eth_dev *dev, int wait_to_complete)
+{
+	int ret;
+	struct rte_eth_link dev_link;
+	time_t start_time = time(NULL);
+
+	do {
+		ret = mlx5_link_update_unlocked_gs(dev, &dev_link);
+		if (ret)
+			ret = mlx5_link_update_unlocked_gset(dev, &dev_link);
+		if (ret == 0)
+			break;
+		/* Handle wait to complete situation. */
+		if (wait_to_complete && ret == -EAGAIN) {
+			if (abs((int)difftime(time(NULL), start_time)) <
+			    MLX5_LINK_STATUS_TIMEOUT) {
+				usleep(0);
+				continue;
+			} else {
+				rte_errno = EBUSY;
+				return -rte_errno;
+			}
+		} else if (ret < 0) {
+			return ret;
+		}
+	} while (wait_to_complete);
+	ret = !!memcmp(&dev->data->dev_link, &dev_link,
+		       sizeof(struct rte_eth_link));
+	dev->data->dev_link = dev_link;
+	return ret;
+}
+
+/**
+ * DPDK callback to change the MTU.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param in_mtu
+ *   New MTU.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
+{
+	struct priv *priv = dev->data->dev_private;
+	uint16_t kern_mtu = 0;
+	int ret;
+
+	ret = mlx5_get_mtu(dev, &kern_mtu);
+	if (ret)
+		return ret;
+	/* Set kernel interface MTU first. */
+	ret = mlx5_set_mtu(dev, mtu);
+	if (ret)
+		return ret;
+	ret = mlx5_get_mtu(dev, &kern_mtu);
+	if (ret)
+		return ret;
+	if (kern_mtu == mtu) {
+		priv->mtu = mtu;
+		DRV_LOG(DEBUG, "port %u adapter MTU set to %u",
+			dev->data->port_id, mtu);
+		return 0;
+	}
+	rte_errno = EAGAIN;
+	return -rte_errno;
+}
+
+/**
+ * DPDK callback to get flow control status.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param[out] fc_conf
+ *   Flow control output buffer.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
+{
+	struct ifreq ifr;
+	struct ethtool_pauseparam ethpause = {
+		.cmd = ETHTOOL_GPAUSEPARAM
+	};
+	int ret;
+
+	ifr.ifr_data = (void *)&ethpause;
+	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr, 1);
+	if (ret) {
+		DRV_LOG(WARNING,
+			"port %u ioctl(SIOCETHTOOL, ETHTOOL_GPAUSEPARAM) failed:"
+			" %s",
+			dev->data->port_id, strerror(rte_errno));
+		return ret;
+	}
+	fc_conf->autoneg = ethpause.autoneg;
+	if (ethpause.rx_pause && ethpause.tx_pause)
+		fc_conf->mode = RTE_FC_FULL;
+	else if (ethpause.rx_pause)
+		fc_conf->mode = RTE_FC_RX_PAUSE;
+	else if (ethpause.tx_pause)
+		fc_conf->mode = RTE_FC_TX_PAUSE;
+	else
+		fc_conf->mode = RTE_FC_NONE;
+	return 0;
+}
+
+/**
+ * DPDK callback to modify flow control parameters.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param[in] fc_conf
+ *   Flow control parameters.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
+{
+	struct ifreq ifr;
+	struct ethtool_pauseparam ethpause = {
+		.cmd = ETHTOOL_SPAUSEPARAM
+	};
+	int ret;
+
+	ifr.ifr_data = (void *)&ethpause;
+	ethpause.autoneg = fc_conf->autoneg;
+	if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) ||
+	    (fc_conf->mode & RTE_FC_RX_PAUSE))
+		ethpause.rx_pause = 1;
+	else
+		ethpause.rx_pause = 0;
+
+	if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) ||
+	    (fc_conf->mode & RTE_FC_TX_PAUSE))
+		ethpause.tx_pause = 1;
+	else
+		ethpause.tx_pause = 0;
+	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr, 0);
+	if (ret) {
+		DRV_LOG(WARNING,
+			"port %u ioctl(SIOCETHTOOL, ETHTOOL_SPAUSEPARAM)"
+			" failed: %s",
+			dev->data->port_id, strerror(rte_errno));
+		return ret;
+	}
+	return 0;
+}
+
+/**
+ * Get PCI information from struct ibv_device.
+ *
+ * @param device
+ *   Pointer to Ethernet device structure.
+ * @param[out] pci_addr
+ *   PCI bus address output buffer.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_ibv_device_to_pci_addr(const struct ibv_device *device,
+			    struct rte_pci_addr *pci_addr)
+{
+	FILE *file;
+	char line[32];
+	MKSTR(path, "%s/device/uevent", device->ibdev_path);
+
+	file = fopen(path, "rb");
+	if (file == NULL) {
+		rte_errno = errno;
+		return -rte_errno;
+	}
+	while (fgets(line, sizeof(line), file) == line) {
+		size_t len = strlen(line);
+		int ret;
+
+		/* Truncate long lines. */
+		if (len == (sizeof(line) - 1))
+			while (line[(len - 1)] != '\n') {
+				ret = fgetc(file);
+				if (ret == EOF)
+					break;
+				line[(len - 1)] = ret;
+			}
+		/* Extract information. */
+		if (sscanf(line,
+			   "PCI_SLOT_NAME="
+			   "%" SCNx32 ":%" SCNx8 ":%" SCNx8 ".%" SCNx8 "\n",
+			   &pci_addr->domain,
+			   &pci_addr->bus,
+			   &pci_addr->devid,
+			   &pci_addr->function) == 4) {
+			ret = 0;
+			break;
+		}
+	}
+	fclose(file);
+	return 0;
+}
+
+/**
+ * Device status handler.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param events
+ *   Pointer to event flags holder.
+ *
+ * @return
+ *   Events bitmap of callback process which can be called immediately.
+ */
+static uint32_t
+mlx5_dev_status_handler(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct ibv_async_event event;
+	uint32_t ret = 0;
+
+	if (mlx5_link_update(dev, 0) == -EAGAIN) {
+		usleep(0);
+		return 0;
+	}
+	/* Read all message and acknowledge them. */
+	for (;;) {
+		if (mlx5_glue->get_async_event(priv->ctx, &event))
+			break;
+		if ((event.event_type == IBV_EVENT_PORT_ACTIVE ||
+			event.event_type == IBV_EVENT_PORT_ERR) &&
+			(dev->data->dev_conf.intr_conf.lsc == 1))
+			ret |= (1 << RTE_ETH_EVENT_INTR_LSC);
+		else if (event.event_type == IBV_EVENT_DEVICE_FATAL &&
+			dev->data->dev_conf.intr_conf.rmv == 1)
+			ret |= (1 << RTE_ETH_EVENT_INTR_RMV);
+		else
+			DRV_LOG(DEBUG,
+				"port %u event type %d on not handled",
+				dev->data->port_id, event.event_type);
+		mlx5_glue->ack_async_event(&event);
+	}
+	return ret;
+}
+
+/**
+ * Handle interrupts from the NIC.
+ *
+ * @param[in] intr_handle
+ *   Interrupt handler.
+ * @param cb_arg
+ *   Callback argument.
+ */
+void
+mlx5_dev_interrupt_handler(void *cb_arg)
+{
+	struct rte_eth_dev *dev = cb_arg;
+	uint32_t events;
+
+	events = mlx5_dev_status_handler(dev);
+	if (events & (1 << RTE_ETH_EVENT_INTR_LSC))
+		_rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
+	if (events & (1 << RTE_ETH_EVENT_INTR_RMV))
+		_rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RMV, NULL);
+}
+
+/**
+ * Handle interrupts from the socket.
+ *
+ * @param cb_arg
+ *   Callback argument.
+ */
+static void
+mlx5_dev_handler_socket(void *cb_arg)
+{
+	struct rte_eth_dev *dev = cb_arg;
+
+	mlx5_socket_handle(dev);
+}
+
+/**
+ * Uninstall interrupt handler.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ */
+void
+mlx5_dev_interrupt_handler_uninstall(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+
+	if (dev->data->dev_conf.intr_conf.lsc ||
+	    dev->data->dev_conf.intr_conf.rmv)
+		rte_intr_callback_unregister(&priv->intr_handle,
+					     mlx5_dev_interrupt_handler, dev);
+	if (priv->primary_socket)
+		rte_intr_callback_unregister(&priv->intr_handle_socket,
+					     mlx5_dev_handler_socket, dev);
+	priv->intr_handle.fd = 0;
+	priv->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN;
+	priv->intr_handle_socket.fd = 0;
+	priv->intr_handle_socket.type = RTE_INTR_HANDLE_UNKNOWN;
+}
+
+/**
+ * Install interrupt handler.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ */
+void
+mlx5_dev_interrupt_handler_install(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	int ret;
+	int flags;
+
+	assert(priv->ctx->async_fd > 0);
+	flags = fcntl(priv->ctx->async_fd, F_GETFL);
+	ret = fcntl(priv->ctx->async_fd, F_SETFL, flags | O_NONBLOCK);
+	if (ret) {
+		DRV_LOG(INFO,
+			"port %u failed to change file descriptor async event"
+			" queue",
+			dev->data->port_id);
+		dev->data->dev_conf.intr_conf.lsc = 0;
+		dev->data->dev_conf.intr_conf.rmv = 0;
+	}
+	if (dev->data->dev_conf.intr_conf.lsc ||
+	    dev->data->dev_conf.intr_conf.rmv) {
+		priv->intr_handle.fd = priv->ctx->async_fd;
+		priv->intr_handle.type = RTE_INTR_HANDLE_EXT;
+		rte_intr_callback_register(&priv->intr_handle,
+					   mlx5_dev_interrupt_handler, dev);
+	}
+	ret = mlx5_socket_init(dev);
+	if (ret)
+		DRV_LOG(ERR, "port %u cannot initialise socket: %s",
+			dev->data->port_id, strerror(rte_errno));
+	else if (priv->primary_socket) {
+		priv->intr_handle_socket.fd = priv->primary_socket;
+		priv->intr_handle_socket.type = RTE_INTR_HANDLE_EXT;
+		rte_intr_callback_register(&priv->intr_handle_socket,
+					   mlx5_dev_handler_socket, dev);
+	}
+}
+
+/**
+ * DPDK callback to bring the link DOWN.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_set_link_down(struct rte_eth_dev *dev)
+{
+	return mlx5_set_flags(dev, ~IFF_UP, ~IFF_UP);
+}
+
+/**
+ * DPDK callback to bring the link UP.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_set_link_up(struct rte_eth_dev *dev)
+{
+	return mlx5_set_flags(dev, ~IFF_UP, IFF_UP);
+}
+
+/**
+ * Configure the TX function to use.
+ *
+ * @param dev
+ *   Pointer to private data structure.
+ *
+ * @return
+ *   Pointer to selected Tx burst function.
+ */
+eth_tx_burst_t
+mlx5_select_tx_function(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	eth_tx_burst_t tx_pkt_burst = mlx5_tx_burst;
+	struct mlx5_dev_config *config = &priv->config;
+	uint64_t tx_offloads = dev->data->dev_conf.txmode.offloads;
+	int tso = !!(tx_offloads & (DEV_TX_OFFLOAD_TCP_TSO |
+				    DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
+				    DEV_TX_OFFLOAD_GRE_TNL_TSO |
+				    DEV_TX_OFFLOAD_IP_TNL_TSO |
+				    DEV_TX_OFFLOAD_UDP_TNL_TSO));
+	int swp = !!(tx_offloads & (DEV_TX_OFFLOAD_IP_TNL_TSO |
+				    DEV_TX_OFFLOAD_UDP_TNL_TSO |
+				    DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM));
+	int vlan_insert = !!(tx_offloads & DEV_TX_OFFLOAD_VLAN_INSERT);
+
+	assert(priv != NULL);
+	/* Select appropriate TX function. */
+	if (vlan_insert || tso || swp)
+		return tx_pkt_burst;
+	if (config->mps == MLX5_MPW_ENHANCED) {
+		if (mlx5_check_vec_tx_support(dev) > 0) {
+			if (mlx5_check_raw_vec_tx_support(dev) > 0)
+				tx_pkt_burst = mlx5_tx_burst_raw_vec;
+			else
+				tx_pkt_burst = mlx5_tx_burst_vec;
+			DRV_LOG(DEBUG,
+				"port %u selected enhanced MPW Tx vectorized"
+				" function",
+				dev->data->port_id);
+		} else {
+			tx_pkt_burst = mlx5_tx_burst_empw;
+			DRV_LOG(DEBUG,
+				"port %u selected enhanced MPW Tx function",
+				dev->data->port_id);
+		}
+	} else if (config->mps && (config->txq_inline > 0)) {
+		tx_pkt_burst = mlx5_tx_burst_mpw_inline;
+		DRV_LOG(DEBUG, "port %u selected MPW inline Tx function",
+			dev->data->port_id);
+	} else if (config->mps) {
+		tx_pkt_burst = mlx5_tx_burst_mpw;
+		DRV_LOG(DEBUG, "port %u selected MPW Tx function",
+			dev->data->port_id);
+	}
+	return tx_pkt_burst;
+}
+
+/**
+ * Configure the RX function to use.
+ *
+ * @param dev
+ *   Pointer to private data structure.
+ *
+ * @return
+ *   Pointer to selected Rx burst function.
+ */
+eth_rx_burst_t
+mlx5_select_rx_function(struct rte_eth_dev *dev)
+{
+	eth_rx_burst_t rx_pkt_burst = mlx5_rx_burst;
+
+	assert(dev != NULL);
+	if (mlx5_check_vec_rx_support(dev) > 0) {
+		rx_pkt_burst = mlx5_rx_burst_vec;
+		DRV_LOG(DEBUG, "port %u selected Rx vectorized function",
+			dev->data->port_id);
+	} else if (mlx5_mprq_enabled(dev)) {
+		rx_pkt_burst = mlx5_rx_burst_mprq;
+	}
+	return rx_pkt_burst;
+}
+
+/**
+ * Check if mlx5 device was removed.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ *
+ * @return
+ *   1 when device is removed, otherwise 0.
+ */
+int
+mlx5_is_removed(struct rte_eth_dev *dev)
+{
+	struct ibv_device_attr device_attr;
+	struct priv *priv = dev->data->dev_private;
+
+	if (mlx5_glue->query_device(priv->ctx, &device_attr) == EIO)
+		return 1;
+	return 0;
+}
+
+/**
+ * Get port ID list of mlx5 instances sharing a common device.
+ *
+ * @param[in] dev
+ *   Device to look for.
+ * @param[out] port_list
+ *   Result buffer for collected port IDs.
+ * @param port_list_n
+ *   Maximum number of entries in result buffer. If 0, @p port_list can be
+ *   NULL.
+ *
+ * @return
+ *   Number of matching instances regardless of the @p port_list_n
+ *   parameter, 0 if none were found.
+ */
+unsigned int
+mlx5_dev_to_port_id(const struct rte_device *dev, uint16_t *port_list,
+		    unsigned int port_list_n)
+{
+	uint16_t id;
+	unsigned int n = 0;
+
+	RTE_ETH_FOREACH_DEV(id) {
+		struct rte_eth_dev *ldev = &rte_eth_devices[id];
+
+		if (!ldev->device ||
+		    !ldev->device->driver ||
+		    strcmp(ldev->device->driver->name, MLX5_DRIVER_NAME) ||
+		    ldev->device != dev)
+			continue;
+		if (n < port_list_n)
+			port_list[n] = id;
+		n++;
+	}
+	return n;
+}
+
+/**
+ * Get switch information associated with network interface.
+ *
+ * @param ifindex
+ *   Network interface index.
+ * @param[out] info
+ *   Switch information object, populated in case of success.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_sysfs_switch_info(unsigned int ifindex, struct mlx5_switch_info *info)
+{
+	char ifname[IF_NAMESIZE];
+	FILE *file;
+	struct mlx5_switch_info data = { .master = 0, };
+	bool port_name_set = false;
+	bool port_switch_id_set = false;
+	char c;
+
+	if (!if_indextoname(ifindex, ifname)) {
+		rte_errno = errno;
+		return -rte_errno;
+	}
+
+	MKSTR(phys_port_name, "/sys/class/net/%s/phys_port_name",
+	      ifname);
+	MKSTR(phys_switch_id, "/sys/class/net/%s/phys_switch_id",
+	      ifname);
+
+	file = fopen(phys_port_name, "rb");
+	if (file != NULL) {
+		port_name_set =
+			fscanf(file, "%d%c", &data.port_name, &c) == 2 &&
+			c == '\n';
+		fclose(file);
+	}
+	file = fopen(phys_switch_id, "rb");
+	if (file == NULL) {
+		rte_errno = errno;
+		return -rte_errno;
+	}
+	port_switch_id_set =
+		fscanf(file, "%" SCNx64 "%c", &data.switch_id, &c) == 2 &&
+		c == '\n';
+	fclose(file);
+	data.master = port_switch_id_set && !port_name_set;
+	data.representor = port_switch_id_set && port_name_set;
+	*info = data;
+	return 0;
+}
diff --git a/src/spdk/dpdk/drivers/net/mlx5/mlx5_flow.c b/src/spdk/dpdk/drivers/net/mlx5/mlx5_flow.c
new file mode 100644
index 00000000..ca4625b6
--- /dev/null
+++ b/src/spdk/dpdk/drivers/net/mlx5/mlx5_flow.c
@@ -0,0 +1,3848 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2016 6WIND S.A.
+ * Copyright 2016 Mellanox Technologies, Ltd
+ */
+
+#include <sys/queue.h>
+#include <stdalign.h>
+#include <stdint.h>
+#include <string.h>
+
+/* Verbs header. */
+/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <infiniband/verbs.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+#include <rte_common.h>
+#include <rte_ether.h>
+#include <rte_eth_ctrl.h>
+#include <rte_ethdev_driver.h>
+#include <rte_flow.h>
+#include <rte_flow_driver.h>
+#include <rte_malloc.h>
+#include <rte_ip.h>
+
+#include "mlx5.h"
+#include "mlx5_defs.h"
+#include "mlx5_prm.h"
+#include "mlx5_glue.h"
+
+/* Dev ops structure defined in mlx5.c */
+extern const struct eth_dev_ops mlx5_dev_ops;
+extern const struct eth_dev_ops mlx5_dev_ops_isolate;
+
+/* Pattern outer Layer bits. */
+#define MLX5_FLOW_LAYER_OUTER_L2 (1u << 0)
+#define MLX5_FLOW_LAYER_OUTER_L3_IPV4 (1u << 1)
+#define MLX5_FLOW_LAYER_OUTER_L3_IPV6 (1u << 2)
+#define MLX5_FLOW_LAYER_OUTER_L4_UDP (1u << 3)
+#define MLX5_FLOW_LAYER_OUTER_L4_TCP (1u << 4)
+#define MLX5_FLOW_LAYER_OUTER_VLAN (1u << 5)
+
+/* Pattern inner Layer bits. */
+#define MLX5_FLOW_LAYER_INNER_L2 (1u << 6)
+#define MLX5_FLOW_LAYER_INNER_L3_IPV4 (1u << 7)
+#define MLX5_FLOW_LAYER_INNER_L3_IPV6 (1u << 8)
+#define MLX5_FLOW_LAYER_INNER_L4_UDP (1u << 9)
+#define MLX5_FLOW_LAYER_INNER_L4_TCP (1u << 10)
+#define MLX5_FLOW_LAYER_INNER_VLAN (1u << 11)
+
+/* Pattern tunnel Layer bits. */
+#define MLX5_FLOW_LAYER_VXLAN (1u << 12)
+#define MLX5_FLOW_LAYER_VXLAN_GPE (1u << 13)
+#define MLX5_FLOW_LAYER_GRE (1u << 14)
+#define MLX5_FLOW_LAYER_MPLS (1u << 15)
+
+/* Outer Masks. */
+#define MLX5_FLOW_LAYER_OUTER_L3 \
+	(MLX5_FLOW_LAYER_OUTER_L3_IPV4 | MLX5_FLOW_LAYER_OUTER_L3_IPV6)
+#define MLX5_FLOW_LAYER_OUTER_L4 \
+	(MLX5_FLOW_LAYER_OUTER_L4_UDP | MLX5_FLOW_LAYER_OUTER_L4_TCP)
+#define MLX5_FLOW_LAYER_OUTER \
+	(MLX5_FLOW_LAYER_OUTER_L2 | MLX5_FLOW_LAYER_OUTER_L3 | \
+	 MLX5_FLOW_LAYER_OUTER_L4)
+
+/* Tunnel Masks. */
+#define MLX5_FLOW_LAYER_TUNNEL \
+	(MLX5_FLOW_LAYER_VXLAN | MLX5_FLOW_LAYER_VXLAN_GPE | \
+	 MLX5_FLOW_LAYER_GRE | MLX5_FLOW_LAYER_MPLS)
+
+/* Inner Masks. */
+#define MLX5_FLOW_LAYER_INNER_L3 \
+	(MLX5_FLOW_LAYER_INNER_L3_IPV4 | MLX5_FLOW_LAYER_INNER_L3_IPV6)
+#define MLX5_FLOW_LAYER_INNER_L4 \
+	(MLX5_FLOW_LAYER_INNER_L4_UDP | MLX5_FLOW_LAYER_INNER_L4_TCP)
+#define MLX5_FLOW_LAYER_INNER \
+	(MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3 | \
+	 MLX5_FLOW_LAYER_INNER_L4)
+
+/* Actions that modify the fate of matching traffic. */
+#define MLX5_FLOW_FATE_DROP (1u << 0)
+#define MLX5_FLOW_FATE_QUEUE (1u << 1)
+#define MLX5_FLOW_FATE_RSS (1u << 2)
+
+/* Modify a packet. */
+#define MLX5_FLOW_MOD_FLAG (1u << 0)
+#define MLX5_FLOW_MOD_MARK (1u << 1)
+#define MLX5_FLOW_MOD_COUNT (1u << 2)
+
+/* possible L3 layers protocols filtering. */
+#define MLX5_IP_PROTOCOL_TCP 6
+#define MLX5_IP_PROTOCOL_UDP 17
+#define MLX5_IP_PROTOCOL_GRE 47
+#define MLX5_IP_PROTOCOL_MPLS 147
+
+/* Priority reserved for default flows. */
+#define MLX5_FLOW_PRIO_RSVD ((uint32_t)-1)
+
+enum mlx5_expansion {
+	MLX5_EXPANSION_ROOT,
+	MLX5_EXPANSION_ROOT_OUTER,
+	MLX5_EXPANSION_ROOT_ETH_VLAN,
+	MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN,
+	MLX5_EXPANSION_OUTER_ETH,
+	MLX5_EXPANSION_OUTER_ETH_VLAN,
+	MLX5_EXPANSION_OUTER_VLAN,
+	MLX5_EXPANSION_OUTER_IPV4,
+	MLX5_EXPANSION_OUTER_IPV4_UDP,
+	MLX5_EXPANSION_OUTER_IPV4_TCP,
+	MLX5_EXPANSION_OUTER_IPV6,
+	MLX5_EXPANSION_OUTER_IPV6_UDP,
+	MLX5_EXPANSION_OUTER_IPV6_TCP,
+	MLX5_EXPANSION_VXLAN,
+	MLX5_EXPANSION_VXLAN_GPE,
+	MLX5_EXPANSION_GRE,
+	MLX5_EXPANSION_MPLS,
+	MLX5_EXPANSION_ETH,
+	MLX5_EXPANSION_ETH_VLAN,
+	MLX5_EXPANSION_VLAN,
+	MLX5_EXPANSION_IPV4,
+	MLX5_EXPANSION_IPV4_UDP,
+	MLX5_EXPANSION_IPV4_TCP,
+	MLX5_EXPANSION_IPV6,
+	MLX5_EXPANSION_IPV6_UDP,
+	MLX5_EXPANSION_IPV6_TCP,
+};
+
+/** Supported expansion of items. */
+static const struct rte_flow_expand_node mlx5_support_expansion[] = {
+	[MLX5_EXPANSION_ROOT] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
+						 MLX5_EXPANSION_IPV4,
+						 MLX5_EXPANSION_IPV6),
+		.type = RTE_FLOW_ITEM_TYPE_END,
+	},
+	[MLX5_EXPANSION_ROOT_OUTER] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH,
+						 MLX5_EXPANSION_OUTER_IPV4,
+						 MLX5_EXPANSION_OUTER_IPV6),
+		.type = RTE_FLOW_ITEM_TYPE_END,
+	},
+	[MLX5_EXPANSION_ROOT_ETH_VLAN] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN),
+		.type = RTE_FLOW_ITEM_TYPE_END,
+	},
+	[MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH_VLAN),
+		.type = RTE_FLOW_ITEM_TYPE_END,
+	},
+	[MLX5_EXPANSION_OUTER_ETH] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
+						 MLX5_EXPANSION_OUTER_IPV6,
+						 MLX5_EXPANSION_MPLS),
+		.type = RTE_FLOW_ITEM_TYPE_ETH,
+		.rss_types = 0,
+	},
+	[MLX5_EXPANSION_OUTER_ETH_VLAN] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN),
+		.type = RTE_FLOW_ITEM_TYPE_ETH,
+		.rss_types = 0,
+	},
+	[MLX5_EXPANSION_OUTER_VLAN] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
+						 MLX5_EXPANSION_OUTER_IPV6),
+		.type = RTE_FLOW_ITEM_TYPE_VLAN,
+	},
+	[MLX5_EXPANSION_OUTER_IPV4] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT
+			(MLX5_EXPANSION_OUTER_IPV4_UDP,
+			 MLX5_EXPANSION_OUTER_IPV4_TCP,
+			 MLX5_EXPANSION_GRE),
+		.type = RTE_FLOW_ITEM_TYPE_IPV4,
+		.rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
+			ETH_RSS_NONFRAG_IPV4_OTHER,
+	},
+	[MLX5_EXPANSION_OUTER_IPV4_UDP] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
+						 MLX5_EXPANSION_VXLAN_GPE),
+		.type = RTE_FLOW_ITEM_TYPE_UDP,
+		.rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
+	},
+	[MLX5_EXPANSION_OUTER_IPV4_TCP] = {
+		.type = RTE_FLOW_ITEM_TYPE_TCP,
+		.rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
+	},
+	[MLX5_EXPANSION_OUTER_IPV6] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT
+			(MLX5_EXPANSION_OUTER_IPV6_UDP,
+			 MLX5_EXPANSION_OUTER_IPV6_TCP),
+		.type = RTE_FLOW_ITEM_TYPE_IPV6,
+		.rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
+			ETH_RSS_NONFRAG_IPV6_OTHER,
+	},
+	[MLX5_EXPANSION_OUTER_IPV6_UDP] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
+						 MLX5_EXPANSION_VXLAN_GPE),
+		.type = RTE_FLOW_ITEM_TYPE_UDP,
+		.rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
+	},
+	[MLX5_EXPANSION_OUTER_IPV6_TCP] = {
+		.type = RTE_FLOW_ITEM_TYPE_TCP,
+		.rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
+	},
+	[MLX5_EXPANSION_VXLAN] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH),
+		.type = RTE_FLOW_ITEM_TYPE_VXLAN,
+	},
+	[MLX5_EXPANSION_VXLAN_GPE] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
+						 MLX5_EXPANSION_IPV4,
+						 MLX5_EXPANSION_IPV6),
+		.type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
+	},
+	[MLX5_EXPANSION_GRE] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4),
+		.type = RTE_FLOW_ITEM_TYPE_GRE,
+	},
+	[MLX5_EXPANSION_MPLS] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
+						 MLX5_EXPANSION_IPV6),
+		.type = RTE_FLOW_ITEM_TYPE_MPLS,
+	},
+	[MLX5_EXPANSION_ETH] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
+						 MLX5_EXPANSION_IPV6),
+		.type = RTE_FLOW_ITEM_TYPE_ETH,
+	},
+	[MLX5_EXPANSION_ETH_VLAN] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN),
+		.type = RTE_FLOW_ITEM_TYPE_ETH,
+	},
+	[MLX5_EXPANSION_VLAN] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
+						 MLX5_EXPANSION_IPV6),
+		.type = RTE_FLOW_ITEM_TYPE_VLAN,
+	},
+	[MLX5_EXPANSION_IPV4] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP,
+						 MLX5_EXPANSION_IPV4_TCP),
+		.type = RTE_FLOW_ITEM_TYPE_IPV4,
+		.rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
+			ETH_RSS_NONFRAG_IPV4_OTHER,
+	},
+	[MLX5_EXPANSION_IPV4_UDP] = {
+		.type = RTE_FLOW_ITEM_TYPE_UDP,
+		.rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
+	},
+	[MLX5_EXPANSION_IPV4_TCP] = {
+		.type = RTE_FLOW_ITEM_TYPE_TCP,
+		.rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
+	},
+	[MLX5_EXPANSION_IPV6] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP,
+						 MLX5_EXPANSION_IPV6_TCP),
+		.type = RTE_FLOW_ITEM_TYPE_IPV6,
+		.rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
+			ETH_RSS_NONFRAG_IPV6_OTHER,
+	},
+	[MLX5_EXPANSION_IPV6_UDP] = {
+		.type = RTE_FLOW_ITEM_TYPE_UDP,
+		.rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
+	},
+	[MLX5_EXPANSION_IPV6_TCP] = {
+		.type = RTE_FLOW_ITEM_TYPE_TCP,
+		.rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
+	},
+};
+
+/** Handles information leading to a drop fate. */
+struct mlx5_flow_verbs {
+	LIST_ENTRY(mlx5_flow_verbs) next;
+	unsigned int size; /**< Size of the attribute. */
+	struct {
+		struct ibv_flow_attr *attr;
+		/**< Pointer to the Specification buffer. */
+		uint8_t *specs; /**< Pointer to the specifications. */
+	};
+	struct ibv_flow *flow; /**< Verbs flow pointer. */
+	struct mlx5_hrxq *hrxq; /**< Hash Rx queue object. */
+	uint64_t hash_fields; /**< Verbs hash Rx queue hash fields. */
+};
+
+/* Counters information. */
+struct mlx5_flow_counter {
+	LIST_ENTRY(mlx5_flow_counter) next; /**< Pointer to the next counter. */
+	uint32_t shared:1; /**< Share counter ID with other flow rules. */
+	uint32_t ref_cnt:31; /**< Reference counter. */
+	uint32_t id; /**< Counter ID. */
+	struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
+	uint64_t hits; /**< Number of packets matched by the rule. */
+	uint64_t bytes; /**< Number of bytes matched by the rule. */
+};
+
+/* Flow structure. */
+struct rte_flow {
+	TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
+	struct rte_flow_attr attributes; /**< User flow attribute. */
+	uint32_t l3_protocol_en:1; /**< Protocol filtering requested. */
+	uint32_t layers;
+	/**< Bit-fields of present layers see MLX5_FLOW_LAYER_*. */
+	uint32_t modifier;
+	/**< Bit-fields of present modifier see MLX5_FLOW_MOD_*. */
+	uint32_t fate;
+	/**< Bit-fields of present fate see MLX5_FLOW_FATE_*. */
+	uint8_t l3_protocol; /**< valid when l3_protocol_en is set. */
+	LIST_HEAD(verbs, mlx5_flow_verbs) verbs; /**< Verbs flows list. */
+	struct mlx5_flow_verbs *cur_verbs;
+	/**< Current Verbs flow structure being filled. */
+	struct mlx5_flow_counter *counter; /**< Holds Verbs flow counter. */
+	struct rte_flow_action_rss rss;/**< RSS context. */
+	uint8_t key[MLX5_RSS_HASH_KEY_LEN]; /**< RSS hash key. */
+	uint16_t (*queue)[]; /**< Destination queues to redirect traffic to. */
+	void *nl_flow; /**< Netlink flow buffer if relevant. */
+};
+
+static const struct rte_flow_ops mlx5_flow_ops = {
+	.validate = mlx5_flow_validate,
+	.create = mlx5_flow_create,
+	.destroy = mlx5_flow_destroy,
+	.flush = mlx5_flow_flush,
+	.isolate = mlx5_flow_isolate,
+	.query = mlx5_flow_query,
+};
+
+/* Convert FDIR request to Generic flow. */
+struct mlx5_fdir {
+	struct rte_flow_attr attr;
+	struct rte_flow_action actions[2];
+	struct rte_flow_item items[4];
+	struct rte_flow_item_eth l2;
+	struct rte_flow_item_eth l2_mask;
+	union {
+		struct rte_flow_item_ipv4 ipv4;
+		struct rte_flow_item_ipv6 ipv6;
+	} l3;
+	union {
+		struct rte_flow_item_ipv4 ipv4;
+		struct rte_flow_item_ipv6 ipv6;
+	} l3_mask;
+	union {
+		struct rte_flow_item_udp udp;
+		struct rte_flow_item_tcp tcp;
+	} l4;
+	union {
+		struct rte_flow_item_udp udp;
+		struct rte_flow_item_tcp tcp;
+	} l4_mask;
+	struct rte_flow_action_queue queue;
+};
+
+/* Verbs specification header. */
+struct ibv_spec_header {
+	enum ibv_flow_spec_type type;
+	uint16_t size;
+};
+
+/*
+ * Number of sub priorities.
+ * For each kind of pattern matching i.e. L2, L3, L4 to have a correct
+ * matching on the NIC (firmware dependent) L4 most have the higher priority
+ * followed by L3 and ending with L2.
+ */
+#define MLX5_PRIORITY_MAP_L2 2
+#define MLX5_PRIORITY_MAP_L3 1
+#define MLX5_PRIORITY_MAP_L4 0
+#define MLX5_PRIORITY_MAP_MAX 3
+
+/* Map of Verbs to Flow priority with 8 Verbs priorities. */
+static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = {
+	{ 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 },
+};
+
+/* Map of Verbs to Flow priority with 16 Verbs priorities. */
+static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = {
+	{ 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 },
+	{ 9, 10, 11 }, { 12, 13, 14 },
+};
+
+/* Tunnel information. */
+struct mlx5_flow_tunnel_info {
+	uint32_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */
+	uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */
+};
+
+static struct mlx5_flow_tunnel_info tunnels_info[] = {
+	{
+		.tunnel = MLX5_FLOW_LAYER_VXLAN,
+		.ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP,
+	},
+	{
+		.tunnel = MLX5_FLOW_LAYER_VXLAN_GPE,
+		.ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP,
+	},
+	{
+		.tunnel = MLX5_FLOW_LAYER_GRE,
+		.ptype = RTE_PTYPE_TUNNEL_GRE,
+	},
+	{
+		.tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP,
+		.ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE | RTE_PTYPE_L4_UDP,
+	},
+	{
+		.tunnel = MLX5_FLOW_LAYER_MPLS,
+		.ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
+	},
+};
+
+/**
+ * Discover the maximum number of priority available.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   number of supported flow priority on success, a negative errno
+ *   value otherwise and rte_errno is set.
+ */
+int
+mlx5_flow_discover_priorities(struct rte_eth_dev *dev)
+{
+	struct {
+		struct ibv_flow_attr attr;
+		struct ibv_flow_spec_eth eth;
+		struct ibv_flow_spec_action_drop drop;
+	} flow_attr = {
+		.attr = {
+			.num_of_specs = 2,
+		},
+		.eth = {
+			.type = IBV_FLOW_SPEC_ETH,
+			.size = sizeof(struct ibv_flow_spec_eth),
+		},
+		.drop = {
+			.size = sizeof(struct ibv_flow_spec_action_drop),
+			.type = IBV_FLOW_SPEC_ACTION_DROP,
+		},
+	};
+	struct ibv_flow *flow;
+	struct mlx5_hrxq *drop = mlx5_hrxq_drop_new(dev);
+	uint16_t vprio[] = { 8, 16 };
+	int i;
+	int priority = 0;
+
+	if (!drop) {
+		rte_errno = ENOTSUP;
+		return -rte_errno;
+	}
+	for (i = 0; i != RTE_DIM(vprio); i++) {
+		flow_attr.attr.priority = vprio[i] - 1;
+		flow = mlx5_glue->create_flow(drop->qp, &flow_attr.attr);
+		if (!flow)
+			break;
+		claim_zero(mlx5_glue->destroy_flow(flow));
+		priority = vprio[i];
+	}
+	switch (priority) {
+	case 8:
+		priority = RTE_DIM(priority_map_3);
+		break;
+	case 16:
+		priority = RTE_DIM(priority_map_5);
+		break;
+	default:
+		rte_errno = ENOTSUP;
+		DRV_LOG(ERR,
+			"port %u verbs maximum priority: %d expected 8/16",
+			dev->data->port_id, vprio[i]);
+		return -rte_errno;
+	}
+	mlx5_hrxq_drop_release(dev);
+	DRV_LOG(INFO, "port %u flow maximum priority: %d",
+		dev->data->port_id, priority);
+	return priority;
+}
+
+/**
+ * Adjust flow priority.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param flow
+ *   Pointer to an rte flow.
+ */
+static void
+mlx5_flow_adjust_priority(struct rte_eth_dev *dev, struct rte_flow *flow)
+{
+	struct priv *priv = dev->data->dev_private;
+	uint32_t priority = flow->attributes.priority;
+	uint32_t subpriority = flow->cur_verbs->attr->priority;
+
+	switch (priv->config.flow_prio) {
+	case RTE_DIM(priority_map_3):
+		priority = priority_map_3[priority][subpriority];
+		break;
+	case RTE_DIM(priority_map_5):
+		priority = priority_map_5[priority][subpriority];
+		break;
+	}
+	flow->cur_verbs->attr->priority = priority;
+}
+
+/**
+ * Get a flow counter.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ * @param[in] shared
+ *   Indicate if this counter is shared with other flows.
+ * @param[in] id
+ *   Counter identifier.
+ *
+ * @return
+ *   A pointer to the counter, NULL otherwise and rte_errno is set.
+ */
+static struct mlx5_flow_counter *
+mlx5_flow_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_flow_counter *cnt;
+
+	LIST_FOREACH(cnt, &priv->flow_counters, next) {
+		if (!cnt->shared || cnt->shared != shared)
+			continue;
+		if (cnt->id != id)
+			continue;
+		cnt->ref_cnt++;
+		return cnt;
+	}
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+
+	struct mlx5_flow_counter tmpl = {
+		.shared = shared,
+		.id = id,
+		.cs = mlx5_glue->create_counter_set
+			(priv->ctx,
+			 &(struct ibv_counter_set_init_attr){
+				 .counter_set_id = id,
+			 }),
+		.hits = 0,
+		.bytes = 0,
+	};
+
+	if (!tmpl.cs) {
+		rte_errno = errno;
+		return NULL;
+	}
+	cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
+	if (!cnt) {
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+	*cnt = tmpl;
+	LIST_INSERT_HEAD(&priv->flow_counters, cnt, next);
+	return cnt;
+#endif
+	rte_errno = ENOTSUP;
+	return NULL;
+}
+
+/**
+ * Release a flow counter.
+ *
+ * @param[in] counter
+ *   Pointer to the counter handler.
+ */
+static void
+mlx5_flow_counter_release(struct mlx5_flow_counter *counter)
+{
+	if (--counter->ref_cnt == 0) {
+		claim_zero(mlx5_glue->destroy_counter_set(counter->cs));
+		LIST_REMOVE(counter, next);
+		rte_free(counter);
+	}
+}
+
+/**
+ * Verify the @p attributes will be correctly understood by the NIC and store
+ * them in the @p flow if everything is correct.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ * @param[in] attributes
+ *   Pointer to flow attributes
+ * @param[in, out] flow
+ *   Pointer to the rte_flow structure.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_attributes(struct rte_eth_dev *dev,
+		     const struct rte_flow_attr *attributes,
+		     struct rte_flow *flow,
+		     struct rte_flow_error *error)
+{
+	uint32_t priority_max =
+		((struct priv *)dev->data->dev_private)->config.flow_prio - 1;
+
+	if (attributes->group)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
+					  NULL,
+					  "groups is not supported");
+	if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
+	    attributes->priority >= priority_max)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
+					  NULL,
+					  "priority out of range");
+	if (attributes->egress)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
+					  NULL,
+					  "egress is not supported");
+	if (attributes->transfer)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
+					  NULL,
+					  "transfer is not supported");
+	if (!attributes->ingress)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
+					  NULL,
+					  "ingress attribute is mandatory");
+	flow->attributes = *attributes;
+	if (attributes->priority == MLX5_FLOW_PRIO_RSVD)
+		flow->attributes.priority = priority_max;
+	return 0;
+}
+
+/**
+ * Verify the @p item specifications (spec, last, mask) are compatible with the
+ * NIC capabilities.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in] mask
+ *   @p item->mask or flow default bit-masks.
+ * @param[in] nic_mask
+ *   Bit-masks covering supported fields by the NIC to compare with user mask.
+ * @param[in] size
+ *   Bit-masks size in bytes.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_item_acceptable(const struct rte_flow_item *item,
+			  const uint8_t *mask,
+			  const uint8_t *nic_mask,
+			  unsigned int size,
+			  struct rte_flow_error *error)
+{
+	unsigned int i;
+
+	assert(nic_mask);
+	for (i = 0; i < size; ++i)
+		if ((nic_mask[i] | mask[i]) != nic_mask[i])
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  item,
+						  "mask enables non supported"
+						  " bits");
+	if (!item->spec && (item->mask || item->last))
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "mask/last without a spec is not"
+					  " supported");
+	if (item->spec && item->last) {
+		uint8_t spec[size];
+		uint8_t last[size];
+		unsigned int i;
+		int ret;
+
+		for (i = 0; i < size; ++i) {
+			spec[i] = ((const uint8_t *)item->spec)[i] & mask[i];
+			last[i] = ((const uint8_t *)item->last)[i] & mask[i];
+		}
+		ret = memcmp(spec, last, size);
+		if (ret != 0)
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  item,
+						  "range is not supported");
+	}
+	return 0;
+}
+
+/**
+ * Add a verbs item specification into @p flow.
+ *
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] src
+ *   Create specification.
+ * @param[in] size
+ *   Size in bytes of the specification to copy.
+ */
+static void
+mlx5_flow_spec_verbs_add(struct rte_flow *flow, void *src, unsigned int size)
+{
+	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
+
+	if (verbs->specs) {
+		void *dst;
+
+		dst = (void *)(verbs->specs + verbs->size);
+		memcpy(dst, src, size);
+		++verbs->attr->num_of_specs;
+	}
+	verbs->size += size;
+}
+
+/**
+ * Adjust verbs hash fields according to the @p flow information.
+ *
+ * @param[in, out] flow.
+ *   Pointer to flow structure.
+ * @param[in] tunnel
+ *   1 when the hash field is for a tunnel item.
+ * @param[in] layer_types
+ *   ETH_RSS_* types.
+ * @param[in] hash_fields
+ *   Item hash fields.
+ */
+static void
+mlx5_flow_verbs_hashfields_adjust(struct rte_flow *flow,
+				  int tunnel __rte_unused,
+				  uint32_t layer_types, uint64_t hash_fields)
+{
+#ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
+	hash_fields |= (tunnel ? IBV_RX_HASH_INNER : 0);
+	if (flow->rss.level == 2 && !tunnel)
+		hash_fields = 0;
+	else if (flow->rss.level < 2 && tunnel)
+		hash_fields = 0;
+#endif
+	if (!(flow->rss.types & layer_types))
+		hash_fields = 0;
+	flow->cur_verbs->hash_fields |= hash_fields;
+}
+
+/**
+ * Convert the @p item into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_item_eth(const struct rte_flow_item *item, struct rte_flow *flow,
+		   const size_t flow_size, struct rte_flow_error *error)
+{
+	const struct rte_flow_item_eth *spec = item->spec;
+	const struct rte_flow_item_eth *mask = item->mask;
+	const struct rte_flow_item_eth nic_mask = {
+		.dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
+		.src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
+		.type = RTE_BE16(0xffff),
+	};
+	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
+	const unsigned int size = sizeof(struct ibv_flow_spec_eth);
+	struct ibv_flow_spec_eth eth = {
+		.type = IBV_FLOW_SPEC_ETH | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
+		.size = size,
+	};
+	int ret;
+
+	if (flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
+			    MLX5_FLOW_LAYER_OUTER_L2))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L2 layers already configured");
+	if (!mask)
+		mask = &rte_flow_item_eth_mask;
+	ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
+					(const uint8_t *)&nic_mask,
+					sizeof(struct rte_flow_item_eth),
+					error);
+	if (ret)
+		return ret;
+	flow->layers |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
+		MLX5_FLOW_LAYER_OUTER_L2;
+	if (size > flow_size)
+		return size;
+	if (spec) {
+		unsigned int i;
+
+		memcpy(&eth.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
+		memcpy(&eth.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
+		eth.val.ether_type = spec->type;
+		memcpy(&eth.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
+		memcpy(&eth.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
+		eth.mask.ether_type = mask->type;
+		/* Remove unwanted bits from values. */
+		for (i = 0; i < ETHER_ADDR_LEN; ++i) {
+			eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
+			eth.val.src_mac[i] &= eth.mask.src_mac[i];
+		}
+		eth.val.ether_type &= eth.mask.ether_type;
+	}
+	flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L2;
+	mlx5_flow_spec_verbs_add(flow, &eth, size);
+	return size;
+}
+
+/**
+ * Update the VLAN tag in the Verbs Ethernet specification.
+ *
+ * @param[in, out] attr
+ *   Pointer to Verbs attributes structure.
+ * @param[in] eth
+ *   Verbs structure containing the VLAN information to copy.
+ */
+static void
+mlx5_flow_item_vlan_update(struct ibv_flow_attr *attr,
+			   struct ibv_flow_spec_eth *eth)
+{
+	unsigned int i;
+	const enum ibv_flow_spec_type search = eth->type;
+	struct ibv_spec_header *hdr = (struct ibv_spec_header *)
+		((uint8_t *)attr + sizeof(struct ibv_flow_attr));
+
+	for (i = 0; i != attr->num_of_specs; ++i) {
+		if (hdr->type == search) {
+			struct ibv_flow_spec_eth *e =
+				(struct ibv_flow_spec_eth *)hdr;
+
+			e->val.vlan_tag = eth->val.vlan_tag;
+			e->mask.vlan_tag = eth->mask.vlan_tag;
+			e->val.ether_type = eth->val.ether_type;
+			e->mask.ether_type = eth->mask.ether_type;
+			break;
+		}
+		hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
+	}
+}
+
+/**
+ * Convert the @p item into @p flow (or by updating the already present
+ * Ethernet Verbs) specification after ensuring the NIC will understand and
+ * process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_item_vlan(const struct rte_flow_item *item, struct rte_flow *flow,
+		    const size_t flow_size, struct rte_flow_error *error)
+{
+	const struct rte_flow_item_vlan *spec = item->spec;
+	const struct rte_flow_item_vlan *mask = item->mask;
+	const struct rte_flow_item_vlan nic_mask = {
+		.tci = RTE_BE16(0x0fff),
+		.inner_type = RTE_BE16(0xffff),
+	};
+	unsigned int size = sizeof(struct ibv_flow_spec_eth);
+	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
+	struct ibv_flow_spec_eth eth = {
+		.type = IBV_FLOW_SPEC_ETH | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
+		.size = size,
+	};
+	int ret;
+	const uint32_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
+					MLX5_FLOW_LAYER_INNER_L4) :
+		(MLX5_FLOW_LAYER_OUTER_L3 | MLX5_FLOW_LAYER_OUTER_L4);
+	const uint32_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
+		MLX5_FLOW_LAYER_OUTER_VLAN;
+	const uint32_t l2m = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
+		MLX5_FLOW_LAYER_OUTER_L2;
+
+	if (flow->layers & vlanm)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "VLAN layer already configured");
+	else if ((flow->layers & l34m) != 0)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L2 layer cannot follow L3/L4 layer");
+	if (!mask)
+		mask = &rte_flow_item_vlan_mask;
+	ret = mlx5_flow_item_acceptable
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&nic_mask,
+		 sizeof(struct rte_flow_item_vlan), error);
+	if (ret)
+		return ret;
+	if (spec) {
+		eth.val.vlan_tag = spec->tci;
+		eth.mask.vlan_tag = mask->tci;
+		eth.val.vlan_tag &= eth.mask.vlan_tag;
+		eth.val.ether_type = spec->inner_type;
+		eth.mask.ether_type = mask->inner_type;
+		eth.val.ether_type &= eth.mask.ether_type;
+	}
+	/*
+	 * From verbs perspective an empty VLAN is equivalent
+	 * to a packet without VLAN layer.
+	 */
+	if (!eth.mask.vlan_tag)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
+					  item->spec,
+					  "VLAN cannot be empty");
+	if (!(flow->layers & l2m)) {
+		if (size <= flow_size) {
+			flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L2;
+			mlx5_flow_spec_verbs_add(flow, &eth, size);
+		}
+	} else {
+		if (flow->cur_verbs)
+			mlx5_flow_item_vlan_update(flow->cur_verbs->attr,
+						   &eth);
+		size = 0; /* Only an update is done in eth specification. */
+	}
+	flow->layers |= tunnel ?
+		(MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_VLAN) :
+		(MLX5_FLOW_LAYER_OUTER_L2 | MLX5_FLOW_LAYER_OUTER_VLAN);
+	return size;
+}
+
+/**
+ * Convert the @p item into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_item_ipv4(const struct rte_flow_item *item, struct rte_flow *flow,
+		    const size_t flow_size, struct rte_flow_error *error)
+{
+	const struct rte_flow_item_ipv4 *spec = item->spec;
+	const struct rte_flow_item_ipv4 *mask = item->mask;
+	const struct rte_flow_item_ipv4 nic_mask = {
+		.hdr = {
+			.src_addr = RTE_BE32(0xffffffff),
+			.dst_addr = RTE_BE32(0xffffffff),
+			.type_of_service = 0xff,
+			.next_proto_id = 0xff,
+		},
+	};
+	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
+	unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext);
+	struct ibv_flow_spec_ipv4_ext ipv4 = {
+		.type = IBV_FLOW_SPEC_IPV4_EXT |
+			(tunnel ? IBV_FLOW_SPEC_INNER : 0),
+		.size = size,
+	};
+	int ret;
+
+	if (flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
+			    MLX5_FLOW_LAYER_OUTER_L3))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "multiple L3 layers not supported");
+	else if (flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
+				 MLX5_FLOW_LAYER_OUTER_L4))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L3 cannot follow an L4 layer.");
+	if (!mask)
+		mask = &rte_flow_item_ipv4_mask;
+	ret = mlx5_flow_item_acceptable
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&nic_mask,
+		 sizeof(struct rte_flow_item_ipv4), error);
+	if (ret < 0)
+		return ret;
+	flow->layers |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
+		MLX5_FLOW_LAYER_OUTER_L3_IPV4;
+	if (spec) {
+		ipv4.val = (struct ibv_flow_ipv4_ext_filter){
+			.src_ip = spec->hdr.src_addr,
+			.dst_ip = spec->hdr.dst_addr,
+			.proto = spec->hdr.next_proto_id,
+			.tos = spec->hdr.type_of_service,
+		};
+		ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
+			.src_ip = mask->hdr.src_addr,
+			.dst_ip = mask->hdr.dst_addr,
+			.proto = mask->hdr.next_proto_id,
+			.tos = mask->hdr.type_of_service,
+		};
+		/* Remove unwanted bits from values. */
+		ipv4.val.src_ip &= ipv4.mask.src_ip;
+		ipv4.val.dst_ip &= ipv4.mask.dst_ip;
+		ipv4.val.proto &= ipv4.mask.proto;
+		ipv4.val.tos &= ipv4.mask.tos;
+	}
+	flow->l3_protocol_en = !!ipv4.mask.proto;
+	flow->l3_protocol = ipv4.val.proto;
+	if (size <= flow_size) {
+		mlx5_flow_verbs_hashfields_adjust
+			(flow, tunnel,
+			 (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
+			  ETH_RSS_NONFRAG_IPV4_OTHER),
+			 (IBV_RX_HASH_SRC_IPV4 | IBV_RX_HASH_DST_IPV4));
+		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L3;
+		mlx5_flow_spec_verbs_add(flow, &ipv4, size);
+	}
+	return size;
+}
+
+/**
+ * Convert the @p item into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_item_ipv6(const struct rte_flow_item *item, struct rte_flow *flow,
+		    const size_t flow_size, struct rte_flow_error *error)
+{
+	const struct rte_flow_item_ipv6 *spec = item->spec;
+	const struct rte_flow_item_ipv6 *mask = item->mask;
+	const struct rte_flow_item_ipv6 nic_mask = {
+		.hdr = {
+			.src_addr =
+				"\xff\xff\xff\xff\xff\xff\xff\xff"
+				"\xff\xff\xff\xff\xff\xff\xff\xff",
+			.dst_addr =
+				"\xff\xff\xff\xff\xff\xff\xff\xff"
+				"\xff\xff\xff\xff\xff\xff\xff\xff",
+			.vtc_flow = RTE_BE32(0xffffffff),
+			.proto = 0xff,
+			.hop_limits = 0xff,
+		},
+	};
+	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
+	unsigned int size = sizeof(struct ibv_flow_spec_ipv6);
+	struct ibv_flow_spec_ipv6 ipv6 = {
+		.type = IBV_FLOW_SPEC_IPV6 | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
+		.size = size,
+	};
+	int ret;
+
+	if (flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
+			    MLX5_FLOW_LAYER_OUTER_L3))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "multiple L3 layers not supported");
+	else if (flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
+				 MLX5_FLOW_LAYER_OUTER_L4))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L3 cannot follow an L4 layer.");
+	/*
+	 * IPv6 is not recognised by the NIC inside a GRE tunnel.
+	 * Such support has to be disabled as the rule will be
+	 * accepted.  Issue reproduced with Mellanox OFED 4.3-3.0.2.1 and
+	 * Mellanox OFED 4.4-1.0.0.0.
+	 */
+	if (tunnel && flow->layers & MLX5_FLOW_LAYER_GRE)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "IPv6 inside a GRE tunnel is"
+					  " not recognised.");
+	if (!mask)
+		mask = &rte_flow_item_ipv6_mask;
+	ret = mlx5_flow_item_acceptable
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&nic_mask,
+		 sizeof(struct rte_flow_item_ipv6), error);
+	if (ret < 0)
+		return ret;
+	flow->layers |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
+		MLX5_FLOW_LAYER_OUTER_L3_IPV6;
+	if (spec) {
+		unsigned int i;
+		uint32_t vtc_flow_val;
+		uint32_t vtc_flow_mask;
+
+		memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
+		       RTE_DIM(ipv6.val.src_ip));
+		memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
+		       RTE_DIM(ipv6.val.dst_ip));
+		memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
+		       RTE_DIM(ipv6.mask.src_ip));
+		memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
+		       RTE_DIM(ipv6.mask.dst_ip));
+		vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
+		vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
+		ipv6.val.flow_label =
+			rte_cpu_to_be_32((vtc_flow_val & IPV6_HDR_FL_MASK) >>
+					 IPV6_HDR_FL_SHIFT);
+		ipv6.val.traffic_class = (vtc_flow_val & IPV6_HDR_TC_MASK) >>
+					 IPV6_HDR_TC_SHIFT;
+		ipv6.val.next_hdr = spec->hdr.proto;
+		ipv6.val.hop_limit = spec->hdr.hop_limits;
+		ipv6.mask.flow_label =
+			rte_cpu_to_be_32((vtc_flow_mask & IPV6_HDR_FL_MASK) >>
+					 IPV6_HDR_FL_SHIFT);
+		ipv6.mask.traffic_class = (vtc_flow_mask & IPV6_HDR_TC_MASK) >>
+					  IPV6_HDR_TC_SHIFT;
+		ipv6.mask.next_hdr = mask->hdr.proto;
+		ipv6.mask.hop_limit = mask->hdr.hop_limits;
+		/* Remove unwanted bits from values. */
+		for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
+			ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
+			ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
+		}
+		ipv6.val.flow_label &= ipv6.mask.flow_label;
+		ipv6.val.traffic_class &= ipv6.mask.traffic_class;
+		ipv6.val.next_hdr &= ipv6.mask.next_hdr;
+		ipv6.val.hop_limit &= ipv6.mask.hop_limit;
+	}
+	flow->l3_protocol_en = !!ipv6.mask.next_hdr;
+	flow->l3_protocol = ipv6.val.next_hdr;
+	if (size <= flow_size) {
+		mlx5_flow_verbs_hashfields_adjust
+			(flow, tunnel,
+			 (ETH_RSS_IPV6 | ETH_RSS_NONFRAG_IPV6_OTHER),
+			 (IBV_RX_HASH_SRC_IPV6 | IBV_RX_HASH_DST_IPV6));
+		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L3;
+		mlx5_flow_spec_verbs_add(flow, &ipv6, size);
+	}
+	return size;
+}
+
+/**
+ * Convert the @p item into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_item_udp(const struct rte_flow_item *item, struct rte_flow *flow,
+		   const size_t flow_size, struct rte_flow_error *error)
+{
+	const struct rte_flow_item_udp *spec = item->spec;
+	const struct rte_flow_item_udp *mask = item->mask;
+	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
+	unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
+	struct ibv_flow_spec_tcp_udp udp = {
+		.type = IBV_FLOW_SPEC_UDP | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
+		.size = size,
+	};
+	int ret;
+
+	if (flow->l3_protocol_en && flow->l3_protocol != MLX5_IP_PROTOCOL_UDP)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "protocol filtering not compatible"
+					  " with UDP layer");
+	if (!(flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
+			      MLX5_FLOW_LAYER_OUTER_L3)))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L3 is mandatory to filter"
+					  " on L4");
+	if (flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
+			    MLX5_FLOW_LAYER_OUTER_L4))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L4 layer is already"
+					  " present");
+	if (!mask)
+		mask = &rte_flow_item_udp_mask;
+	ret = mlx5_flow_item_acceptable
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&rte_flow_item_udp_mask,
+		 sizeof(struct rte_flow_item_udp), error);
+	if (ret < 0)
+		return ret;
+	flow->layers |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
+		MLX5_FLOW_LAYER_OUTER_L4_UDP;
+	if (spec) {
+		udp.val.dst_port = spec->hdr.dst_port;
+		udp.val.src_port = spec->hdr.src_port;
+		udp.mask.dst_port = mask->hdr.dst_port;
+		udp.mask.src_port = mask->hdr.src_port;
+		/* Remove unwanted bits from values. */
+		udp.val.src_port &= udp.mask.src_port;
+		udp.val.dst_port &= udp.mask.dst_port;
+	}
+	if (size <= flow_size) {
+		mlx5_flow_verbs_hashfields_adjust(flow, tunnel, ETH_RSS_UDP,
+						  (IBV_RX_HASH_SRC_PORT_UDP |
+						   IBV_RX_HASH_DST_PORT_UDP));
+		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L4;
+		mlx5_flow_spec_verbs_add(flow, &udp, size);
+	}
+	return size;
+}
+
+/**
+ * Convert the @p item into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_item_tcp(const struct rte_flow_item *item, struct rte_flow *flow,
+		   const size_t flow_size, struct rte_flow_error *error)
+{
+	const struct rte_flow_item_tcp *spec = item->spec;
+	const struct rte_flow_item_tcp *mask = item->mask;
+	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
+	unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
+	struct ibv_flow_spec_tcp_udp tcp = {
+		.type = IBV_FLOW_SPEC_TCP | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
+		.size = size,
+	};
+	int ret;
+
+	if (flow->l3_protocol_en && flow->l3_protocol != MLX5_IP_PROTOCOL_TCP)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "protocol filtering not compatible"
+					  " with TCP layer");
+	if (!(flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
+			      MLX5_FLOW_LAYER_OUTER_L3)))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L3 is mandatory to filter on L4");
+	if (flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
+			    MLX5_FLOW_LAYER_OUTER_L4))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L4 layer is already present");
+	if (!mask)
+		mask = &rte_flow_item_tcp_mask;
+	ret = mlx5_flow_item_acceptable
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&rte_flow_item_tcp_mask,
+		 sizeof(struct rte_flow_item_tcp), error);
+	if (ret < 0)
+		return ret;
+	flow->layers |=  tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
+		MLX5_FLOW_LAYER_OUTER_L4_TCP;
+	if (spec) {
+		tcp.val.dst_port = spec->hdr.dst_port;
+		tcp.val.src_port = spec->hdr.src_port;
+		tcp.mask.dst_port = mask->hdr.dst_port;
+		tcp.mask.src_port = mask->hdr.src_port;
+		/* Remove unwanted bits from values. */
+		tcp.val.src_port &= tcp.mask.src_port;
+		tcp.val.dst_port &= tcp.mask.dst_port;
+	}
+	if (size <= flow_size) {
+		mlx5_flow_verbs_hashfields_adjust(flow, tunnel, ETH_RSS_TCP,
+						  (IBV_RX_HASH_SRC_PORT_TCP |
+						   IBV_RX_HASH_DST_PORT_TCP));
+		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L4;
+		mlx5_flow_spec_verbs_add(flow, &tcp, size);
+	}
+	return size;
+}
+
+/**
+ * Convert the @p item into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_item_vxlan(const struct rte_flow_item *item, struct rte_flow *flow,
+		     const size_t flow_size, struct rte_flow_error *error)
+{
+	const struct rte_flow_item_vxlan *spec = item->spec;
+	const struct rte_flow_item_vxlan *mask = item->mask;
+	unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
+	struct ibv_flow_spec_tunnel vxlan = {
+		.type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
+		.size = size,
+	};
+	int ret;
+	union vni {
+		uint32_t vlan_id;
+		uint8_t vni[4];
+	} id = { .vlan_id = 0, };
+
+	if (flow->layers & MLX5_FLOW_LAYER_TUNNEL)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "a tunnel is already present");
+	/*
+	 * Verify only UDPv4 is present as defined in
+	 * https://tools.ietf.org/html/rfc7348
+	 */
+	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER_L4_UDP))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "no outer UDP layer found");
+	if (!mask)
+		mask = &rte_flow_item_vxlan_mask;
+	ret = mlx5_flow_item_acceptable
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&rte_flow_item_vxlan_mask,
+		 sizeof(struct rte_flow_item_vxlan), error);
+	if (ret < 0)
+		return ret;
+	if (spec) {
+		memcpy(&id.vni[1], spec->vni, 3);
+		vxlan.val.tunnel_id = id.vlan_id;
+		memcpy(&id.vni[1], mask->vni, 3);
+		vxlan.mask.tunnel_id = id.vlan_id;
+		/* Remove unwanted bits from values. */
+		vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
+	}
+	/*
+	 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if
+	 * only this layer is defined in the Verbs specification it is
+	 * interpreted as wildcard and all packets will match this
+	 * rule, if it follows a full stack layer (ex: eth / ipv4 /
+	 * udp), all packets matching the layers before will also
+	 * match this rule.  To avoid such situation, VNI 0 is
+	 * currently refused.
+	 */
+	if (!vxlan.val.tunnel_id)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "VXLAN vni cannot be 0");
+	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER))
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "VXLAN tunnel must be fully defined");
+	if (size <= flow_size) {
+		mlx5_flow_spec_verbs_add(flow, &vxlan, size);
+		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L2;
+	}
+	flow->layers |= MLX5_FLOW_LAYER_VXLAN;
+	return size;
+}
+
+/**
+ * Convert the @p item into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_item_vxlan_gpe(struct rte_eth_dev *dev,
+			 const struct rte_flow_item *item,
+			 struct rte_flow *flow, const size_t flow_size,
+			 struct rte_flow_error *error)
+{
+	const struct rte_flow_item_vxlan_gpe *spec = item->spec;
+	const struct rte_flow_item_vxlan_gpe *mask = item->mask;
+	unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
+	struct ibv_flow_spec_tunnel vxlan_gpe = {
+		.type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
+		.size = size,
+	};
+	int ret;
+	union vni {
+		uint32_t vlan_id;
+		uint8_t vni[4];
+	} id = { .vlan_id = 0, };
+
+	if (!((struct priv *)dev->data->dev_private)->config.l3_vxlan_en)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L3 VXLAN is not enabled by device"
+					  " parameter and/or not configured in"
+					  " firmware");
+	if (flow->layers & MLX5_FLOW_LAYER_TUNNEL)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "a tunnel is already present");
+	/*
+	 * Verify only UDPv4 is present as defined in
+	 * https://tools.ietf.org/html/rfc7348
+	 */
+	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER_L4_UDP))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "no outer UDP layer found");
+	if (!mask)
+		mask = &rte_flow_item_vxlan_gpe_mask;
+	ret = mlx5_flow_item_acceptable
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask,
+		 sizeof(struct rte_flow_item_vxlan_gpe), error);
+	if (ret < 0)
+		return ret;
+	if (spec) {
+		memcpy(&id.vni[1], spec->vni, 3);
+		vxlan_gpe.val.tunnel_id = id.vlan_id;
+		memcpy(&id.vni[1], mask->vni, 3);
+		vxlan_gpe.mask.tunnel_id = id.vlan_id;
+		if (spec->protocol)
+			return rte_flow_error_set
+				(error, EINVAL,
+				 RTE_FLOW_ERROR_TYPE_ITEM,
+				 item,
+				 "VxLAN-GPE protocol not supported");
+		/* Remove unwanted bits from values. */
+		vxlan_gpe.val.tunnel_id &= vxlan_gpe.mask.tunnel_id;
+	}
+	/*
+	 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this
+	 * layer is defined in the Verbs specification it is interpreted as
+	 * wildcard and all packets will match this rule, if it follows a full
+	 * stack layer (ex: eth / ipv4 / udp), all packets matching the layers
+	 * before will also match this rule.  To avoid such situation, VNI 0
+	 * is currently refused.
+	 */
+	if (!vxlan_gpe.val.tunnel_id)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "VXLAN-GPE vni cannot be 0");
+	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER))
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "VXLAN-GPE tunnel must be fully"
+					  " defined");
+	if (size <= flow_size) {
+		mlx5_flow_spec_verbs_add(flow, &vxlan_gpe, size);
+		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L2;
+	}
+	flow->layers |= MLX5_FLOW_LAYER_VXLAN_GPE;
+	return size;
+}
+
+/**
+ * Update the protocol in Verbs IPv4/IPv6 spec.
+ *
+ * @param[in, out] attr
+ *   Pointer to Verbs attributes structure.
+ * @param[in] search
+ *   Specification type to search in order to update the IP protocol.
+ * @param[in] protocol
+ *   Protocol value to set if none is present in the specification.
+ */
+static void
+mlx5_flow_item_gre_ip_protocol_update(struct ibv_flow_attr *attr,
+				      enum ibv_flow_spec_type search,
+				      uint8_t protocol)
+{
+	unsigned int i;
+	struct ibv_spec_header *hdr = (struct ibv_spec_header *)
+		((uint8_t *)attr + sizeof(struct ibv_flow_attr));
+
+	if (!attr)
+		return;
+	for (i = 0; i != attr->num_of_specs; ++i) {
+		if (hdr->type == search) {
+			union {
+				struct ibv_flow_spec_ipv4_ext *ipv4;
+				struct ibv_flow_spec_ipv6 *ipv6;
+			} ip;
+
+			switch (search) {
+			case IBV_FLOW_SPEC_IPV4_EXT:
+				ip.ipv4 = (struct ibv_flow_spec_ipv4_ext *)hdr;
+				if (!ip.ipv4->val.proto) {
+					ip.ipv4->val.proto = protocol;
+					ip.ipv4->mask.proto = 0xff;
+				}
+				break;
+			case IBV_FLOW_SPEC_IPV6:
+				ip.ipv6 = (struct ibv_flow_spec_ipv6 *)hdr;
+				if (!ip.ipv6->val.next_hdr) {
+					ip.ipv6->val.next_hdr = protocol;
+					ip.ipv6->mask.next_hdr = 0xff;
+				}
+				break;
+			default:
+				break;
+			}
+			break;
+		}
+		hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
+	}
+}
+
+/**
+ * Convert the @p item into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * It will also update the previous L3 layer with the protocol value matching
+ * the GRE.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_item_gre(const struct rte_flow_item *item,
+		   struct rte_flow *flow, const size_t flow_size,
+		   struct rte_flow_error *error)
+{
+	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
+	const struct rte_flow_item_gre *spec = item->spec;
+	const struct rte_flow_item_gre *mask = item->mask;
+#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
+	unsigned int size = sizeof(struct ibv_flow_spec_gre);
+	struct ibv_flow_spec_gre tunnel = {
+		.type = IBV_FLOW_SPEC_GRE,
+		.size = size,
+	};
+#else
+	unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
+	struct ibv_flow_spec_tunnel tunnel = {
+		.type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
+		.size = size,
+	};
+#endif
+	int ret;
+
+	if (flow->l3_protocol_en && flow->l3_protocol != MLX5_IP_PROTOCOL_GRE)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "protocol filtering not compatible"
+					  " with this GRE layer");
+	if (flow->layers & MLX5_FLOW_LAYER_TUNNEL)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "a tunnel is already present");
+	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER_L3))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L3 Layer is missing");
+	if (!mask)
+		mask = &rte_flow_item_gre_mask;
+	ret = mlx5_flow_item_acceptable
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&rte_flow_item_gre_mask,
+		 sizeof(struct rte_flow_item_gre), error);
+	if (ret < 0)
+		return ret;
+#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
+	if (spec) {
+		tunnel.val.c_ks_res0_ver = spec->c_rsvd0_ver;
+		tunnel.val.protocol = spec->protocol;
+		tunnel.mask.c_ks_res0_ver = mask->c_rsvd0_ver;
+		tunnel.mask.protocol = mask->protocol;
+		/* Remove unwanted bits from values. */
+		tunnel.val.c_ks_res0_ver &= tunnel.mask.c_ks_res0_ver;
+		tunnel.val.protocol &= tunnel.mask.protocol;
+		tunnel.val.key &= tunnel.mask.key;
+	}
+#else
+	if (spec && (spec->protocol & mask->protocol))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "without MPLS support the"
+					  " specification cannot be used for"
+					  " filtering");
+#endif /* !HAVE_IBV_DEVICE_MPLS_SUPPORT */
+	if (size <= flow_size) {
+		if (flow->layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
+			mlx5_flow_item_gre_ip_protocol_update
+				(verbs->attr, IBV_FLOW_SPEC_IPV4_EXT,
+				 MLX5_IP_PROTOCOL_GRE);
+		else
+			mlx5_flow_item_gre_ip_protocol_update
+				(verbs->attr, IBV_FLOW_SPEC_IPV6,
+				 MLX5_IP_PROTOCOL_GRE);
+		mlx5_flow_spec_verbs_add(flow, &tunnel, size);
+		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L2;
+	}
+	flow->layers |= MLX5_FLOW_LAYER_GRE;
+	return size;
+}
+
+/**
+ * Convert the @p item into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_item_mpls(const struct rte_flow_item *item __rte_unused,
+		    struct rte_flow *flow __rte_unused,
+		    const size_t flow_size __rte_unused,
+		    struct rte_flow_error *error)
+{
+#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
+	const struct rte_flow_item_mpls *spec = item->spec;
+	const struct rte_flow_item_mpls *mask = item->mask;
+	unsigned int size = sizeof(struct ibv_flow_spec_mpls);
+	struct ibv_flow_spec_mpls mpls = {
+		.type = IBV_FLOW_SPEC_MPLS,
+		.size = size,
+	};
+	int ret;
+
+	if (flow->l3_protocol_en && flow->l3_protocol != MLX5_IP_PROTOCOL_MPLS)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "protocol filtering not compatible"
+					  " with MPLS layer");
+	/* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */
+	if (flow->layers & MLX5_FLOW_LAYER_TUNNEL &&
+	    (flow->layers & MLX5_FLOW_LAYER_GRE) != MLX5_FLOW_LAYER_GRE)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "a tunnel is already"
+					  " present");
+	if (!mask)
+		mask = &rte_flow_item_mpls_mask;
+	ret = mlx5_flow_item_acceptable
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&rte_flow_item_mpls_mask,
+		 sizeof(struct rte_flow_item_mpls), error);
+	if (ret < 0)
+		return ret;
+	if (spec) {
+		memcpy(&mpls.val.label, spec, sizeof(mpls.val.label));
+		memcpy(&mpls.mask.label, mask, sizeof(mpls.mask.label));
+		/* Remove unwanted bits from values.  */
+		mpls.val.label &= mpls.mask.label;
+	}
+	if (size <= flow_size) {
+		mlx5_flow_spec_verbs_add(flow, &mpls, size);
+		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L2;
+	}
+	flow->layers |= MLX5_FLOW_LAYER_MPLS;
+	return size;
+#endif /* !HAVE_IBV_DEVICE_MPLS_SUPPORT */
+	return rte_flow_error_set(error, ENOTSUP,
+				  RTE_FLOW_ERROR_TYPE_ITEM,
+				  item,
+				  "MPLS is not supported by Verbs, please"
+				  " update.");
+}
+
+/**
+ * Convert the @p pattern into a Verbs specifications after ensuring the NIC
+ * will understand and process it correctly.
+ * The conversion is performed item per item, each of them is written into
+ * the @p flow if its size is lesser or equal to @p flow_size.
+ * Validation and memory consumption computation are still performed until the
+ * end of @p pattern, unless an error is encountered.
+ *
+ * @param[in] pattern
+ *   Flow pattern.
+ * @param[in, out] flow
+ *   Pointer to the rte_flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small some
+ *   garbage may be present.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @pattern  has fully been
+ *   converted, otherwise another call with this returned memory size should
+ *   be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_items(struct rte_eth_dev *dev,
+		const struct rte_flow_item pattern[],
+		struct rte_flow *flow, const size_t flow_size,
+		struct rte_flow_error *error)
+{
+	int remain = flow_size;
+	size_t size = 0;
+
+	for (; pattern->type != RTE_FLOW_ITEM_TYPE_END; pattern++) {
+		int ret = 0;
+
+		switch (pattern->type) {
+		case RTE_FLOW_ITEM_TYPE_VOID:
+			break;
+		case RTE_FLOW_ITEM_TYPE_ETH:
+			ret = mlx5_flow_item_eth(pattern, flow, remain, error);
+			break;
+		case RTE_FLOW_ITEM_TYPE_VLAN:
+			ret = mlx5_flow_item_vlan(pattern, flow, remain, error);
+			break;
+		case RTE_FLOW_ITEM_TYPE_IPV4:
+			ret = mlx5_flow_item_ipv4(pattern, flow, remain, error);
+			break;
+		case RTE_FLOW_ITEM_TYPE_IPV6:
+			ret = mlx5_flow_item_ipv6(pattern, flow, remain, error);
+			break;
+		case RTE_FLOW_ITEM_TYPE_UDP:
+			ret = mlx5_flow_item_udp(pattern, flow, remain, error);
+			break;
+		case RTE_FLOW_ITEM_TYPE_TCP:
+			ret = mlx5_flow_item_tcp(pattern, flow, remain, error);
+			break;
+		case RTE_FLOW_ITEM_TYPE_VXLAN:
+			ret = mlx5_flow_item_vxlan(pattern, flow, remain,
+						   error);
+			break;
+		case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
+			ret = mlx5_flow_item_vxlan_gpe(dev, pattern, flow,
+						       remain, error);
+			break;
+		case RTE_FLOW_ITEM_TYPE_GRE:
+			ret = mlx5_flow_item_gre(pattern, flow, remain, error);
+			break;
+		case RTE_FLOW_ITEM_TYPE_MPLS:
+			ret = mlx5_flow_item_mpls(pattern, flow, remain, error);
+			break;
+		default:
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  pattern,
+						  "item not supported");
+		}
+		if (ret < 0)
+			return ret;
+		if (remain > ret)
+			remain -= ret;
+		else
+			remain = 0;
+		size += ret;
+	}
+	if (!flow->layers) {
+		const struct rte_flow_item item = {
+			.type = RTE_FLOW_ITEM_TYPE_ETH,
+		};
+
+		return mlx5_flow_item_eth(&item, flow, flow_size, error);
+	}
+	return size;
+}
+
+/**
+ * Convert the @p action into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param[in] action
+ *   Action configuration.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p action has fully been
+ *   converted, otherwise another call with this returned memory size should
+ *   be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_action_drop(const struct rte_flow_action *action,
+		      struct rte_flow *flow, const size_t flow_size,
+		      struct rte_flow_error *error)
+{
+	unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
+	struct ibv_flow_spec_action_drop drop = {
+			.type = IBV_FLOW_SPEC_ACTION_DROP,
+			.size = size,
+	};
+
+	if (flow->fate)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "multiple fate actions are not"
+					  " supported");
+	if (flow->modifier & (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "drop is not compatible with"
+					  " flag/mark action");
+	if (size < flow_size)
+		mlx5_flow_spec_verbs_add(flow, &drop, size);
+	flow->fate |= MLX5_FLOW_FATE_DROP;
+	return size;
+}
+
+/**
+ * Convert the @p action into @p flow after ensuring the NIC will understand
+ * and process it correctly.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device structure.
+ * @param[in] action
+ *   Action configuration.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_action_queue(struct rte_eth_dev *dev,
+		       const struct rte_flow_action *action,
+		       struct rte_flow *flow,
+		       struct rte_flow_error *error)
+{
+	struct priv *priv = dev->data->dev_private;
+	const struct rte_flow_action_queue *queue = action->conf;
+
+	if (flow->fate)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "multiple fate actions are not"
+					  " supported");
+	if (queue->index >= priv->rxqs_n)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &queue->index,
+					  "queue index out of range");
+	if (!(*priv->rxqs)[queue->index])
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &queue->index,
+					  "queue is not configured");
+	if (flow->queue)
+		(*flow->queue)[0] = queue->index;
+	flow->rss.queue_num = 1;
+	flow->fate |= MLX5_FLOW_FATE_QUEUE;
+	return 0;
+}
+
+/**
+ * Ensure the @p action will be understood and used correctly by the  NIC.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param action[in]
+ *   Pointer to flow actions array.
+ * @param flow[in, out]
+ *   Pointer to the rte_flow structure.
+ * @param error[in, out]
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success @p flow->queue array and @p flow->rss are filled and valid.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_action_rss(struct rte_eth_dev *dev,
+		     const struct rte_flow_action *action,
+		     struct rte_flow *flow,
+		     struct rte_flow_error *error)
+{
+	struct priv *priv = dev->data->dev_private;
+	const struct rte_flow_action_rss *rss = action->conf;
+	unsigned int i;
+
+	if (flow->fate)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "multiple fate actions are not"
+					  " supported");
+	if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT &&
+	    rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->func,
+					  "RSS hash function not supported");
+#ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
+	if (rss->level > 2)
+#else
+	if (rss->level > 1)
+#endif
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->level,
+					  "tunnel RSS is not supported");
+	if (rss->key_len < MLX5_RSS_HASH_KEY_LEN)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->key_len,
+					  "RSS hash key too small");
+	if (rss->key_len > MLX5_RSS_HASH_KEY_LEN)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->key_len,
+					  "RSS hash key too large");
+	if (!rss->queue_num)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  rss,
+					  "no queues were provided for RSS");
+	if (rss->queue_num > priv->config.ind_table_max_size)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->queue_num,
+					  "number of queues too large");
+	if (rss->types & MLX5_RSS_HF_MASK)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->types,
+					  "some RSS protocols are not"
+					  " supported");
+	for (i = 0; i != rss->queue_num; ++i) {
+		if (rss->queue[i] >= priv->rxqs_n)
+			return rte_flow_error_set
+				(error, EINVAL,
+				 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+				 rss,
+				 "queue index out of range");
+		if (!(*priv->rxqs)[rss->queue[i]])
+			return rte_flow_error_set
+				(error, EINVAL,
+				 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+				 &rss->queue[i],
+				 "queue is not configured");
+	}
+	if (flow->queue)
+		memcpy((*flow->queue), rss->queue,
+		       rss->queue_num * sizeof(uint16_t));
+	flow->rss.queue_num = rss->queue_num;
+	memcpy(flow->key, rss->key, MLX5_RSS_HASH_KEY_LEN);
+	flow->rss.types = rss->types;
+	flow->rss.level = rss->level;
+	flow->fate |= MLX5_FLOW_FATE_RSS;
+	return 0;
+}
+
+/**
+ * Convert the @p action into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param[in] action
+ *   Action configuration.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p action has fully been
+ *   converted, otherwise another call with this returned memory size should
+ *   be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_action_flag(const struct rte_flow_action *action,
+		      struct rte_flow *flow, const size_t flow_size,
+		      struct rte_flow_error *error)
+{
+	unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
+	struct ibv_flow_spec_action_tag tag = {
+		.type = IBV_FLOW_SPEC_ACTION_TAG,
+		.size = size,
+		.tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT),
+	};
+	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
+
+	if (flow->modifier & MLX5_FLOW_MOD_FLAG)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "flag action already present");
+	if (flow->fate & MLX5_FLOW_FATE_DROP)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "flag is not compatible with drop"
+					  " action");
+	if (flow->modifier & MLX5_FLOW_MOD_MARK)
+		size = 0;
+	else if (size <= flow_size && verbs)
+		mlx5_flow_spec_verbs_add(flow, &tag, size);
+	flow->modifier |= MLX5_FLOW_MOD_FLAG;
+	return size;
+}
+
+/**
+ * Update verbs specification to modify the flag to mark.
+ *
+ * @param[in, out] verbs
+ *   Pointer to the mlx5_flow_verbs structure.
+ * @param[in] mark_id
+ *   Mark identifier to replace the flag.
+ */
+static void
+mlx5_flow_verbs_mark_update(struct mlx5_flow_verbs *verbs, uint32_t mark_id)
+{
+	struct ibv_spec_header *hdr;
+	int i;
+
+	if (!verbs)
+		return;
+	/* Update Verbs specification. */
+	hdr = (struct ibv_spec_header *)verbs->specs;
+	if (!hdr)
+		return;
+	for (i = 0; i != verbs->attr->num_of_specs; ++i) {
+		if (hdr->type == IBV_FLOW_SPEC_ACTION_TAG) {
+			struct ibv_flow_spec_action_tag *t =
+				(struct ibv_flow_spec_action_tag *)hdr;
+
+			t->tag_id = mlx5_flow_mark_set(mark_id);
+		}
+		hdr = (struct ibv_spec_header *)((uintptr_t)hdr + hdr->size);
+	}
+}
+
+/**
+ * Convert the @p action into @p flow (or by updating the already present
+ * Flag Verbs specification) after ensuring the NIC will understand and
+ * process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param[in] action
+ *   Action configuration.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p action has fully been
+ *   converted, otherwise another call with this returned memory size should
+ *   be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_action_mark(const struct rte_flow_action *action,
+		      struct rte_flow *flow, const size_t flow_size,
+		      struct rte_flow_error *error)
+{
+	const struct rte_flow_action_mark *mark = action->conf;
+	unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
+	struct ibv_flow_spec_action_tag tag = {
+		.type = IBV_FLOW_SPEC_ACTION_TAG,
+		.size = size,
+	};
+	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
+
+	if (!mark)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "configuration cannot be null");
+	if (mark->id >= MLX5_FLOW_MARK_MAX)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &mark->id,
+					  "mark id must in 0 <= id < "
+					  RTE_STR(MLX5_FLOW_MARK_MAX));
+	if (flow->modifier & MLX5_FLOW_MOD_MARK)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "mark action already present");
+	if (flow->fate & MLX5_FLOW_FATE_DROP)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "mark is not compatible with drop"
+					  " action");
+	if (flow->modifier & MLX5_FLOW_MOD_FLAG) {
+		mlx5_flow_verbs_mark_update(verbs, mark->id);
+		size = 0;
+	} else if (size <= flow_size) {
+		tag.tag_id = mlx5_flow_mark_set(mark->id);
+		mlx5_flow_spec_verbs_add(flow, &tag, size);
+	}
+	flow->modifier |= MLX5_FLOW_MOD_MARK;
+	return size;
+}
+
+/**
+ * Convert the @p action into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param action[in]
+ *   Action configuration.
+ * @param flow[in, out]
+ *   Pointer to flow structure.
+ * @param flow_size[in]
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param error[int, out]
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p action has fully been
+ *   converted, otherwise another call with this returned memory size should
+ *   be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_action_count(struct rte_eth_dev *dev,
+		       const struct rte_flow_action *action,
+		       struct rte_flow *flow,
+		       const size_t flow_size __rte_unused,
+		       struct rte_flow_error *error)
+{
+	const struct rte_flow_action_count *count = action->conf;
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+	unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
+	struct ibv_flow_spec_counter_action counter = {
+		.type = IBV_FLOW_SPEC_ACTION_COUNT,
+		.size = size,
+	};
+#endif
+
+	if (!flow->counter) {
+		flow->counter = mlx5_flow_counter_new(dev, count->shared,
+						      count->id);
+		if (!flow->counter)
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  action,
+						  "cannot get counter"
+						  " context.");
+	}
+	if (!((struct priv *)dev->data->dev_private)->config.flow_counter_en)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "flow counters are not supported.");
+	flow->modifier |= MLX5_FLOW_MOD_COUNT;
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+	counter.counter_set_handle = flow->counter->cs->handle;
+	if (size <= flow_size)
+		mlx5_flow_spec_verbs_add(flow, &counter, size);
+	return size;
+#endif
+	return 0;
+}
+
+/**
+ * Convert the @p action into @p flow after ensuring the NIC will understand
+ * and process it correctly.
+ * The conversion is performed action per action, each of them is written into
+ * the @p flow if its size is lesser or equal to @p flow_size.
+ * Validation and memory consumption computation are still performed until the
+ * end of @p action, unless an error is encountered.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device structure.
+ * @param[in] actions
+ *   Pointer to flow actions array.
+ * @param[in, out] flow
+ *   Pointer to the rte_flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small some
+ *   garbage may be present.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p actions has fully been
+ *   converted, otherwise another call with this returned memory size should
+ *   be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_actions(struct rte_eth_dev *dev,
+		  const struct rte_flow_action actions[],
+		  struct rte_flow *flow, const size_t flow_size,
+		  struct rte_flow_error *error)
+{
+	size_t size = 0;
+	int remain = flow_size;
+	int ret = 0;
+
+	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
+		switch (actions->type) {
+		case RTE_FLOW_ACTION_TYPE_VOID:
+			break;
+		case RTE_FLOW_ACTION_TYPE_FLAG:
+			ret = mlx5_flow_action_flag(actions, flow, remain,
+						    error);
+			break;
+		case RTE_FLOW_ACTION_TYPE_MARK:
+			ret = mlx5_flow_action_mark(actions, flow, remain,
+						    error);
+			break;
+		case RTE_FLOW_ACTION_TYPE_DROP:
+			ret = mlx5_flow_action_drop(actions, flow, remain,
+						    error);
+			break;
+		case RTE_FLOW_ACTION_TYPE_QUEUE:
+			ret = mlx5_flow_action_queue(dev, actions, flow, error);
+			break;
+		case RTE_FLOW_ACTION_TYPE_RSS:
+			ret = mlx5_flow_action_rss(dev, actions, flow, error);
+			break;
+		case RTE_FLOW_ACTION_TYPE_COUNT:
+			ret = mlx5_flow_action_count(dev, actions, flow, remain,
+						     error);
+			break;
+		default:
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  actions,
+						  "action not supported");
+		}
+		if (ret < 0)
+			return ret;
+		if (remain > ret)
+			remain -= ret;
+		else
+			remain = 0;
+		size += ret;
+	}
+	if (!flow->fate)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					  NULL,
+					  "no fate action found");
+	return size;
+}
+
+/**
+ * Validate flow rule and fill flow structure accordingly.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param[out] flow
+ *   Pointer to flow structure.
+ * @param flow_size
+ *   Size of allocated space for @p flow.
+ * @param[in] attr
+ *   Flow rule attributes.
+ * @param[in] pattern
+ *   Pattern specification (list terminated by the END pattern item).
+ * @param[in] actions
+ *   Associated actions (list terminated by the END action).
+ * @param[out] error
+ *   Perform verbose error reporting if not NULL.
+ *
+ * @return
+ *   A positive value representing the size of the flow object in bytes
+ *   regardless of @p flow_size on success, a negative errno value otherwise
+ *   and rte_errno is set.
+ */
+static int
+mlx5_flow_merge_switch(struct rte_eth_dev *dev,
+		       struct rte_flow *flow,
+		       size_t flow_size,
+		       const struct rte_flow_attr *attr,
+		       const struct rte_flow_item pattern[],
+		       const struct rte_flow_action actions[],
+		       struct rte_flow_error *error)
+{
+	unsigned int n = mlx5_dev_to_port_id(dev->device, NULL, 0);
+	uint16_t port_id[!n + n];
+	struct mlx5_nl_flow_ptoi ptoi[!n + n + 1];
+	size_t off = RTE_ALIGN_CEIL(sizeof(*flow), alignof(max_align_t));
+	unsigned int i;
+	unsigned int own = 0;
+	int ret;
+
+	/* At least one port is needed when no switch domain is present. */
+	if (!n) {
+		n = 1;
+		port_id[0] = dev->data->port_id;
+	} else {
+		n = RTE_MIN(mlx5_dev_to_port_id(dev->device, port_id, n), n);
+	}
+	for (i = 0; i != n; ++i) {
+		struct rte_eth_dev_info dev_info;
+
+		rte_eth_dev_info_get(port_id[i], &dev_info);
+		if (port_id[i] == dev->data->port_id)
+			own = i;
+		ptoi[i].port_id = port_id[i];
+		ptoi[i].ifindex = dev_info.if_index;
+	}
+	/* Ensure first entry of ptoi[] is the current device. */
+	if (own) {
+		ptoi[n] = ptoi[0];
+		ptoi[0] = ptoi[own];
+		ptoi[own] = ptoi[n];
+	}
+	/* An entry with zero ifindex terminates ptoi[]. */
+	ptoi[n].port_id = 0;
+	ptoi[n].ifindex = 0;
+	if (flow_size < off)
+		flow_size = 0;
+	ret = mlx5_nl_flow_transpose((uint8_t *)flow + off,
+				     flow_size ? flow_size - off : 0,
+				     ptoi, attr, pattern, actions, error);
+	if (ret < 0)
+		return ret;
+	if (flow_size) {
+		*flow = (struct rte_flow){
+			.attributes = *attr,
+			.nl_flow = (uint8_t *)flow + off,
+		};
+		/*
+		 * Generate a reasonably unique handle based on the address
+		 * of the target buffer.
+		 *
+		 * This is straightforward on 32-bit systems where the flow
+		 * pointer can be used directly. Otherwise, its least
+		 * significant part is taken after shifting it by the
+		 * previous power of two of the pointed buffer size.
+		 */
+		if (sizeof(flow) <= 4)
+			mlx5_nl_flow_brand(flow->nl_flow, (uintptr_t)flow);
+		else
+			mlx5_nl_flow_brand
+				(flow->nl_flow,
+				 (uintptr_t)flow >>
+				 rte_log2_u32(rte_align32prevpow2(flow_size)));
+	}
+	return off + ret;
+}
+
+static unsigned int
+mlx5_find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level)
+{
+	const struct rte_flow_item *item;
+	unsigned int has_vlan = 0;
+
+	for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
+		if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
+			has_vlan = 1;
+			break;
+		}
+	}
+	if (has_vlan)
+		return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN :
+				       MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN;
+	return rss_level < 2 ? MLX5_EXPANSION_ROOT :
+			       MLX5_EXPANSION_ROOT_OUTER;
+}
+
+/**
+ * Convert the @p attributes, @p pattern, @p action, into an flow for the NIC
+ * after ensuring the NIC will understand and process it correctly.
+ * The conversion is only performed item/action per item/action, each of
+ * them is written into the @p flow if its size is lesser or equal to @p
+ * flow_size.
+ * Validation and memory consumption computation are still performed until the
+ * end, unless an error is encountered.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small some
+ *   garbage may be present.
+ * @param[in] attributes
+ *   Flow rule attributes.
+ * @param[in] pattern
+ *   Pattern specification (list terminated by the END pattern item).
+ * @param[in] actions
+ *   Associated actions (list terminated by the END action).
+ * @param[out] error
+ *   Perform verbose error reporting if not NULL.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the flow has fully been converted and
+ *   can be applied, otherwise another call with this returned memory size
+ *   should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_merge(struct rte_eth_dev *dev, struct rte_flow *flow,
+		const size_t flow_size,
+		const struct rte_flow_attr *attributes,
+		const struct rte_flow_item pattern[],
+		const struct rte_flow_action actions[],
+		struct rte_flow_error *error)
+{
+	struct rte_flow local_flow = { .layers = 0, };
+	size_t size = sizeof(*flow);
+	union {
+		struct rte_flow_expand_rss buf;
+		uint8_t buffer[2048];
+	} expand_buffer;
+	struct rte_flow_expand_rss *buf = &expand_buffer.buf;
+	struct mlx5_flow_verbs *original_verbs = NULL;
+	size_t original_verbs_size = 0;
+	uint32_t original_layers = 0;
+	int expanded_pattern_idx = 0;
+	int ret;
+	uint32_t i;
+
+	if (attributes->transfer)
+		return mlx5_flow_merge_switch(dev, flow, flow_size,
+					      attributes, pattern,
+					      actions, error);
+	if (size > flow_size)
+		flow = &local_flow;
+	ret = mlx5_flow_attributes(dev, attributes, flow, error);
+	if (ret < 0)
+		return ret;
+	ret = mlx5_flow_actions(dev, actions, &local_flow, 0, error);
+	if (ret < 0)
+		return ret;
+	if (local_flow.rss.types) {
+		unsigned int graph_root;
+
+		graph_root = mlx5_find_graph_root(pattern,
+						  local_flow.rss.level);
+		ret = rte_flow_expand_rss(buf, sizeof(expand_buffer.buffer),
+					  pattern, local_flow.rss.types,
+					  mlx5_support_expansion,
+					  graph_root);
+		assert(ret > 0 &&
+		       (unsigned int)ret < sizeof(expand_buffer.buffer));
+	} else {
+		buf->entries = 1;
+		buf->entry[0].pattern = (void *)(uintptr_t)pattern;
+	}
+	size += RTE_ALIGN_CEIL(local_flow.rss.queue_num * sizeof(uint16_t),
+			       sizeof(void *));
+	if (size <= flow_size)
+		flow->queue = (void *)(flow + 1);
+	LIST_INIT(&flow->verbs);
+	flow->layers = 0;
+	flow->modifier = 0;
+	flow->fate = 0;
+	for (i = 0; i != buf->entries; ++i) {
+		size_t off = size;
+		size_t off2;
+
+		flow->layers = original_layers;
+		size += sizeof(struct ibv_flow_attr) +
+			sizeof(struct mlx5_flow_verbs);
+		off2 = size;
+		if (size < flow_size) {
+			flow->cur_verbs = (void *)((uintptr_t)flow + off);
+			flow->cur_verbs->attr = (void *)(flow->cur_verbs + 1);
+			flow->cur_verbs->specs =
+				(void *)(flow->cur_verbs->attr + 1);
+		}
+		/* First iteration convert the pattern into Verbs. */
+		if (i == 0) {
+			/* Actions don't need to be converted several time. */
+			ret = mlx5_flow_actions(dev, actions, flow,
+						(size < flow_size) ?
+						flow_size - size : 0,
+						error);
+			if (ret < 0)
+				return ret;
+			size += ret;
+		} else {
+			/*
+			 * Next iteration means the pattern has already been
+			 * converted and an expansion is necessary to match
+			 * the user RSS request.  For that only the expanded
+			 * items will be converted, the common part with the
+			 * user pattern are just copied into the next buffer
+			 * zone.
+			 */
+			size += original_verbs_size;
+			if (size < flow_size) {
+				rte_memcpy(flow->cur_verbs->attr,
+					   original_verbs->attr,
+					   original_verbs_size +
+					   sizeof(struct ibv_flow_attr));
+				flow->cur_verbs->size = original_verbs_size;
+			}
+		}
+		ret = mlx5_flow_items
+			(dev,
+			 (const struct rte_flow_item *)
+			 &buf->entry[i].pattern[expanded_pattern_idx],
+			 flow,
+			 (size < flow_size) ? flow_size - size : 0, error);
+		if (ret < 0)
+			return ret;
+		size += ret;
+		if (size <= flow_size) {
+			mlx5_flow_adjust_priority(dev, flow);
+			LIST_INSERT_HEAD(&flow->verbs, flow->cur_verbs, next);
+		}
+		/*
+		 * Keep a pointer of the first verbs conversion and the layers
+		 * it has encountered.
+		 */
+		if (i == 0) {
+			original_verbs = flow->cur_verbs;
+			original_verbs_size = size - off2;
+			original_layers = flow->layers;
+			/*
+			 * move the index of the expanded pattern to the
+			 * first item not addressed yet.
+			 */
+			if (pattern->type == RTE_FLOW_ITEM_TYPE_END) {
+				expanded_pattern_idx++;
+			} else {
+				const struct rte_flow_item *item = pattern;
+
+				for (item = pattern;
+				     item->type != RTE_FLOW_ITEM_TYPE_END;
+				     ++item)
+					expanded_pattern_idx++;
+			}
+		}
+	}
+	/* Restore the origin layers in the flow. */
+	flow->layers = original_layers;
+	return size;
+}
+
+/**
+ * Lookup and set the ptype in the data Rx part.  A single Ptype can be used,
+ * if several tunnel rules are used on this queue, the tunnel ptype will be
+ * cleared.
+ *
+ * @param rxq_ctrl
+ *   Rx queue to update.
+ */
+static void
+mlx5_flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl)
+{
+	unsigned int i;
+	uint32_t tunnel_ptype = 0;
+
+	/* Look up for the ptype to use. */
+	for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) {
+		if (!rxq_ctrl->flow_tunnels_n[i])
+			continue;
+		if (!tunnel_ptype) {
+			tunnel_ptype = tunnels_info[i].ptype;
+		} else {
+			tunnel_ptype = 0;
+			break;
+		}
+	}
+	rxq_ctrl->rxq.tunnel = tunnel_ptype;
+}
+
+/**
+ * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the flow.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ * @param[in] flow
+ *   Pointer to flow structure.
+ */
+static void
+mlx5_flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow)
+{
+	struct priv *priv = dev->data->dev_private;
+	const int mark = !!(flow->modifier &
+			    (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK));
+	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
+	unsigned int i;
+
+	for (i = 0; i != flow->rss.queue_num; ++i) {
+		int idx = (*flow->queue)[i];
+		struct mlx5_rxq_ctrl *rxq_ctrl =
+			container_of((*priv->rxqs)[idx],
+				     struct mlx5_rxq_ctrl, rxq);
+
+		if (mark) {
+			rxq_ctrl->rxq.mark = 1;
+			rxq_ctrl->flow_mark_n++;
+		}
+		if (tunnel) {
+			unsigned int j;
+
+			/* Increase the counter matching the flow. */
+			for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
+				if ((tunnels_info[j].tunnel & flow->layers) ==
+				    tunnels_info[j].tunnel) {
+					rxq_ctrl->flow_tunnels_n[j]++;
+					break;
+				}
+			}
+			mlx5_flow_rxq_tunnel_ptype_update(rxq_ctrl);
+		}
+	}
+}
+
+/**
+ * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
+ * @p flow if no other flow uses it with the same kind of request.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param[in] flow
+ *   Pointer to the flow.
+ */
+static void
+mlx5_flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
+{
+	struct priv *priv = dev->data->dev_private;
+	const int mark = !!(flow->modifier &
+			    (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK));
+	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
+	unsigned int i;
+
+	assert(dev->data->dev_started);
+	for (i = 0; i != flow->rss.queue_num; ++i) {
+		int idx = (*flow->queue)[i];
+		struct mlx5_rxq_ctrl *rxq_ctrl =
+			container_of((*priv->rxqs)[idx],
+				     struct mlx5_rxq_ctrl, rxq);
+
+		if (mark) {
+			rxq_ctrl->flow_mark_n--;
+			rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
+		}
+		if (tunnel) {
+			unsigned int j;
+
+			/* Decrease the counter matching the flow. */
+			for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
+				if ((tunnels_info[j].tunnel & flow->layers) ==
+				    tunnels_info[j].tunnel) {
+					rxq_ctrl->flow_tunnels_n[j]--;
+					break;
+				}
+			}
+			mlx5_flow_rxq_tunnel_ptype_update(rxq_ctrl);
+		}
+	}
+}
+
+/**
+ * Clear the Mark/Flag and Tunnel ptype information in all Rx queues.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ */
+static void
+mlx5_flow_rxq_flags_clear(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	unsigned int i;
+
+	for (i = 0; i != priv->rxqs_n; ++i) {
+		struct mlx5_rxq_ctrl *rxq_ctrl;
+		unsigned int j;
+
+		if (!(*priv->rxqs)[i])
+			continue;
+		rxq_ctrl = container_of((*priv->rxqs)[i],
+					struct mlx5_rxq_ctrl, rxq);
+		rxq_ctrl->flow_mark_n = 0;
+		rxq_ctrl->rxq.mark = 0;
+		for (j = 0; j != MLX5_FLOW_TUNNEL; ++j)
+			rxq_ctrl->flow_tunnels_n[j] = 0;
+		rxq_ctrl->rxq.tunnel = 0;
+	}
+}
+
+/**
+ * Validate a flow supported by the NIC.
+ *
+ * @see rte_flow_validate()
+ * @see rte_flow_ops
+ */
+int
+mlx5_flow_validate(struct rte_eth_dev *dev,
+		   const struct rte_flow_attr *attr,
+		   const struct rte_flow_item items[],
+		   const struct rte_flow_action actions[],
+		   struct rte_flow_error *error)
+{
+	int ret = mlx5_flow_merge(dev, NULL, 0, attr, items, actions, error);
+
+	if (ret < 0)
+		return ret;
+	return 0;
+}
+
+/**
+ * Remove the flow.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ */
+static void
+mlx5_flow_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_flow_verbs *verbs;
+
+	if (flow->nl_flow && priv->mnl_socket)
+		mlx5_nl_flow_destroy(priv->mnl_socket, flow->nl_flow, NULL);
+	LIST_FOREACH(verbs, &flow->verbs, next) {
+		if (verbs->flow) {
+			claim_zero(mlx5_glue->destroy_flow(verbs->flow));
+			verbs->flow = NULL;
+		}
+		if (verbs->hrxq) {
+			if (flow->fate & MLX5_FLOW_FATE_DROP)
+				mlx5_hrxq_drop_release(dev);
+			else
+				mlx5_hrxq_release(dev, verbs->hrxq);
+			verbs->hrxq = NULL;
+		}
+	}
+	if (flow->counter) {
+		mlx5_flow_counter_release(flow->counter);
+		flow->counter = NULL;
+	}
+}
+
+/**
+ * Apply the flow.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device structure.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
+		struct rte_flow_error *error)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_flow_verbs *verbs;
+	int err;
+
+	LIST_FOREACH(verbs, &flow->verbs, next) {
+		if (flow->fate & MLX5_FLOW_FATE_DROP) {
+			verbs->hrxq = mlx5_hrxq_drop_new(dev);
+			if (!verbs->hrxq) {
+				rte_flow_error_set
+					(error, errno,
+					 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					 NULL,
+					 "cannot get drop hash queue");
+				goto error;
+			}
+		} else {
+			struct mlx5_hrxq *hrxq;
+
+			hrxq = mlx5_hrxq_get(dev, flow->key,
+					     MLX5_RSS_HASH_KEY_LEN,
+					     verbs->hash_fields,
+					     (*flow->queue),
+					     flow->rss.queue_num);
+			if (!hrxq)
+				hrxq = mlx5_hrxq_new(dev, flow->key,
+						     MLX5_RSS_HASH_KEY_LEN,
+						     verbs->hash_fields,
+						     (*flow->queue),
+						     flow->rss.queue_num,
+						     !!(flow->layers &
+						      MLX5_FLOW_LAYER_TUNNEL));
+			if (!hrxq) {
+				rte_flow_error_set
+					(error, rte_errno,
+					 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					 NULL,
+					 "cannot get hash queue");
+				goto error;
+			}
+			verbs->hrxq = hrxq;
+		}
+		verbs->flow =
+			mlx5_glue->create_flow(verbs->hrxq->qp, verbs->attr);
+		if (!verbs->flow) {
+			rte_flow_error_set(error, errno,
+					   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					   NULL,
+					   "hardware refuses to create flow");
+			goto error;
+		}
+	}
+	if (flow->nl_flow &&
+	    priv->mnl_socket &&
+	    mlx5_nl_flow_create(priv->mnl_socket, flow->nl_flow, error))
+		goto error;
+	return 0;
+error:
+	err = rte_errno; /* Save rte_errno before cleanup. */
+	LIST_FOREACH(verbs, &flow->verbs, next) {
+		if (verbs->hrxq) {
+			if (flow->fate & MLX5_FLOW_FATE_DROP)
+				mlx5_hrxq_drop_release(dev);
+			else
+				mlx5_hrxq_release(dev, verbs->hrxq);
+			verbs->hrxq = NULL;
+		}
+	}
+	rte_errno = err; /* Restore rte_errno. */
+	return -rte_errno;
+}
+
+/**
+ * Create a flow and add it to @p list.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param list
+ *   Pointer to a TAILQ flow list.
+ * @param[in] attr
+ *   Flow rule attributes.
+ * @param[in] items
+ *   Pattern specification (list terminated by the END pattern item).
+ * @param[in] actions
+ *   Associated actions (list terminated by the END action).
+ * @param[out] error
+ *   Perform verbose error reporting if not NULL.
+ *
+ * @return
+ *   A flow on success, NULL otherwise and rte_errno is set.
+ */
+static struct rte_flow *
+mlx5_flow_list_create(struct rte_eth_dev *dev,
+		      struct mlx5_flows *list,
+		      const struct rte_flow_attr *attr,
+		      const struct rte_flow_item items[],
+		      const struct rte_flow_action actions[],
+		      struct rte_flow_error *error)
+{
+	struct rte_flow *flow = NULL;
+	size_t size = 0;
+	int ret;
+
+	ret = mlx5_flow_merge(dev, flow, size, attr, items, actions, error);
+	if (ret < 0)
+		return NULL;
+	size = ret;
+	flow = rte_calloc(__func__, 1, size, 0);
+	if (!flow) {
+		rte_flow_error_set(error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				   NULL,
+				   "not enough memory to create flow");
+		return NULL;
+	}
+	ret = mlx5_flow_merge(dev, flow, size, attr, items, actions, error);
+	if (ret < 0) {
+		rte_free(flow);
+		return NULL;
+	}
+	assert((size_t)ret == size);
+	if (dev->data->dev_started) {
+		ret = mlx5_flow_apply(dev, flow, error);
+		if (ret < 0) {
+			ret = rte_errno; /* Save rte_errno before cleanup. */
+			if (flow) {
+				mlx5_flow_remove(dev, flow);
+				rte_free(flow);
+			}
+			rte_errno = ret; /* Restore rte_errno. */
+			return NULL;
+		}
+	}
+	TAILQ_INSERT_TAIL(list, flow, next);
+	mlx5_flow_rxq_flags_set(dev, flow);
+	return flow;
+}
+
+/**
+ * Create a flow.
+ *
+ * @see rte_flow_create()
+ * @see rte_flow_ops
+ */
+struct rte_flow *
+mlx5_flow_create(struct rte_eth_dev *dev,
+		 const struct rte_flow_attr *attr,
+		 const struct rte_flow_item items[],
+		 const struct rte_flow_action actions[],
+		 struct rte_flow_error *error)
+{
+	return mlx5_flow_list_create
+		(dev, &((struct priv *)dev->data->dev_private)->flows,
+		 attr, items, actions, error);
+}
+
+/**
+ * Destroy a flow in a list.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param list
+ *   Pointer to a TAILQ flow list.
+ * @param[in] flow
+ *   Flow to destroy.
+ */
+static void
+mlx5_flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
+		       struct rte_flow *flow)
+{
+	mlx5_flow_remove(dev, flow);
+	TAILQ_REMOVE(list, flow, next);
+	/*
+	 * Update RX queue flags only if port is started, otherwise it is
+	 * already clean.
+	 */
+	if (dev->data->dev_started)
+		mlx5_flow_rxq_flags_trim(dev, flow);
+	rte_free(flow);
+}
+
+/**
+ * Destroy all flows.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param list
+ *   Pointer to a TAILQ flow list.
+ */
+void
+mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list)
+{
+	while (!TAILQ_EMPTY(list)) {
+		struct rte_flow *flow;
+
+		flow = TAILQ_FIRST(list);
+		mlx5_flow_list_destroy(dev, list, flow);
+	}
+}
+
+/**
+ * Remove all flows.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param list
+ *   Pointer to a TAILQ flow list.
+ */
+void
+mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list)
+{
+	struct rte_flow *flow;
+
+	TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next)
+		mlx5_flow_remove(dev, flow);
+	mlx5_flow_rxq_flags_clear(dev);
+}
+
+/**
+ * Add all flows.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param list
+ *   Pointer to a TAILQ flow list.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list)
+{
+	struct rte_flow *flow;
+	struct rte_flow_error error;
+	int ret = 0;
+
+	TAILQ_FOREACH(flow, list, next) {
+		ret = mlx5_flow_apply(dev, flow, &error);
+		if (ret < 0)
+			goto error;
+		mlx5_flow_rxq_flags_set(dev, flow);
+	}
+	return 0;
+error:
+	ret = rte_errno; /* Save rte_errno before cleanup. */
+	mlx5_flow_stop(dev, list);
+	rte_errno = ret; /* Restore rte_errno. */
+	return -rte_errno;
+}
+
+/**
+ * Verify the flow list is empty
+ *
+ * @param dev
+ *  Pointer to Ethernet device.
+ *
+ * @return the number of flows not released.
+ */
+int
+mlx5_flow_verify(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct rte_flow *flow;
+	int ret = 0;
+
+	TAILQ_FOREACH(flow, &priv->flows, next) {
+		DRV_LOG(DEBUG, "port %u flow %p still referenced",
+			dev->data->port_id, (void *)flow);
+		++ret;
+	}
+	return ret;
+}
+
+/**
+ * Enable a control flow configured from the control plane.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param eth_spec
+ *   An Ethernet flow spec to apply.
+ * @param eth_mask
+ *   An Ethernet flow mask to apply.
+ * @param vlan_spec
+ *   A VLAN flow spec to apply.
+ * @param vlan_mask
+ *   A VLAN flow mask to apply.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
+		    struct rte_flow_item_eth *eth_spec,
+		    struct rte_flow_item_eth *eth_mask,
+		    struct rte_flow_item_vlan *vlan_spec,
+		    struct rte_flow_item_vlan *vlan_mask)
+{
+	struct priv *priv = dev->data->dev_private;
+	const struct rte_flow_attr attr = {
+		.ingress = 1,
+		.priority = MLX5_FLOW_PRIO_RSVD,
+	};
+	struct rte_flow_item items[] = {
+		{
+			.type = RTE_FLOW_ITEM_TYPE_ETH,
+			.spec = eth_spec,
+			.last = NULL,
+			.mask = eth_mask,
+		},
+		{
+			.type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
+				RTE_FLOW_ITEM_TYPE_END,
+			.spec = vlan_spec,
+			.last = NULL,
+			.mask = vlan_mask,
+		},
+		{
+			.type = RTE_FLOW_ITEM_TYPE_END,
+		},
+	};
+	uint16_t queue[priv->reta_idx_n];
+	struct rte_flow_action_rss action_rss = {
+		.func = RTE_ETH_HASH_FUNCTION_DEFAULT,
+		.level = 0,
+		.types = priv->rss_conf.rss_hf,
+		.key_len = priv->rss_conf.rss_key_len,
+		.queue_num = priv->reta_idx_n,
+		.key = priv->rss_conf.rss_key,
+		.queue = queue,
+	};
+	struct rte_flow_action actions[] = {
+		{
+			.type = RTE_FLOW_ACTION_TYPE_RSS,
+			.conf = &action_rss,
+		},
+		{
+			.type = RTE_FLOW_ACTION_TYPE_END,
+		},
+	};
+	struct rte_flow *flow;
+	struct rte_flow_error error;
+	unsigned int i;
+
+	if (!priv->reta_idx_n) {
+		rte_errno = EINVAL;
+		return -rte_errno;
+	}
+	for (i = 0; i != priv->reta_idx_n; ++i)
+		queue[i] = (*priv->reta_idx)[i];
+	flow = mlx5_flow_list_create(dev, &priv->ctrl_flows, &attr, items,
+				     actions, &error);
+	if (!flow)
+		return -rte_errno;
+	return 0;
+}
+
+/**
+ * Enable a flow control configured from the control plane.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param eth_spec
+ *   An Ethernet flow spec to apply.
+ * @param eth_mask
+ *   An Ethernet flow mask to apply.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_ctrl_flow(struct rte_eth_dev *dev,
+	       struct rte_flow_item_eth *eth_spec,
+	       struct rte_flow_item_eth *eth_mask)
+{
+	return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
+}
+
+/**
+ * Destroy a flow.
+ *
+ * @see rte_flow_destroy()
+ * @see rte_flow_ops
+ */
+int
+mlx5_flow_destroy(struct rte_eth_dev *dev,
+		  struct rte_flow *flow,
+		  struct rte_flow_error *error __rte_unused)
+{
+	struct priv *priv = dev->data->dev_private;
+
+	mlx5_flow_list_destroy(dev, &priv->flows, flow);
+	return 0;
+}
+
+/**
+ * Destroy all flows.
+ *
+ * @see rte_flow_flush()
+ * @see rte_flow_ops
+ */
+int
+mlx5_flow_flush(struct rte_eth_dev *dev,
+		struct rte_flow_error *error __rte_unused)
+{
+	struct priv *priv = dev->data->dev_private;
+
+	mlx5_flow_list_flush(dev, &priv->flows);
+	return 0;
+}
+
+/**
+ * Isolated mode.
+ *
+ * @see rte_flow_isolate()
+ * @see rte_flow_ops
+ */
+int
+mlx5_flow_isolate(struct rte_eth_dev *dev,
+		  int enable,
+		  struct rte_flow_error *error)
+{
+	struct priv *priv = dev->data->dev_private;
+
+	if (dev->data->dev_started) {
+		rte_flow_error_set(error, EBUSY,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				   NULL,
+				   "port must be stopped first");
+		return -rte_errno;
+	}
+	priv->isolated = !!enable;
+	if (enable)
+		dev->dev_ops = &mlx5_dev_ops_isolate;
+	else
+		dev->dev_ops = &mlx5_dev_ops;
+	return 0;
+}
+
+/**
+ * Query flow counter.
+ *
+ * @param flow
+ *   Pointer to the flow.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_query_count(struct rte_flow *flow __rte_unused,
+		      void *data __rte_unused,
+		      struct rte_flow_error *error)
+{
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+	if (flow->modifier & MLX5_FLOW_MOD_COUNT) {
+		struct rte_flow_query_count *qc = data;
+		uint64_t counters[2] = {0, 0};
+		struct ibv_query_counter_set_attr query_cs_attr = {
+			.cs = flow->counter->cs,
+			.query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
+		};
+		struct ibv_counter_set_data query_out = {
+			.out = counters,
+			.outlen = 2 * sizeof(uint64_t),
+		};
+		int err = mlx5_glue->query_counter_set(&query_cs_attr,
+						       &query_out);
+
+		if (err)
+			return rte_flow_error_set
+				(error, err,
+				 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				 NULL,
+				 "cannot read counter");
+		qc->hits_set = 1;
+		qc->bytes_set = 1;
+		qc->hits = counters[0] - flow->counter->hits;
+		qc->bytes = counters[1] - flow->counter->bytes;
+		if (qc->reset) {
+			flow->counter->hits = counters[0];
+			flow->counter->bytes = counters[1];
+		}
+		return 0;
+	}
+	return rte_flow_error_set(error, ENOTSUP,
+				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				  NULL,
+				  "flow does not have counter");
+#endif
+	return rte_flow_error_set(error, ENOTSUP,
+				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				  NULL,
+				  "counters are not available");
+}
+
+/**
+ * Query a flows.
+ *
+ * @see rte_flow_query()
+ * @see rte_flow_ops
+ */
+int
+mlx5_flow_query(struct rte_eth_dev *dev __rte_unused,
+		struct rte_flow *flow,
+		const struct rte_flow_action *actions,
+		void *data,
+		struct rte_flow_error *error)
+{
+	int ret = 0;
+
+	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
+		switch (actions->type) {
+		case RTE_FLOW_ACTION_TYPE_VOID:
+			break;
+		case RTE_FLOW_ACTION_TYPE_COUNT:
+			ret = mlx5_flow_query_count(flow, data, error);
+			break;
+		default:
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  actions,
+						  "action not supported");
+		}
+		if (ret < 0)
+			return ret;
+	}
+	return 0;
+}
+
+/**
+ * Convert a flow director filter to a generic flow.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param fdir_filter
+ *   Flow director filter to add.
+ * @param attributes
+ *   Generic flow parameters structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_fdir_filter_convert(struct rte_eth_dev *dev,
+			 const struct rte_eth_fdir_filter *fdir_filter,
+			 struct mlx5_fdir *attributes)
+{
+	struct priv *priv = dev->data->dev_private;
+	const struct rte_eth_fdir_input *input = &fdir_filter->input;
+	const struct rte_eth_fdir_masks *mask =
+		&dev->data->dev_conf.fdir_conf.mask;
+
+	/* Validate queue number. */
+	if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
+		DRV_LOG(ERR, "port %u invalid queue number %d",
+			dev->data->port_id, fdir_filter->action.rx_queue);
+		rte_errno = EINVAL;
+		return -rte_errno;
+	}
+	attributes->attr.ingress = 1;
+	attributes->items[0] = (struct rte_flow_item) {
+		.type = RTE_FLOW_ITEM_TYPE_ETH,
+		.spec = &attributes->l2,
+		.mask = &attributes->l2_mask,
+	};
+	switch (fdir_filter->action.behavior) {
+	case RTE_ETH_FDIR_ACCEPT:
+		attributes->actions[0] = (struct rte_flow_action){
+			.type = RTE_FLOW_ACTION_TYPE_QUEUE,
+			.conf = &attributes->queue,
+		};
+		break;
+	case RTE_ETH_FDIR_REJECT:
+		attributes->actions[0] = (struct rte_flow_action){
+			.type = RTE_FLOW_ACTION_TYPE_DROP,
+		};
+		break;
+	default:
+		DRV_LOG(ERR, "port %u invalid behavior %d",
+			dev->data->port_id,
+			fdir_filter->action.behavior);
+		rte_errno = ENOTSUP;
+		return -rte_errno;
+	}
+	attributes->queue.index = fdir_filter->action.rx_queue;
+	/* Handle L3. */
+	switch (fdir_filter->input.flow_type) {
+	case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
+	case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
+	case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
+		attributes->l3.ipv4.hdr = (struct ipv4_hdr){
+			.src_addr = input->flow.ip4_flow.src_ip,
+			.dst_addr = input->flow.ip4_flow.dst_ip,
+			.time_to_live = input->flow.ip4_flow.ttl,
+			.type_of_service = input->flow.ip4_flow.tos,
+			.next_proto_id = input->flow.ip4_flow.proto,
+		};
+		attributes->l3_mask.ipv4.hdr = (struct ipv4_hdr){
+			.src_addr = mask->ipv4_mask.src_ip,
+			.dst_addr = mask->ipv4_mask.dst_ip,
+			.time_to_live = mask->ipv4_mask.ttl,
+			.type_of_service = mask->ipv4_mask.tos,
+			.next_proto_id = mask->ipv4_mask.proto,
+		};
+		attributes->items[1] = (struct rte_flow_item){
+			.type = RTE_FLOW_ITEM_TYPE_IPV4,
+			.spec = &attributes->l3,
+			.mask = &attributes->l3_mask,
+		};
+		break;
+	case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
+	case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
+	case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
+		attributes->l3.ipv6.hdr = (struct ipv6_hdr){
+			.hop_limits = input->flow.ipv6_flow.hop_limits,
+			.proto = input->flow.ipv6_flow.proto,
+		};
+
+		memcpy(attributes->l3.ipv6.hdr.src_addr,
+		       input->flow.ipv6_flow.src_ip,
+		       RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
+		memcpy(attributes->l3.ipv6.hdr.dst_addr,
+		       input->flow.ipv6_flow.dst_ip,
+		       RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
+		memcpy(attributes->l3_mask.ipv6.hdr.src_addr,
+		       mask->ipv6_mask.src_ip,
+		       RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr));
+		memcpy(attributes->l3_mask.ipv6.hdr.dst_addr,
+		       mask->ipv6_mask.dst_ip,
+		       RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr));
+		attributes->items[1] = (struct rte_flow_item){
+			.type = RTE_FLOW_ITEM_TYPE_IPV6,
+			.spec = &attributes->l3,
+			.mask = &attributes->l3_mask,
+		};
+		break;
+	default:
+		DRV_LOG(ERR, "port %u invalid flow type%d",
+			dev->data->port_id, fdir_filter->input.flow_type);
+		rte_errno = ENOTSUP;
+		return -rte_errno;
+	}
+	/* Handle L4. */
+	switch (fdir_filter->input.flow_type) {
+	case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
+		attributes->l4.udp.hdr = (struct udp_hdr){
+			.src_port = input->flow.udp4_flow.src_port,
+			.dst_port = input->flow.udp4_flow.dst_port,
+		};
+		attributes->l4_mask.udp.hdr = (struct udp_hdr){
+			.src_port = mask->src_port_mask,
+			.dst_port = mask->dst_port_mask,
+		};
+		attributes->items[2] = (struct rte_flow_item){
+			.type = RTE_FLOW_ITEM_TYPE_UDP,
+			.spec = &attributes->l4,
+			.mask = &attributes->l4_mask,
+		};
+		break;
+	case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
+		attributes->l4.tcp.hdr = (struct tcp_hdr){
+			.src_port = input->flow.tcp4_flow.src_port,
+			.dst_port = input->flow.tcp4_flow.dst_port,
+		};
+		attributes->l4_mask.tcp.hdr = (struct tcp_hdr){
+			.src_port = mask->src_port_mask,
+			.dst_port = mask->dst_port_mask,
+		};
+		attributes->items[2] = (struct rte_flow_item){
+			.type = RTE_FLOW_ITEM_TYPE_TCP,
+			.spec = &attributes->l4,
+			.mask = &attributes->l4_mask,
+		};
+		break;
+	case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
+		attributes->l4.udp.hdr = (struct udp_hdr){
+			.src_port = input->flow.udp6_flow.src_port,
+			.dst_port = input->flow.udp6_flow.dst_port,
+		};
+		attributes->l4_mask.udp.hdr = (struct udp_hdr){
+			.src_port = mask->src_port_mask,
+			.dst_port = mask->dst_port_mask,
+		};
+		attributes->items[2] = (struct rte_flow_item){
+			.type = RTE_FLOW_ITEM_TYPE_UDP,
+			.spec = &attributes->l4,
+			.mask = &attributes->l4_mask,
+		};
+		break;
+	case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
+		attributes->l4.tcp.hdr = (struct tcp_hdr){
+			.src_port = input->flow.tcp6_flow.src_port,
+			.dst_port = input->flow.tcp6_flow.dst_port,
+		};
+		attributes->l4_mask.tcp.hdr = (struct tcp_hdr){
+			.src_port = mask->src_port_mask,
+			.dst_port = mask->dst_port_mask,
+		};
+		attributes->items[2] = (struct rte_flow_item){
+			.type = RTE_FLOW_ITEM_TYPE_TCP,
+			.spec = &attributes->l4,
+			.mask = &attributes->l4_mask,
+		};
+		break;
+	case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
+	case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
+		break;
+	default:
+		DRV_LOG(ERR, "port %u invalid flow type%d",
+			dev->data->port_id, fdir_filter->input.flow_type);
+		rte_errno = ENOTSUP;
+		return -rte_errno;
+	}
+	return 0;
+}
+
+/**
+ * Add new flow director filter and store it in list.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param fdir_filter
+ *   Flow director filter to add.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_fdir_filter_add(struct rte_eth_dev *dev,
+		     const struct rte_eth_fdir_filter *fdir_filter)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_fdir attributes = {
+		.attr.group = 0,
+		.l2_mask = {
+			.dst.addr_bytes = "\x00\x00\x00\x00\x00\x00",
+			.src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
+			.type = 0,
+		},
+	};
+	struct rte_flow_error error;
+	struct rte_flow *flow;
+	int ret;
+
+	ret = mlx5_fdir_filter_convert(dev, fdir_filter, &attributes);
+	if (ret)
+		return ret;
+	flow = mlx5_flow_list_create(dev, &priv->flows, &attributes.attr,
+				     attributes.items, attributes.actions,
+				     &error);
+	if (flow) {
+		DRV_LOG(DEBUG, "port %u FDIR created %p", dev->data->port_id,
+			(void *)flow);
+		return 0;
+	}
+	return -rte_errno;
+}
+
+/**
+ * Delete specific filter.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param fdir_filter
+ *   Filter to be deleted.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_fdir_filter_delete(struct rte_eth_dev *dev __rte_unused,
+			const struct rte_eth_fdir_filter *fdir_filter
+			__rte_unused)
+{
+	rte_errno = ENOTSUP;
+	return -rte_errno;
+}
+
+/**
+ * Update queue for specific filter.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param fdir_filter
+ *   Filter to be updated.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_fdir_filter_update(struct rte_eth_dev *dev,
+			const struct rte_eth_fdir_filter *fdir_filter)
+{
+	int ret;
+
+	ret = mlx5_fdir_filter_delete(dev, fdir_filter);
+	if (ret)
+		return ret;
+	return mlx5_fdir_filter_add(dev, fdir_filter);
+}
+
+/**
+ * Flush all filters.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ */
+static void
+mlx5_fdir_filter_flush(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+
+	mlx5_flow_list_flush(dev, &priv->flows);
+}
+
+/**
+ * Get flow director information.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param[out] fdir_info
+ *   Resulting flow director information.
+ */
+static void
+mlx5_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info)
+{
+	struct rte_eth_fdir_masks *mask =
+		&dev->data->dev_conf.fdir_conf.mask;
+
+	fdir_info->mode = dev->data->dev_conf.fdir_conf.mode;
+	fdir_info->guarant_spc = 0;
+	rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask));
+	fdir_info->max_flexpayload = 0;
+	fdir_info->flow_types_mask[0] = 0;
+	fdir_info->flex_payload_unit = 0;
+	fdir_info->max_flex_payload_segment_num = 0;
+	fdir_info->flex_payload_limit = 0;
+	memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf));
+}
+
+/**
+ * Deal with flow director operations.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param filter_op
+ *   Operation to perform.
+ * @param arg
+ *   Pointer to operation-specific structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op,
+		    void *arg)
+{
+	enum rte_fdir_mode fdir_mode =
+		dev->data->dev_conf.fdir_conf.mode;
+
+	if (filter_op == RTE_ETH_FILTER_NOP)
+		return 0;
+	if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
+	    fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
+		DRV_LOG(ERR, "port %u flow director mode %d not supported",
+			dev->data->port_id, fdir_mode);
+		rte_errno = EINVAL;
+		return -rte_errno;
+	}
+	switch (filter_op) {
+	case RTE_ETH_FILTER_ADD:
+		return mlx5_fdir_filter_add(dev, arg);
+	case RTE_ETH_FILTER_UPDATE:
+		return mlx5_fdir_filter_update(dev, arg);
+	case RTE_ETH_FILTER_DELETE:
+		return mlx5_fdir_filter_delete(dev, arg);
+	case RTE_ETH_FILTER_FLUSH:
+		mlx5_fdir_filter_flush(dev);
+		break;
+	case RTE_ETH_FILTER_INFO:
+		mlx5_fdir_info_get(dev, arg);
+		break;
+	default:
+		DRV_LOG(DEBUG, "port %u unknown operation %u",
+			dev->data->port_id, filter_op);
+		rte_errno = EINVAL;
+		return -rte_errno;
+	}
+	return 0;
+}
+
+/**
+ * Manage filter operations.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param filter_type
+ *   Filter type.
+ * @param filter_op
+ *   Operation to perform.
+ * @param arg
+ *   Pointer to operation-specific structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
+		     enum rte_filter_type filter_type,
+		     enum rte_filter_op filter_op,
+		     void *arg)
+{
+	switch (filter_type) {
+	case RTE_ETH_FILTER_GENERIC:
+		if (filter_op != RTE_ETH_FILTER_GET) {
+			rte_errno = EINVAL;
+			return -rte_errno;
+		}
+		*(const void **)arg = &mlx5_flow_ops;
+		return 0;
+	case RTE_ETH_FILTER_FDIR:
+		return mlx5_fdir_ctrl_func(dev, filter_op, arg);
+	default:
+		DRV_LOG(ERR, "port %u filter type (%d) not supported",
+			dev->data->port_id, filter_type);
+		rte_errno = ENOTSUP;
+		return -rte_errno;
+	}
+	return 0;
+}
diff --git a/src/spdk/dpdk/drivers/net/mlx5/mlx5_glue.c b/src/spdk/dpdk/drivers/net/mlx5/mlx5_glue.c
new file mode 100644
index 00000000..84f9492a
--- /dev/null
+++ b/src/spdk/dpdk/drivers/net/mlx5/mlx5_glue.c
@@ -0,0 +1,395 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 6WIND S.A.
+ * Copyright 2018 Mellanox Technologies, Ltd
+ */
+
+#include <errno.h>
+#include <stdalign.h>
+#include <stddef.h>
+#include <stdint.h>
+
+/*
+ * Not needed by this file; included to work around the lack of off_t
+ * definition for mlx5dv.h with unpatched rdma-core versions.
+ */
+#include <sys/types.h>
+
+/* Verbs headers do not support -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <infiniband/mlx5dv.h>
+#include <infiniband/verbs.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+#include <rte_config.h>
+
+#include "mlx5_autoconf.h"
+#include "mlx5_glue.h"
+
+static int
+mlx5_glue_fork_init(void)
+{
+	return ibv_fork_init();
+}
+
+static struct ibv_pd *
+mlx5_glue_alloc_pd(struct ibv_context *context)
+{
+	return ibv_alloc_pd(context);
+}
+
+static int
+mlx5_glue_dealloc_pd(struct ibv_pd *pd)
+{
+	return ibv_dealloc_pd(pd);
+}
+
+static struct ibv_device **
+mlx5_glue_get_device_list(int *num_devices)
+{
+	return ibv_get_device_list(num_devices);
+}
+
+static void
+mlx5_glue_free_device_list(struct ibv_device **list)
+{
+	ibv_free_device_list(list);
+}
+
+static struct ibv_context *
+mlx5_glue_open_device(struct ibv_device *device)
+{
+	return ibv_open_device(device);
+}
+
+static int
+mlx5_glue_close_device(struct ibv_context *context)
+{
+	return ibv_close_device(context);
+}
+
+static int
+mlx5_glue_query_device(struct ibv_context *context,
+		       struct ibv_device_attr *device_attr)
+{
+	return ibv_query_device(context, device_attr);
+}
+
+static int
+mlx5_glue_query_device_ex(struct ibv_context *context,
+			  const struct ibv_query_device_ex_input *input,
+			  struct ibv_device_attr_ex *attr)
+{
+	return ibv_query_device_ex(context, input, attr);
+}
+
+static int
+mlx5_glue_query_port(struct ibv_context *context, uint8_t port_num,
+		     struct ibv_port_attr *port_attr)
+{
+	return ibv_query_port(context, port_num, port_attr);
+}
+
+static struct ibv_comp_channel *
+mlx5_glue_create_comp_channel(struct ibv_context *context)
+{
+	return ibv_create_comp_channel(context);
+}
+
+static int
+mlx5_glue_destroy_comp_channel(struct ibv_comp_channel *channel)
+{
+	return ibv_destroy_comp_channel(channel);
+}
+
+static struct ibv_cq *
+mlx5_glue_create_cq(struct ibv_context *context, int cqe, void *cq_context,
+		    struct ibv_comp_channel *channel, int comp_vector)
+{
+	return ibv_create_cq(context, cqe, cq_context, channel, comp_vector);
+}
+
+static int
+mlx5_glue_destroy_cq(struct ibv_cq *cq)
+{
+	return ibv_destroy_cq(cq);
+}
+
+static int
+mlx5_glue_get_cq_event(struct ibv_comp_channel *channel, struct ibv_cq **cq,
+		       void **cq_context)
+{
+	return ibv_get_cq_event(channel, cq, cq_context);
+}
+
+static void
+mlx5_glue_ack_cq_events(struct ibv_cq *cq, unsigned int nevents)
+{
+	ibv_ack_cq_events(cq, nevents);
+}
+
+static struct ibv_rwq_ind_table *
+mlx5_glue_create_rwq_ind_table(struct ibv_context *context,
+			       struct ibv_rwq_ind_table_init_attr *init_attr)
+{
+	return ibv_create_rwq_ind_table(context, init_attr);
+}
+
+static int
+mlx5_glue_destroy_rwq_ind_table(struct ibv_rwq_ind_table *rwq_ind_table)
+{
+	return ibv_destroy_rwq_ind_table(rwq_ind_table);
+}
+
+static struct ibv_wq *
+mlx5_glue_create_wq(struct ibv_context *context,
+		    struct ibv_wq_init_attr *wq_init_attr)
+{
+	return ibv_create_wq(context, wq_init_attr);
+}
+
+static int
+mlx5_glue_destroy_wq(struct ibv_wq *wq)
+{
+	return ibv_destroy_wq(wq);
+}
+static int
+mlx5_glue_modify_wq(struct ibv_wq *wq, struct ibv_wq_attr *wq_attr)
+{
+	return ibv_modify_wq(wq, wq_attr);
+}
+
+static struct ibv_flow *
+mlx5_glue_create_flow(struct ibv_qp *qp, struct ibv_flow_attr *flow)
+{
+	return ibv_create_flow(qp, flow);
+}
+
+static int
+mlx5_glue_destroy_flow(struct ibv_flow *flow_id)
+{
+	return ibv_destroy_flow(flow_id);
+}
+
+static struct ibv_qp *
+mlx5_glue_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr)
+{
+	return ibv_create_qp(pd, qp_init_attr);
+}
+
+static struct ibv_qp *
+mlx5_glue_create_qp_ex(struct ibv_context *context,
+		       struct ibv_qp_init_attr_ex *qp_init_attr_ex)
+{
+	return ibv_create_qp_ex(context, qp_init_attr_ex);
+}
+
+static int
+mlx5_glue_destroy_qp(struct ibv_qp *qp)
+{
+	return ibv_destroy_qp(qp);
+}
+
+static int
+mlx5_glue_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask)
+{
+	return ibv_modify_qp(qp, attr, attr_mask);
+}
+
+static struct ibv_mr *
+mlx5_glue_reg_mr(struct ibv_pd *pd, void *addr, size_t length, int access)
+{
+	return ibv_reg_mr(pd, addr, length, access);
+}
+
+static int
+mlx5_glue_dereg_mr(struct ibv_mr *mr)
+{
+	return ibv_dereg_mr(mr);
+}
+
+static struct ibv_counter_set *
+mlx5_glue_create_counter_set(struct ibv_context *context,
+			     struct ibv_counter_set_init_attr *init_attr)
+{
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+	(void)context;
+	(void)init_attr;
+	return NULL;
+#else
+	return ibv_create_counter_set(context, init_attr);
+#endif
+}
+
+static int
+mlx5_glue_destroy_counter_set(struct ibv_counter_set *cs)
+{
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+	(void)cs;
+	return ENOTSUP;
+#else
+	return ibv_destroy_counter_set(cs);
+#endif
+}
+
+static int
+mlx5_glue_describe_counter_set(struct ibv_context *context,
+			       uint16_t counter_set_id,
+			       struct ibv_counter_set_description *cs_desc)
+{
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+	(void)context;
+	(void)counter_set_id;
+	(void)cs_desc;
+	return ENOTSUP;
+#else
+	return ibv_describe_counter_set(context, counter_set_id, cs_desc);
+#endif
+}
+
+static int
+mlx5_glue_query_counter_set(struct ibv_query_counter_set_attr *query_attr,
+			    struct ibv_counter_set_data *cs_data)
+{
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+	(void)query_attr;
+	(void)cs_data;
+	return ENOTSUP;
+#else
+	return ibv_query_counter_set(query_attr, cs_data);
+#endif
+}
+
+static void
+mlx5_glue_ack_async_event(struct ibv_async_event *event)
+{
+	ibv_ack_async_event(event);
+}
+
+static int
+mlx5_glue_get_async_event(struct ibv_context *context,
+			  struct ibv_async_event *event)
+{
+	return ibv_get_async_event(context, event);
+}
+
+static const char *
+mlx5_glue_port_state_str(enum ibv_port_state port_state)
+{
+	return ibv_port_state_str(port_state);
+}
+
+static struct ibv_cq *
+mlx5_glue_cq_ex_to_cq(struct ibv_cq_ex *cq)
+{
+	return ibv_cq_ex_to_cq(cq);
+}
+
+static struct ibv_cq_ex *
+mlx5_glue_dv_create_cq(struct ibv_context *context,
+		       struct ibv_cq_init_attr_ex *cq_attr,
+		       struct mlx5dv_cq_init_attr *mlx5_cq_attr)
+{
+	return mlx5dv_create_cq(context, cq_attr, mlx5_cq_attr);
+}
+
+static struct ibv_wq *
+mlx5_glue_dv_create_wq(struct ibv_context *context,
+		       struct ibv_wq_init_attr *wq_attr,
+		       struct mlx5dv_wq_init_attr *mlx5_wq_attr)
+{
+#ifndef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
+	(void)context;
+	(void)wq_attr;
+	(void)mlx5_wq_attr;
+	return NULL;
+#else
+	return mlx5dv_create_wq(context, wq_attr, mlx5_wq_attr);
+#endif
+}
+
+static int
+mlx5_glue_dv_query_device(struct ibv_context *ctx,
+			  struct mlx5dv_context *attrs_out)
+{
+	return mlx5dv_query_device(ctx, attrs_out);
+}
+
+static int
+mlx5_glue_dv_set_context_attr(struct ibv_context *ibv_ctx,
+			      enum mlx5dv_set_ctx_attr_type type, void *attr)
+{
+	return mlx5dv_set_context_attr(ibv_ctx, type, attr);
+}
+
+static int
+mlx5_glue_dv_init_obj(struct mlx5dv_obj *obj, uint64_t obj_type)
+{
+	return mlx5dv_init_obj(obj, obj_type);
+}
+
+static struct ibv_qp *
+mlx5_glue_dv_create_qp(struct ibv_context *context,
+		       struct ibv_qp_init_attr_ex *qp_init_attr_ex,
+		       struct mlx5dv_qp_init_attr *dv_qp_init_attr)
+{
+#ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
+	return mlx5dv_create_qp(context, qp_init_attr_ex, dv_qp_init_attr);
+#else
+	(void)context;
+	(void)qp_init_attr_ex;
+	(void)dv_qp_init_attr;
+	return NULL;
+#endif
+}
+
+alignas(RTE_CACHE_LINE_SIZE)
+const struct mlx5_glue *mlx5_glue = &(const struct mlx5_glue){
+	.version = MLX5_GLUE_VERSION,
+	.fork_init = mlx5_glue_fork_init,
+	.alloc_pd = mlx5_glue_alloc_pd,
+	.dealloc_pd = mlx5_glue_dealloc_pd,
+	.get_device_list = mlx5_glue_get_device_list,
+	.free_device_list = mlx5_glue_free_device_list,
+	.open_device = mlx5_glue_open_device,
+	.close_device = mlx5_glue_close_device,
+	.query_device = mlx5_glue_query_device,
+	.query_device_ex = mlx5_glue_query_device_ex,
+	.query_port = mlx5_glue_query_port,
+	.create_comp_channel = mlx5_glue_create_comp_channel,
+	.destroy_comp_channel = mlx5_glue_destroy_comp_channel,
+	.create_cq = mlx5_glue_create_cq,
+	.destroy_cq = mlx5_glue_destroy_cq,
+	.get_cq_event = mlx5_glue_get_cq_event,
+	.ack_cq_events = mlx5_glue_ack_cq_events,
+	.create_rwq_ind_table = mlx5_glue_create_rwq_ind_table,
+	.destroy_rwq_ind_table = mlx5_glue_destroy_rwq_ind_table,
+	.create_wq = mlx5_glue_create_wq,
+	.destroy_wq = mlx5_glue_destroy_wq,
+	.modify_wq = mlx5_glue_modify_wq,
+	.create_flow = mlx5_glue_create_flow,
+	.destroy_flow = mlx5_glue_destroy_flow,
+	.create_qp = mlx5_glue_create_qp,
+	.create_qp_ex = mlx5_glue_create_qp_ex,
+	.destroy_qp = mlx5_glue_destroy_qp,
+	.modify_qp = mlx5_glue_modify_qp,
+	.reg_mr = mlx5_glue_reg_mr,
+	.dereg_mr = mlx5_glue_dereg_mr,
+	.create_counter_set = mlx5_glue_create_counter_set,
+	.destroy_counter_set = mlx5_glue_destroy_counter_set,
+	.describe_counter_set = mlx5_glue_describe_counter_set,
+	.query_counter_set = mlx5_glue_query_counter_set,
+	.ack_async_event = mlx5_glue_ack_async_event,
+	.get_async_event = mlx5_glue_get_async_event,
+	.port_state_str = mlx5_glue_port_state_str,
+	.cq_ex_to_cq = mlx5_glue_cq_ex_to_cq,
+	.dv_create_cq = mlx5_glue_dv_create_cq,
+	.dv_create_wq = mlx5_glue_dv_create_wq,
+	.dv_query_device = mlx5_glue_dv_query_device,
+	.dv_set_context_attr = mlx5_glue_dv_set_context_attr,
+	.dv_init_obj = mlx5_glue_dv_init_obj,
+	.dv_create_qp = mlx5_glue_dv_create_qp,
+};
diff --git a/src/spdk/dpdk/drivers/net/mlx5/mlx5_glue.h b/src/spdk/dpdk/drivers/net/mlx5/mlx5_glue.h
new file mode 100644
index 00000000..e584d367
--- /dev/null
+++ b/src/spdk/dpdk/drivers/net/mlx5/mlx5_glue.h
@@ -0,0 +1,129 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 6WIND S.A.
+ * Copyright 2018 Mellanox Technologies, Ltd
+ */
+
+#ifndef MLX5_GLUE_H_
+#define MLX5_GLUE_H_
+
+#include <stddef.h>
+#include <stdint.h>
+
+/* Verbs headers do not support -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <infiniband/mlx5dv.h>
+#include <infiniband/verbs.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+#ifndef MLX5_GLUE_VERSION
+#define MLX5_GLUE_VERSION ""
+#endif
+
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+struct ibv_counter_set;
+struct ibv_counter_set_data;
+struct ibv_counter_set_description;
+struct ibv_counter_set_init_attr;
+struct ibv_query_counter_set_attr;
+#endif
+
+#ifndef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
+struct mlx5dv_qp_init_attr;
+#endif
+
+#ifndef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
+struct mlx5dv_wq_init_attr;
+#endif
+
+/* LIB_GLUE_VERSION must be updated every time this structure is modified. */
+struct mlx5_glue {
+	const char *version;
+	int (*fork_init)(void);
+	struct ibv_pd *(*alloc_pd)(struct ibv_context *context);
+	int (*dealloc_pd)(struct ibv_pd *pd);
+	struct ibv_device **(*get_device_list)(int *num_devices);
+	void (*free_device_list)(struct ibv_device **list);
+	struct ibv_context *(*open_device)(struct ibv_device *device);
+	int (*close_device)(struct ibv_context *context);
+	int (*query_device)(struct ibv_context *context,
+			    struct ibv_device_attr *device_attr);
+	int (*query_device_ex)(struct ibv_context *context,
+			       const struct ibv_query_device_ex_input *input,
+			       struct ibv_device_attr_ex *attr);
+	int (*query_port)(struct ibv_context *context, uint8_t port_num,
+			  struct ibv_port_attr *port_attr);
+	struct ibv_comp_channel *(*create_comp_channel)
+		(struct ibv_context *context);
+	int (*destroy_comp_channel)(struct ibv_comp_channel *channel);
+	struct ibv_cq *(*create_cq)(struct ibv_context *context, int cqe,
+				    void *cq_context,
+				    struct ibv_comp_channel *channel,
+				    int comp_vector);
+	int (*destroy_cq)(struct ibv_cq *cq);
+	int (*get_cq_event)(struct ibv_comp_channel *channel,
+			    struct ibv_cq **cq, void **cq_context);
+	void (*ack_cq_events)(struct ibv_cq *cq, unsigned int nevents);
+	struct ibv_rwq_ind_table *(*create_rwq_ind_table)
+		(struct ibv_context *context,
+		 struct ibv_rwq_ind_table_init_attr *init_attr);
+	int (*destroy_rwq_ind_table)(struct ibv_rwq_ind_table *rwq_ind_table);
+	struct ibv_wq *(*create_wq)(struct ibv_context *context,
+				    struct ibv_wq_init_attr *wq_init_attr);
+	int (*destroy_wq)(struct ibv_wq *wq);
+	int (*modify_wq)(struct ibv_wq *wq, struct ibv_wq_attr *wq_attr);
+	struct ibv_flow *(*create_flow)(struct ibv_qp *qp,
+					struct ibv_flow_attr *flow);
+	int (*destroy_flow)(struct ibv_flow *flow_id);
+	struct ibv_qp *(*create_qp)(struct ibv_pd *pd,
+				    struct ibv_qp_init_attr *qp_init_attr);
+	struct ibv_qp *(*create_qp_ex)
+		(struct ibv_context *context,
+		 struct ibv_qp_init_attr_ex *qp_init_attr_ex);
+	int (*destroy_qp)(struct ibv_qp *qp);
+	int (*modify_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr,
+			 int attr_mask);
+	struct ibv_mr *(*reg_mr)(struct ibv_pd *pd, void *addr,
+				 size_t length, int access);
+	int (*dereg_mr)(struct ibv_mr *mr);
+	struct ibv_counter_set *(*create_counter_set)
+		(struct ibv_context *context,
+		 struct ibv_counter_set_init_attr *init_attr);
+	int (*destroy_counter_set)(struct ibv_counter_set *cs);
+	int (*describe_counter_set)
+		(struct ibv_context *context,
+		 uint16_t counter_set_id,
+		 struct ibv_counter_set_description *cs_desc);
+	int (*query_counter_set)(struct ibv_query_counter_set_attr *query_attr,
+				 struct ibv_counter_set_data *cs_data);
+	void (*ack_async_event)(struct ibv_async_event *event);
+	int (*get_async_event)(struct ibv_context *context,
+			       struct ibv_async_event *event);
+	const char *(*port_state_str)(enum ibv_port_state port_state);
+	struct ibv_cq *(*cq_ex_to_cq)(struct ibv_cq_ex *cq);
+	struct ibv_cq_ex *(*dv_create_cq)
+		(struct ibv_context *context,
+		 struct ibv_cq_init_attr_ex *cq_attr,
+		 struct mlx5dv_cq_init_attr *mlx5_cq_attr);
+	struct ibv_wq *(*dv_create_wq)
+		(struct ibv_context *context,
+		 struct ibv_wq_init_attr *wq_attr,
+		 struct mlx5dv_wq_init_attr *mlx5_wq_attr);
+	int (*dv_query_device)(struct ibv_context *ctx_in,
+			       struct mlx5dv_context *attrs_out);
+	int (*dv_set_context_attr)(struct ibv_context *ibv_ctx,
+				   enum mlx5dv_set_ctx_attr_type type,
+				   void *attr);
+	int (*dv_init_obj)(struct mlx5dv_obj *obj, uint64_t obj_type);
+	struct ibv_qp *(*dv_create_qp)
+		(struct ibv_context *context,
+		 struct ibv_qp_init_attr_ex *qp_init_attr_ex,
+		 struct mlx5dv_qp_init_attr *dv_qp_init_attr);
+};
+
+const struct mlx5_glue *mlx5_glue;
+
+#endif /* MLX5_GLUE_H_ */
diff --git a/src/spdk/dpdk/drivers/net/mlx5/mlx5_mac.c b/src/spdk/dpdk/drivers/net/mlx5/mlx5_mac.c
new file mode 100644
index 00000000..12ee37f5
--- /dev/null
+++ b/src/spdk/dpdk/drivers/net/mlx5/mlx5_mac.c
@@ -0,0 +1,232 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2015 6WIND S.A.
+ * Copyright 2015 Mellanox Technologies, Ltd
+ */
+
+#include <stddef.h>
+#include <assert.h>
+#include <stdint.h>
+#include <string.h>
+#include <inttypes.h>
+#include <errno.h>
+#include <netinet/in.h>
+#include <sys/ioctl.h>
+#include <arpa/inet.h>
+
+/* Verbs header. */
+/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <infiniband/verbs.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+#include <rte_ether.h>
+#include <rte_ethdev_driver.h>
+#include <rte_common.h>
+
+#include "mlx5.h"
+#include "mlx5_utils.h"
+#include "mlx5_rxtx.h"
+#include "mlx5_defs.h"
+
+/**
+ * Get MAC address by querying netdevice.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ * @param[out] mac
+ *   MAC address output buffer.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_get_mac(struct rte_eth_dev *dev, uint8_t (*mac)[ETHER_ADDR_LEN])
+{
+	struct ifreq request;
+	int ret;
+
+	ret = mlx5_ifreq(dev, SIOCGIFHWADDR, &request, 0);
+	if (ret)
+		return ret;
+	memcpy(mac, request.ifr_hwaddr.sa_data, ETHER_ADDR_LEN);
+	return 0;
+}
+
+/**
+ * Remove a MAC address from the internal array.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param index
+ *   MAC address index.
+ */
+static void
+mlx5_internal_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
+{
+	struct priv *priv = dev->data->dev_private;
+	const int vf = priv->config.vf;
+
+	assert(index < MLX5_MAX_MAC_ADDRESSES);
+	if (is_zero_ether_addr(&dev->data->mac_addrs[index]))
+		return;
+	if (vf)
+		mlx5_nl_mac_addr_remove(dev, &dev->data->mac_addrs[index],
+					index);
+	memset(&dev->data->mac_addrs[index], 0, sizeof(struct ether_addr));
+}
+
+/**
+ * Adds a MAC address to the internal array.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param mac_addr
+ *   MAC address to register.
+ * @param index
+ *   MAC address index.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_internal_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac,
+			   uint32_t index)
+{
+	struct priv *priv = dev->data->dev_private;
+	const int vf = priv->config.vf;
+	unsigned int i;
+
+	assert(index < MLX5_MAX_MAC_ADDRESSES);
+	if (is_zero_ether_addr(mac)) {
+		rte_errno = EINVAL;
+		return -rte_errno;
+	}
+	/* First, make sure this address isn't already configured. */
+	for (i = 0; (i != MLX5_MAX_MAC_ADDRESSES); ++i) {
+		/* Skip this index, it's going to be reconfigured. */
+		if (i == index)
+			continue;
+		if (memcmp(&dev->data->mac_addrs[i], mac, sizeof(*mac)))
+			continue;
+		/* Address already configured elsewhere, return with error. */
+		rte_errno = EADDRINUSE;
+		return -rte_errno;
+	}
+	if (vf) {
+		int ret = mlx5_nl_mac_addr_add(dev, mac, index);
+
+		if (ret)
+			return ret;
+	}
+	dev->data->mac_addrs[index] = *mac;
+	return 0;
+}
+
+/**
+ * DPDK callback to remove a MAC address.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param index
+ *   MAC address index.
+ */
+void
+mlx5_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
+{
+	int ret;
+
+	if (index >= MLX5_MAX_UC_MAC_ADDRESSES)
+		return;
+	mlx5_internal_mac_addr_remove(dev, index);
+	if (!dev->data->promiscuous) {
+		ret = mlx5_traffic_restart(dev);
+		if (ret)
+			DRV_LOG(ERR, "port %u cannot restart traffic: %s",
+				dev->data->port_id, strerror(rte_errno));
+	}
+}
+
+/**
+ * DPDK callback to add a MAC address.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param mac_addr
+ *   MAC address to register.
+ * @param index
+ *   MAC address index.
+ * @param vmdq
+ *   VMDq pool index to associate address with (ignored).
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac,
+		  uint32_t index, uint32_t vmdq __rte_unused)
+{
+	int ret;
+
+	if (index >= MLX5_MAX_UC_MAC_ADDRESSES) {
+		rte_errno = EINVAL;
+		return -rte_errno;
+	}
+	ret = mlx5_internal_mac_addr_add(dev, mac, index);
+	if (ret < 0)
+		return ret;
+	if (!dev->data->promiscuous)
+		return mlx5_traffic_restart(dev);
+	return 0;
+}
+
+/**
+ * DPDK callback to set primary MAC address.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param mac_addr
+ *   MAC address to register.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
+{
+	DRV_LOG(DEBUG, "port %u setting primary MAC address",
+		dev->data->port_id);
+	return mlx5_mac_addr_add(dev, mac_addr, 0, 0);
+}
+
+/**
+ * DPDK callback to set multicast addresses list.
+ *
+ * @see rte_eth_dev_set_mc_addr_list()
+ */
+int
+mlx5_set_mc_addr_list(struct rte_eth_dev *dev,
+		      struct ether_addr *mc_addr_set, uint32_t nb_mc_addr)
+{
+	uint32_t i;
+	int ret;
+
+	if (nb_mc_addr >= MLX5_MAX_MC_MAC_ADDRESSES) {
+		rte_errno = ENOSPC;
+		return -rte_errno;
+	}
+	for (i = MLX5_MAX_UC_MAC_ADDRESSES; i != MLX5_MAX_MAC_ADDRESSES; ++i)
+		mlx5_internal_mac_addr_remove(dev, i);
+	i = MLX5_MAX_UC_MAC_ADDRESSES;
+	while (nb_mc_addr--) {
+		ret = mlx5_internal_mac_addr_add(dev, mc_addr_set++, i++);
+		if (ret)
+			return ret;
+	}
+	if (!dev->data->promiscuous)
+		return mlx5_traffic_restart(dev);
+	return 0;
+}
diff --git a/src/spdk/dpdk/drivers/net/mlx5/mlx5_mr.c b/src/spdk/dpdk/drivers/net/mlx5/mlx5_mr.c
new file mode 100644
index 00000000..1d1bcb5f
--- /dev/null
+++ b/src/spdk/dpdk/drivers/net/mlx5/mlx5_mr.c
@@ -0,0 +1,1186 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2016 6WIND S.A.
+ * Copyright 2016 Mellanox Technologies, Ltd
+ */
+
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <infiniband/verbs.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+#include <rte_mempool.h>
+#include <rte_malloc.h>
+#include <rte_rwlock.h>
+
+#include "mlx5.h"
+#include "mlx5_mr.h"
+#include "mlx5_rxtx.h"
+#include "mlx5_glue.h"
+
+struct mr_find_contig_memsegs_data {
+	uintptr_t addr;
+	uintptr_t start;
+	uintptr_t end;
+	const struct rte_memseg_list *msl;
+};
+
+struct mr_update_mp_data {
+	struct rte_eth_dev *dev;
+	struct mlx5_mr_ctrl *mr_ctrl;
+	int ret;
+};
+
+/**
+ * Expand B-tree table to a given size. Can't be called with holding
+ * memory_hotplug_lock or priv->mr.rwlock due to rte_realloc().
+ *
+ * @param bt
+ *   Pointer to B-tree structure.
+ * @param n
+ *   Number of entries for expansion.
+ *
+ * @return
+ *   0 on success, -1 on failure.
+ */
+static int
+mr_btree_expand(struct mlx5_mr_btree *bt, int n)
+{
+	void *mem;
+	int ret = 0;
+
+	if (n <= bt->size)
+		return ret;
+	/*
+	 * Downside of directly using rte_realloc() is that SOCKET_ID_ANY is
+	 * used inside if there's no room to expand. Because this is a quite
+	 * rare case and a part of very slow path, it is very acceptable.
+	 * Initially cache_bh[] will be given practically enough space and once
+	 * it is expanded, expansion wouldn't be needed again ever.
+	 */
+	mem = rte_realloc(bt->table, n * sizeof(struct mlx5_mr_cache), 0);
+	if (mem == NULL) {
+		/* Not an error, B-tree search will be skipped. */
+		DRV_LOG(WARNING, "failed to expand MR B-tree (%p) table",
+			(void *)bt);
+		ret = -1;
+	} else {
+		DRV_LOG(DEBUG, "expanded MR B-tree table (size=%u)", n);
+		bt->table = mem;
+		bt->size = n;
+	}
+	return ret;
+}
+
+/**
+ * Look up LKey from given B-tree lookup table, store the last index and return
+ * searched LKey.
+ *
+ * @param bt
+ *   Pointer to B-tree structure.
+ * @param[out] idx
+ *   Pointer to index. Even on search failure, returns index where it stops
+ *   searching so that index can be used when inserting a new entry.
+ * @param addr
+ *   Search key.
+ *
+ * @return
+ *   Searched LKey on success, UINT32_MAX on no match.
+ */
+static uint32_t
+mr_btree_lookup(struct mlx5_mr_btree *bt, uint16_t *idx, uintptr_t addr)
+{
+	struct mlx5_mr_cache *lkp_tbl;
+	uint16_t n;
+	uint16_t base = 0;
+
+	assert(bt != NULL);
+	lkp_tbl = *bt->table;
+	n = bt->len;
+	/* First entry must be NULL for comparison. */
+	assert(bt->len > 0 || (lkp_tbl[0].start == 0 &&
+			       lkp_tbl[0].lkey == UINT32_MAX));
+	/* Binary search. */
+	do {
+		register uint16_t delta = n >> 1;
+
+		if (addr < lkp_tbl[base + delta].start) {
+			n = delta;
+		} else {
+			base += delta;
+			n -= delta;
+		}
+	} while (n > 1);
+	assert(addr >= lkp_tbl[base].start);
+	*idx = base;
+	if (addr < lkp_tbl[base].end)
+		return lkp_tbl[base].lkey;
+	/* Not found. */
+	return UINT32_MAX;
+}
+
+/**
+ * Insert an entry to B-tree lookup table.
+ *
+ * @param bt
+ *   Pointer to B-tree structure.
+ * @param entry
+ *   Pointer to new entry to insert.
+ *
+ * @return
+ *   0 on success, -1 on failure.
+ */
+static int
+mr_btree_insert(struct mlx5_mr_btree *bt, struct mlx5_mr_cache *entry)
+{
+	struct mlx5_mr_cache *lkp_tbl;
+	uint16_t idx = 0;
+	size_t shift;
+
+	assert(bt != NULL);
+	assert(bt->len <= bt->size);
+	assert(bt->len > 0);
+	lkp_tbl = *bt->table;
+	/* Find out the slot for insertion. */
+	if (mr_btree_lookup(bt, &idx, entry->start) != UINT32_MAX) {
+		DRV_LOG(DEBUG,
+			"abort insertion to B-tree(%p): already exist at"
+			" idx=%u [0x%" PRIxPTR ", 0x%" PRIxPTR ") lkey=0x%x",
+			(void *)bt, idx, entry->start, entry->end, entry->lkey);
+		/* Already exist, return. */
+		return 0;
+	}
+	/* If table is full, return error. */
+	if (unlikely(bt->len == bt->size)) {
+		bt->overflow = 1;
+		return -1;
+	}
+	/* Insert entry. */
+	++idx;
+	shift = (bt->len - idx) * sizeof(struct mlx5_mr_cache);
+	if (shift)
+		memmove(&lkp_tbl[idx + 1], &lkp_tbl[idx], shift);
+	lkp_tbl[idx] = *entry;
+	bt->len++;
+	DRV_LOG(DEBUG,
+		"inserted B-tree(%p)[%u],"
+		" [0x%" PRIxPTR ", 0x%" PRIxPTR ") lkey=0x%x",
+		(void *)bt, idx, entry->start, entry->end, entry->lkey);
+	return 0;
+}
+
+/**
+ * Initialize B-tree and allocate memory for lookup table.
+ *
+ * @param bt
+ *   Pointer to B-tree structure.
+ * @param n
+ *   Number of entries to allocate.
+ * @param socket
+ *   NUMA socket on which memory must be allocated.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_mr_btree_init(struct mlx5_mr_btree *bt, int n, int socket)
+{
+	if (bt == NULL) {
+		rte_errno = EINVAL;
+		return -rte_errno;
+	}
+	assert(!bt->table && !bt->size);
+	memset(bt, 0, sizeof(*bt));
+	bt->table = rte_calloc_socket("B-tree table",
+				      n, sizeof(struct mlx5_mr_cache),
+				      0, socket);
+	if (bt->table == NULL) {
+		rte_errno = ENOMEM;
+		DEBUG("failed to allocate memory for btree cache on socket %d",
+		      socket);
+		return -rte_errno;
+	}
+	bt->size = n;
+	/* First entry must be NULL for binary search. */
+	(*bt->table)[bt->len++] = (struct mlx5_mr_cache) {
+		.lkey = UINT32_MAX,
+	};
+	DEBUG("initialized B-tree %p with table %p",
+	      (void *)bt, (void *)bt->table);
+	return 0;
+}
+
+/**
+ * Free B-tree resources.
+ *
+ * @param bt
+ *   Pointer to B-tree structure.
+ */
+void
+mlx5_mr_btree_free(struct mlx5_mr_btree *bt)
+{
+	if (bt == NULL)
+		return;
+	DEBUG("freeing B-tree %p with table %p",
+	      (void *)bt, (void *)bt->table);
+	rte_free(bt->table);
+	memset(bt, 0, sizeof(*bt));
+}
+
+/**
+ * Dump all the entries in a B-tree
+ *
+ * @param bt
+ *   Pointer to B-tree structure.
+ */
+void
+mlx5_mr_btree_dump(struct mlx5_mr_btree *bt __rte_unused)
+{
+#ifndef NDEBUG
+	int idx;
+	struct mlx5_mr_cache *lkp_tbl;
+
+	if (bt == NULL)
+		return;
+	lkp_tbl = *bt->table;
+	for (idx = 0; idx < bt->len; ++idx) {
+		struct mlx5_mr_cache *entry = &lkp_tbl[idx];
+
+		DEBUG("B-tree(%p)[%u],"
+		      " [0x%" PRIxPTR ", 0x%" PRIxPTR ") lkey=0x%x",
+		      (void *)bt, idx, entry->start, entry->end, entry->lkey);
+	}
+#endif
+}
+
+/**
+ * Find virtually contiguous memory chunk in a given MR.
+ *
+ * @param dev
+ *   Pointer to MR structure.
+ * @param[out] entry
+ *   Pointer to returning MR cache entry. If not found, this will not be
+ *   updated.
+ * @param start_idx
+ *   Start index of the memseg bitmap.
+ *
+ * @return
+ *   Next index to go on lookup.
+ */
+static int
+mr_find_next_chunk(struct mlx5_mr *mr, struct mlx5_mr_cache *entry,
+		   int base_idx)
+{
+	uintptr_t start = 0;
+	uintptr_t end = 0;
+	uint32_t idx = 0;
+
+	for (idx = base_idx; idx < mr->ms_bmp_n; ++idx) {
+		if (rte_bitmap_get(mr->ms_bmp, idx)) {
+			const struct rte_memseg_list *msl;
+			const struct rte_memseg *ms;
+
+			msl = mr->msl;
+			ms = rte_fbarray_get(&msl->memseg_arr,
+					     mr->ms_base_idx + idx);
+			assert(msl->page_sz == ms->hugepage_sz);
+			if (!start)
+				start = ms->addr_64;
+			end = ms->addr_64 + ms->hugepage_sz;
+		} else if (start) {
+			/* Passed the end of a fragment. */
+			break;
+		}
+	}
+	if (start) {
+		/* Found one chunk. */
+		entry->start = start;
+		entry->end = end;
+		entry->lkey = rte_cpu_to_be_32(mr->ibv_mr->lkey);
+	}
+	return idx;
+}
+
+/**
+ * Insert a MR to the global B-tree cache. It may fail due to low-on-memory.
+ * Then, this entry will have to be searched by mr_lookup_dev_list() in
+ * mlx5_mr_create() on miss.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param mr
+ *   Pointer to MR to insert.
+ *
+ * @return
+ *   0 on success, -1 on failure.
+ */
+static int
+mr_insert_dev_cache(struct rte_eth_dev *dev, struct mlx5_mr *mr)
+{
+	struct priv *priv = dev->data->dev_private;
+	unsigned int n;
+
+	DRV_LOG(DEBUG, "port %u inserting MR(%p) to global cache",
+		dev->data->port_id, (void *)mr);
+	for (n = 0; n < mr->ms_bmp_n; ) {
+		struct mlx5_mr_cache entry = { 0, };
+
+		/* Find a contiguous chunk and advance the index. */
+		n = mr_find_next_chunk(mr, &entry, n);
+		if (!entry.end)
+			break;
+		if (mr_btree_insert(&priv->mr.cache, &entry) < 0) {
+			/*
+			 * Overflowed, but the global table cannot be expanded
+			 * because of deadlock.
+			 */
+			return -1;
+		}
+	}
+	return 0;
+}
+
+/**
+ * Look up address in the original global MR list.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param[out] entry
+ *   Pointer to returning MR cache entry. If no match, this will not be updated.
+ * @param addr
+ *   Search key.
+ *
+ * @return
+ *   Found MR on match, NULL otherwise.
+ */
+static struct mlx5_mr *
+mr_lookup_dev_list(struct rte_eth_dev *dev, struct mlx5_mr_cache *entry,
+		   uintptr_t addr)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_mr *mr;
+
+	/* Iterate all the existing MRs. */
+	LIST_FOREACH(mr, &priv->mr.mr_list, mr) {
+		unsigned int n;
+
+		if (mr->ms_n == 0)
+			continue;
+		for (n = 0; n < mr->ms_bmp_n; ) {
+			struct mlx5_mr_cache ret = { 0, };
+
+			n = mr_find_next_chunk(mr, &ret, n);
+			if (addr >= ret.start && addr < ret.end) {
+				/* Found. */
+				*entry = ret;
+				return mr;
+			}
+		}
+	}
+	return NULL;
+}
+
+/**
+ * Look up address on device.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param[out] entry
+ *   Pointer to returning MR cache entry. If no match, this will not be updated.
+ * @param addr
+ *   Search key.
+ *
+ * @return
+ *   Searched LKey on success, UINT32_MAX on failure and rte_errno is set.
+ */
+static uint32_t
+mr_lookup_dev(struct rte_eth_dev *dev, struct mlx5_mr_cache *entry,
+	      uintptr_t addr)
+{
+	struct priv *priv = dev->data->dev_private;
+	uint16_t idx;
+	uint32_t lkey = UINT32_MAX;
+	struct mlx5_mr *mr;
+
+	/*
+	 * If the global cache has overflowed since it failed to expand the
+	 * B-tree table, it can't have all the existing MRs. Then, the address
+	 * has to be searched by traversing the original MR list instead, which
+	 * is very slow path. Otherwise, the global cache is all inclusive.
+	 */
+	if (!unlikely(priv->mr.cache.overflow)) {
+		lkey = mr_btree_lookup(&priv->mr.cache, &idx, addr);
+		if (lkey != UINT32_MAX)
+			*entry = (*priv->mr.cache.table)[idx];
+	} else {
+		/* Falling back to the slowest path. */
+		mr = mr_lookup_dev_list(dev, entry, addr);
+		if (mr != NULL)
+			lkey = entry->lkey;
+	}
+	assert(lkey == UINT32_MAX || (addr >= entry->start &&
+				      addr < entry->end));
+	return lkey;
+}
+
+/**
+ * Free MR resources. MR lock must not be held to avoid a deadlock. rte_free()
+ * can raise memory free event and the callback function will spin on the lock.
+ *
+ * @param mr
+ *   Pointer to MR to free.
+ */
+static void
+mr_free(struct mlx5_mr *mr)
+{
+	if (mr == NULL)
+		return;
+	DRV_LOG(DEBUG, "freeing MR(%p):", (void *)mr);
+	if (mr->ibv_mr != NULL)
+		claim_zero(mlx5_glue->dereg_mr(mr->ibv_mr));
+	if (mr->ms_bmp != NULL)
+		rte_bitmap_free(mr->ms_bmp);
+	rte_free(mr);
+}
+
+/**
+ * Releass resources of detached MR having no online entry.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ */
+static void
+mlx5_mr_garbage_collect(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_mr *mr_next;
+	struct mlx5_mr_list free_list = LIST_HEAD_INITIALIZER(free_list);
+
+	/* Must be called from the primary process. */
+	assert(rte_eal_process_type() == RTE_PROC_PRIMARY);
+	/*
+	 * MR can't be freed with holding the lock because rte_free() could call
+	 * memory free callback function. This will be a deadlock situation.
+	 */
+	rte_rwlock_write_lock(&priv->mr.rwlock);
+	/* Detach the whole free list and release it after unlocking. */
+	free_list = priv->mr.mr_free_list;
+	LIST_INIT(&priv->mr.mr_free_list);
+	rte_rwlock_write_unlock(&priv->mr.rwlock);
+	/* Release resources. */
+	mr_next = LIST_FIRST(&free_list);
+	while (mr_next != NULL) {
+		struct mlx5_mr *mr = mr_next;
+
+		mr_next = LIST_NEXT(mr, mr);
+		mr_free(mr);
+	}
+}
+
+/* Called during rte_memseg_contig_walk() by mlx5_mr_create(). */
+static int
+mr_find_contig_memsegs_cb(const struct rte_memseg_list *msl,
+			  const struct rte_memseg *ms, size_t len, void *arg)
+{
+	struct mr_find_contig_memsegs_data *data = arg;
+
+	if (data->addr < ms->addr_64 || data->addr >= ms->addr_64 + len)
+		return 0;
+	/* Found, save it and stop walking. */
+	data->start = ms->addr_64;
+	data->end = ms->addr_64 + len;
+	data->msl = msl;
+	return 1;
+}
+
+/**
+ * Create a new global Memroy Region (MR) for a missing virtual address.
+ * Register entire virtually contiguous memory chunk around the address.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param[out] entry
+ *   Pointer to returning MR cache entry, found in the global cache or newly
+ *   created. If failed to create one, this will not be updated.
+ * @param addr
+ *   Target virtual address to register.
+ *
+ * @return
+ *   Searched LKey on success, UINT32_MAX on failure and rte_errno is set.
+ */
+static uint32_t
+mlx5_mr_create(struct rte_eth_dev *dev, struct mlx5_mr_cache *entry,
+	       uintptr_t addr)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+	const struct rte_memseg_list *msl;
+	const struct rte_memseg *ms;
+	struct mlx5_mr *mr = NULL;
+	size_t len;
+	uint32_t ms_n;
+	uint32_t bmp_size;
+	void *bmp_mem;
+	int ms_idx_shift = -1;
+	unsigned int n;
+	struct mr_find_contig_memsegs_data data = {
+		.addr = addr,
+	};
+	struct mr_find_contig_memsegs_data data_re;
+
+	DRV_LOG(DEBUG, "port %u creating a MR using address (%p)",
+		dev->data->port_id, (void *)addr);
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+		DRV_LOG(WARNING,
+			"port %u using address (%p) of unregistered mempool"
+			" in secondary process, please create mempool"
+			" before rte_eth_dev_start()",
+			dev->data->port_id, (void *)addr);
+		rte_errno = EPERM;
+		goto err_nolock;
+	}
+	/*
+	 * Release detached MRs if any. This can't be called with holding either
+	 * memory_hotplug_lock or priv->mr.rwlock. MRs on the free list have
+	 * been detached by the memory free event but it couldn't be released
+	 * inside the callback due to deadlock. As a result, releasing resources
+	 * is quite opportunistic.
+	 */
+	mlx5_mr_garbage_collect(dev);
+	/*
+	 * Find out a contiguous virtual address chunk in use, to which the
+	 * given address belongs, in order to register maximum range. In the
+	 * best case where mempools are not dynamically recreated and
+	 * '--socket-mem' is speicified as an EAL option, it is very likely to
+	 * have only one MR(LKey) per a socket and per a hugepage-size even
+	 * though the system memory is highly fragmented.
+	 */
+	if (!rte_memseg_contig_walk(mr_find_contig_memsegs_cb, &data)) {
+		DRV_LOG(WARNING,
+			"port %u unable to find virtually contiguous"
+			" chunk for address (%p)."
+			" rte_memseg_contig_walk() failed.",
+			dev->data->port_id, (void *)addr);
+		rte_errno = ENXIO;
+		goto err_nolock;
+	}
+alloc_resources:
+	/* Addresses must be page-aligned. */
+	assert(rte_is_aligned((void *)data.start, data.msl->page_sz));
+	assert(rte_is_aligned((void *)data.end, data.msl->page_sz));
+	msl = data.msl;
+	ms = rte_mem_virt2memseg((void *)data.start, msl);
+	len = data.end - data.start;
+	assert(msl->page_sz == ms->hugepage_sz);
+	/* Number of memsegs in the range. */
+	ms_n = len / msl->page_sz;
+	DEBUG("port %u extending %p to [0x%" PRIxPTR ", 0x%" PRIxPTR "),"
+	      " page_sz=0x%" PRIx64 ", ms_n=%u",
+	      dev->data->port_id, (void *)addr,
+	      data.start, data.end, msl->page_sz, ms_n);
+	/* Size of memory for bitmap. */
+	bmp_size = rte_bitmap_get_memory_footprint(ms_n);
+	mr = rte_zmalloc_socket(NULL,
+				RTE_ALIGN_CEIL(sizeof(*mr),
+					       RTE_CACHE_LINE_SIZE) +
+				bmp_size,
+				RTE_CACHE_LINE_SIZE, msl->socket_id);
+	if (mr == NULL) {
+		DEBUG("port %u unable to allocate memory for a new MR of"
+		      " address (%p).",
+		      dev->data->port_id, (void *)addr);
+		rte_errno = ENOMEM;
+		goto err_nolock;
+	}
+	mr->msl = msl;
+	/*
+	 * Save the index of the first memseg and initialize memseg bitmap. To
+	 * see if a memseg of ms_idx in the memseg-list is still valid, check:
+	 *	rte_bitmap_get(mr->bmp, ms_idx - mr->ms_base_idx)
+	 */
+	mr->ms_base_idx = rte_fbarray_find_idx(&msl->memseg_arr, ms);
+	bmp_mem = RTE_PTR_ALIGN_CEIL(mr + 1, RTE_CACHE_LINE_SIZE);
+	mr->ms_bmp = rte_bitmap_init(ms_n, bmp_mem, bmp_size);
+	if (mr->ms_bmp == NULL) {
+		DEBUG("port %u unable to initialize bitamp for a new MR of"
+		      " address (%p).",
+		      dev->data->port_id, (void *)addr);
+		rte_errno = EINVAL;
+		goto err_nolock;
+	}
+	/*
+	 * Should recheck whether the extended contiguous chunk is still valid.
+	 * Because memory_hotplug_lock can't be held if there's any memory
+	 * related calls in a critical path, resource allocation above can't be
+	 * locked. If the memory has been changed at this point, try again with
+	 * just single page. If not, go on with the big chunk atomically from
+	 * here.
+	 */
+	rte_rwlock_read_lock(&mcfg->memory_hotplug_lock);
+	data_re = data;
+	if (len > msl->page_sz &&
+	    !rte_memseg_contig_walk(mr_find_contig_memsegs_cb, &data_re)) {
+		DEBUG("port %u unable to find virtually contiguous"
+		      " chunk for address (%p)."
+		      " rte_memseg_contig_walk() failed.",
+		      dev->data->port_id, (void *)addr);
+		rte_errno = ENXIO;
+		goto err_memlock;
+	}
+	if (data.start != data_re.start || data.end != data_re.end) {
+		/*
+		 * The extended contiguous chunk has been changed. Try again
+		 * with single memseg instead.
+		 */
+		data.start = RTE_ALIGN_FLOOR(addr, msl->page_sz);
+		data.end = data.start + msl->page_sz;
+		rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
+		mr_free(mr);
+		goto alloc_resources;
+	}
+	assert(data.msl == data_re.msl);
+	rte_rwlock_write_lock(&priv->mr.rwlock);
+	/*
+	 * Check the address is really missing. If other thread already created
+	 * one or it is not found due to overflow, abort and return.
+	 */
+	if (mr_lookup_dev(dev, entry, addr) != UINT32_MAX) {
+		/*
+		 * Insert to the global cache table. It may fail due to
+		 * low-on-memory. Then, this entry will have to be searched
+		 * here again.
+		 */
+		mr_btree_insert(&priv->mr.cache, entry);
+		DEBUG("port %u found MR for %p on final lookup, abort",
+		      dev->data->port_id, (void *)addr);
+		rte_rwlock_write_unlock(&priv->mr.rwlock);
+		rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
+		/*
+		 * Must be unlocked before calling rte_free() because
+		 * mlx5_mr_mem_event_free_cb() can be called inside.
+		 */
+		mr_free(mr);
+		return entry->lkey;
+	}
+	/*
+	 * Trim start and end addresses for verbs MR. Set bits for registering
+	 * memsegs but exclude already registered ones. Bitmap can be
+	 * fragmented.
+	 */
+	for (n = 0; n < ms_n; ++n) {
+		uintptr_t start;
+		struct mlx5_mr_cache ret = { 0, };
+
+		start = data_re.start + n * msl->page_sz;
+		/* Exclude memsegs already registered by other MRs. */
+		if (mr_lookup_dev(dev, &ret, start) == UINT32_MAX) {
+			/*
+			 * Start from the first unregistered memseg in the
+			 * extended range.
+			 */
+			if (ms_idx_shift == -1) {
+				mr->ms_base_idx += n;
+				data.start = start;
+				ms_idx_shift = n;
+			}
+			data.end = start + msl->page_sz;
+			rte_bitmap_set(mr->ms_bmp, n - ms_idx_shift);
+			++mr->ms_n;
+		}
+	}
+	len = data.end - data.start;
+	mr->ms_bmp_n = len / msl->page_sz;
+	assert(ms_idx_shift + mr->ms_bmp_n <= ms_n);
+	/*
+	 * Finally create a verbs MR for the memory chunk. ibv_reg_mr() can be
+	 * called with holding the memory lock because it doesn't use
+	 * mlx5_alloc_buf_extern() which eventually calls rte_malloc_socket()
+	 * through mlx5_alloc_verbs_buf().
+	 */
+	mr->ibv_mr = mlx5_glue->reg_mr(priv->pd, (void *)data.start, len,
+				       IBV_ACCESS_LOCAL_WRITE);
+	if (mr->ibv_mr == NULL) {
+		DEBUG("port %u fail to create a verbs MR for address (%p)",
+		      dev->data->port_id, (void *)addr);
+		rte_errno = EINVAL;
+		goto err_mrlock;
+	}
+	assert((uintptr_t)mr->ibv_mr->addr == data.start);
+	assert(mr->ibv_mr->length == len);
+	LIST_INSERT_HEAD(&priv->mr.mr_list, mr, mr);
+	DEBUG("port %u MR CREATED (%p) for %p:\n"
+	      "  [0x%" PRIxPTR ", 0x%" PRIxPTR "),"
+	      " lkey=0x%x base_idx=%u ms_n=%u, ms_bmp_n=%u",
+	      dev->data->port_id, (void *)mr, (void *)addr,
+	      data.start, data.end, rte_cpu_to_be_32(mr->ibv_mr->lkey),
+	      mr->ms_base_idx, mr->ms_n, mr->ms_bmp_n);
+	/* Insert to the global cache table. */
+	mr_insert_dev_cache(dev, mr);
+	/* Fill in output data. */
+	mr_lookup_dev(dev, entry, addr);
+	/* Lookup can't fail. */
+	assert(entry->lkey != UINT32_MAX);
+	rte_rwlock_write_unlock(&priv->mr.rwlock);
+	rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
+	return entry->lkey;
+err_mrlock:
+	rte_rwlock_write_unlock(&priv->mr.rwlock);
+err_memlock:
+	rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
+err_nolock:
+	/*
+	 * In case of error, as this can be called in a datapath, a warning
+	 * message per an error is preferable instead. Must be unlocked before
+	 * calling rte_free() because mlx5_mr_mem_event_free_cb() can be called
+	 * inside.
+	 */
+	mr_free(mr);
+	return UINT32_MAX;
+}
+
+/**
+ * Rebuild the global B-tree cache of device from the original MR list.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ */
+static void
+mr_rebuild_dev_cache(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_mr *mr;
+
+	DRV_LOG(DEBUG, "port %u rebuild dev cache[]", dev->data->port_id);
+	/* Flush cache to rebuild. */
+	priv->mr.cache.len = 1;
+	priv->mr.cache.overflow = 0;
+	/* Iterate all the existing MRs. */
+	LIST_FOREACH(mr, &priv->mr.mr_list, mr)
+		if (mr_insert_dev_cache(dev, mr) < 0)
+			return;
+}
+
+/**
+ * Callback for memory free event. Iterate freed memsegs and check whether it
+ * belongs to an existing MR. If found, clear the bit from bitmap of MR. As a
+ * result, the MR would be fragmented. If it becomes empty, the MR will be freed
+ * later by mlx5_mr_garbage_collect(). Even if this callback is called from a
+ * secondary process, the garbage collector will be called in primary process
+ * as the secondary process can't call mlx5_mr_create().
+ *
+ * The global cache must be rebuilt if there's any change and this event has to
+ * be propagated to dataplane threads to flush the local caches.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param addr
+ *   Address of freed memory.
+ * @param len
+ *   Size of freed memory.
+ */
+static void
+mlx5_mr_mem_event_free_cb(struct rte_eth_dev *dev, const void *addr, size_t len)
+{
+	struct priv *priv = dev->data->dev_private;
+	const struct rte_memseg_list *msl;
+	struct mlx5_mr *mr;
+	int ms_n;
+	int i;
+	int rebuild = 0;
+
+	DEBUG("port %u free callback: addr=%p, len=%zu",
+	      dev->data->port_id, addr, len);
+	msl = rte_mem_virt2memseg_list(addr);
+	/* addr and len must be page-aligned. */
+	assert((uintptr_t)addr == RTE_ALIGN((uintptr_t)addr, msl->page_sz));
+	assert(len == RTE_ALIGN(len, msl->page_sz));
+	ms_n = len / msl->page_sz;
+	rte_rwlock_write_lock(&priv->mr.rwlock);
+	/* Clear bits of freed memsegs from MR. */
+	for (i = 0; i < ms_n; ++i) {
+		const struct rte_memseg *ms;
+		struct mlx5_mr_cache entry;
+		uintptr_t start;
+		int ms_idx;
+		uint32_t pos;
+
+		/* Find MR having this memseg. */
+		start = (uintptr_t)addr + i * msl->page_sz;
+		mr = mr_lookup_dev_list(dev, &entry, start);
+		if (mr == NULL)
+			continue;
+		ms = rte_mem_virt2memseg((void *)start, msl);
+		assert(ms != NULL);
+		assert(msl->page_sz == ms->hugepage_sz);
+		ms_idx = rte_fbarray_find_idx(&msl->memseg_arr, ms);
+		pos = ms_idx - mr->ms_base_idx;
+		assert(rte_bitmap_get(mr->ms_bmp, pos));
+		assert(pos < mr->ms_bmp_n);
+		DEBUG("port %u MR(%p): clear bitmap[%u] for addr %p",
+		      dev->data->port_id, (void *)mr, pos, (void *)start);
+		rte_bitmap_clear(mr->ms_bmp, pos);
+		if (--mr->ms_n == 0) {
+			LIST_REMOVE(mr, mr);
+			LIST_INSERT_HEAD(&priv->mr.mr_free_list, mr, mr);
+			DEBUG("port %u remove MR(%p) from list",
+			      dev->data->port_id, (void *)mr);
+		}
+		/*
+		 * MR is fragmented or will be freed. the global cache must be
+		 * rebuilt.
+		 */
+		rebuild = 1;
+	}
+	if (rebuild) {
+		mr_rebuild_dev_cache(dev);
+		/*
+		 * Flush local caches by propagating invalidation across cores.
+		 * rte_smp_wmb() is enough to synchronize this event. If one of
+		 * freed memsegs is seen by other core, that means the memseg
+		 * has been allocated by allocator, which will come after this
+		 * free call. Therefore, this store instruction (incrementing
+		 * generation below) will be guaranteed to be seen by other core
+		 * before the core sees the newly allocated memory.
+		 */
+		++priv->mr.dev_gen;
+		DEBUG("broadcasting local cache flush, gen=%d",
+		      priv->mr.dev_gen);
+		rte_smp_wmb();
+	}
+	rte_rwlock_write_unlock(&priv->mr.rwlock);
+}
+
+/**
+ * Callback for memory event. This can be called from both primary and secondary
+ * process.
+ *
+ * @param event_type
+ *   Memory event type.
+ * @param addr
+ *   Address of memory.
+ * @param len
+ *   Size of memory.
+ */
+void
+mlx5_mr_mem_event_cb(enum rte_mem_event event_type, const void *addr,
+		     size_t len, void *arg __rte_unused)
+{
+	struct priv *priv;
+	struct mlx5_dev_list *dev_list = &mlx5_shared_data->mem_event_cb_list;
+
+	switch (event_type) {
+	case RTE_MEM_EVENT_FREE:
+		rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock);
+		/* Iterate all the existing mlx5 devices. */
+		LIST_FOREACH(priv, dev_list, mem_event_cb)
+			mlx5_mr_mem_event_free_cb(ETH_DEV(priv), addr, len);
+		rte_rwlock_write_unlock(&mlx5_shared_data->mem_event_rwlock);
+		break;
+	case RTE_MEM_EVENT_ALLOC:
+	default:
+		break;
+	}
+}
+
+/**
+ * Look up address in the global MR cache table. If not found, create a new MR.
+ * Insert the found/created entry to local bottom-half cache table.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param mr_ctrl
+ *   Pointer to per-queue MR control structure.
+ * @param[out] entry
+ *   Pointer to returning MR cache entry, found in the global cache or newly
+ *   created. If failed to create one, this is not written.
+ * @param addr
+ *   Search key.
+ *
+ * @return
+ *   Searched LKey on success, UINT32_MAX on no match.
+ */
+static uint32_t
+mlx5_mr_lookup_dev(struct rte_eth_dev *dev, struct mlx5_mr_ctrl *mr_ctrl,
+		   struct mlx5_mr_cache *entry, uintptr_t addr)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_mr_btree *bt = &mr_ctrl->cache_bh;
+	uint16_t idx;
+	uint32_t lkey;
+
+	/* If local cache table is full, try to double it. */
+	if (unlikely(bt->len == bt->size))
+		mr_btree_expand(bt, bt->size << 1);
+	/* Look up in the global cache. */
+	rte_rwlock_read_lock(&priv->mr.rwlock);
+	lkey = mr_btree_lookup(&priv->mr.cache, &idx, addr);
+	if (lkey != UINT32_MAX) {
+		/* Found. */
+		*entry = (*priv->mr.cache.table)[idx];
+		rte_rwlock_read_unlock(&priv->mr.rwlock);
+		/*
+		 * Update local cache. Even if it fails, return the found entry
+		 * to update top-half cache. Next time, this entry will be found
+		 * in the global cache.
+		 */
+		mr_btree_insert(bt, entry);
+		return lkey;
+	}
+	rte_rwlock_read_unlock(&priv->mr.rwlock);
+	/* First time to see the address? Create a new MR. */
+	lkey = mlx5_mr_create(dev, entry, addr);
+	/*
+	 * Update the local cache if successfully created a new global MR. Even
+	 * if failed to create one, there's no action to take in this datapath
+	 * code. As returning LKey is invalid, this will eventually make HW
+	 * fail.
+	 */
+	if (lkey != UINT32_MAX)
+		mr_btree_insert(bt, entry);
+	return lkey;
+}
+
+/**
+ * Bottom-half of LKey search on datapath. Firstly search in cache_bh[] and if
+ * misses, search in the global MR cache table and update the new entry to
+ * per-queue local caches.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param mr_ctrl
+ *   Pointer to per-queue MR control structure.
+ * @param addr
+ *   Search key.
+ *
+ * @return
+ *   Searched LKey on success, UINT32_MAX on no match.
+ */
+static uint32_t
+mlx5_mr_addr2mr_bh(struct rte_eth_dev *dev, struct mlx5_mr_ctrl *mr_ctrl,
+		   uintptr_t addr)
+{
+	uint32_t lkey;
+	uint16_t bh_idx = 0;
+	/* Victim in top-half cache to replace with new entry. */
+	struct mlx5_mr_cache *repl = &mr_ctrl->cache[mr_ctrl->head];
+
+	/* Binary-search MR translation table. */
+	lkey = mr_btree_lookup(&mr_ctrl->cache_bh, &bh_idx, addr);
+	/* Update top-half cache. */
+	if (likely(lkey != UINT32_MAX)) {
+		*repl = (*mr_ctrl->cache_bh.table)[bh_idx];
+	} else {
+		/*
+		 * If missed in local lookup table, search in the global cache
+		 * and local cache_bh[] will be updated inside if possible.
+		 * Top-half cache entry will also be updated.
+		 */
+		lkey = mlx5_mr_lookup_dev(dev, mr_ctrl, repl, addr);
+		if (unlikely(lkey == UINT32_MAX))
+			return UINT32_MAX;
+	}
+	/* Update the most recently used entry. */
+	mr_ctrl->mru = mr_ctrl->head;
+	/* Point to the next victim, the oldest. */
+	mr_ctrl->head = (mr_ctrl->head + 1) % MLX5_MR_CACHE_N;
+	return lkey;
+}
+
+/**
+ * Bottom-half of LKey search on Rx.
+ *
+ * @param rxq
+ *   Pointer to Rx queue structure.
+ * @param addr
+ *   Search key.
+ *
+ * @return
+ *   Searched LKey on success, UINT32_MAX on no match.
+ */
+uint32_t
+mlx5_rx_addr2mr_bh(struct mlx5_rxq_data *rxq, uintptr_t addr)
+{
+	struct mlx5_rxq_ctrl *rxq_ctrl =
+		container_of(rxq, struct mlx5_rxq_ctrl, rxq);
+	struct mlx5_mr_ctrl *mr_ctrl = &rxq->mr_ctrl;
+	struct priv *priv = rxq_ctrl->priv;
+
+	DRV_LOG(DEBUG,
+		"Rx queue %u: miss on top-half, mru=%u, head=%u, addr=%p",
+		rxq_ctrl->idx, mr_ctrl->mru, mr_ctrl->head, (void *)addr);
+	return mlx5_mr_addr2mr_bh(ETH_DEV(priv), mr_ctrl, addr);
+}
+
+/**
+ * Bottom-half of LKey search on Tx.
+ *
+ * @param txq
+ *   Pointer to Tx queue structure.
+ * @param addr
+ *   Search key.
+ *
+ * @return
+ *   Searched LKey on success, UINT32_MAX on no match.
+ */
+uint32_t
+mlx5_tx_addr2mr_bh(struct mlx5_txq_data *txq, uintptr_t addr)
+{
+	struct mlx5_txq_ctrl *txq_ctrl =
+		container_of(txq, struct mlx5_txq_ctrl, txq);
+	struct mlx5_mr_ctrl *mr_ctrl = &txq->mr_ctrl;
+	struct priv *priv = txq_ctrl->priv;
+
+	DRV_LOG(DEBUG,
+		"Tx queue %u: miss on top-half, mru=%u, head=%u, addr=%p",
+		txq_ctrl->idx, mr_ctrl->mru, mr_ctrl->head, (void *)addr);
+	return mlx5_mr_addr2mr_bh(ETH_DEV(priv), mr_ctrl, addr);
+}
+
+/**
+ * Flush all of the local cache entries.
+ *
+ * @param mr_ctrl
+ *   Pointer to per-queue MR control structure.
+ */
+void
+mlx5_mr_flush_local_cache(struct mlx5_mr_ctrl *mr_ctrl)
+{
+	/* Reset the most-recently-used index. */
+	mr_ctrl->mru = 0;
+	/* Reset the linear search array. */
+	mr_ctrl->head = 0;
+	memset(mr_ctrl->cache, 0, sizeof(mr_ctrl->cache));
+	/* Reset the B-tree table. */
+	mr_ctrl->cache_bh.len = 1;
+	mr_ctrl->cache_bh.overflow = 0;
+	/* Update the generation number. */
+	mr_ctrl->cur_gen = *mr_ctrl->dev_gen_ptr;
+	DRV_LOG(DEBUG, "mr_ctrl(%p): flushed, cur_gen=%d",
+		(void *)mr_ctrl, mr_ctrl->cur_gen);
+}
+
+/* Called during rte_mempool_mem_iter() by mlx5_mr_update_mp(). */
+static void
+mlx5_mr_update_mp_cb(struct rte_mempool *mp __rte_unused, void *opaque,
+		     struct rte_mempool_memhdr *memhdr,
+		     unsigned mem_idx __rte_unused)
+{
+	struct mr_update_mp_data *data = opaque;
+	uint32_t lkey;
+
+	/* Stop iteration if failed in the previous walk. */
+	if (data->ret < 0)
+		return;
+	/* Register address of the chunk and update local caches. */
+	lkey = mlx5_mr_addr2mr_bh(data->dev, data->mr_ctrl,
+				  (uintptr_t)memhdr->addr);
+	if (lkey == UINT32_MAX)
+		data->ret = -1;
+}
+
+/**
+ * Register entire memory chunks in a Mempool.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param mr_ctrl
+ *   Pointer to per-queue MR control structure.
+ * @param mp
+ *   Pointer to registering Mempool.
+ *
+ * @return
+ *   0 on success, -1 on failure.
+ */
+int
+mlx5_mr_update_mp(struct rte_eth_dev *dev, struct mlx5_mr_ctrl *mr_ctrl,
+		  struct rte_mempool *mp)
+{
+	struct mr_update_mp_data data = {
+		.dev = dev,
+		.mr_ctrl = mr_ctrl,
+		.ret = 0,
+	};
+
+	rte_mempool_mem_iter(mp, mlx5_mr_update_mp_cb, &data);
+	return data.ret;
+}
+
+/**
+ * Dump all the created MRs and the global cache entries.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ */
+void
+mlx5_mr_dump_dev(struct rte_eth_dev *dev __rte_unused)
+{
+#ifndef NDEBUG
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_mr *mr;
+	int mr_n = 0;
+	int chunk_n = 0;
+
+	rte_rwlock_read_lock(&priv->mr.rwlock);
+	/* Iterate all the existing MRs. */
+	LIST_FOREACH(mr, &priv->mr.mr_list, mr) {
+		unsigned int n;
+
+		DEBUG("port %u MR[%u], LKey = 0x%x, ms_n = %u, ms_bmp_n = %u",
+		      dev->data->port_id, mr_n++,
+		      rte_cpu_to_be_32(mr->ibv_mr->lkey),
+		      mr->ms_n, mr->ms_bmp_n);
+		if (mr->ms_n == 0)
+			continue;
+		for (n = 0; n < mr->ms_bmp_n; ) {
+			struct mlx5_mr_cache ret = { 0, };
+
+			n = mr_find_next_chunk(mr, &ret, n);
+			if (!ret.end)
+				break;
+			DEBUG("  chunk[%u], [0x%" PRIxPTR ", 0x%" PRIxPTR ")",
+			      chunk_n++, ret.start, ret.end);
+		}
+	}
+	DEBUG("port %u dumping global cache", dev->data->port_id);
+	mlx5_mr_btree_dump(&priv->mr.cache);
+	rte_rwlock_read_unlock(&priv->mr.rwlock);
+#endif
+}
+
+/**
+ * Release all the created MRs and resources. Remove device from memory callback
+ * list.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ */
+void
+mlx5_mr_release(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_mr *mr_next = LIST_FIRST(&priv->mr.mr_list);
+
+	/* Remove from memory callback device list. */
+	rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock);
+	LIST_REMOVE(priv, mem_event_cb);
+	rte_rwlock_write_unlock(&mlx5_shared_data->mem_event_rwlock);
+	if (rte_log_get_level(mlx5_logtype) == RTE_LOG_DEBUG)
+		mlx5_mr_dump_dev(dev);
+	rte_rwlock_write_lock(&priv->mr.rwlock);
+	/* Detach from MR list and move to free list. */
+	while (mr_next != NULL) {
+		struct mlx5_mr *mr = mr_next;
+
+		mr_next = LIST_NEXT(mr, mr);
+		LIST_REMOVE(mr, mr);
+		LIST_INSERT_HEAD(&priv->mr.mr_free_list, mr, mr);
+	}
+	LIST_INIT(&priv->mr.mr_list);
+	/* Free global cache. */
+	mlx5_mr_btree_free(&priv->mr.cache);
+	rte_rwlock_write_unlock(&priv->mr.rwlock);
+	/* Free all remaining MRs. */
+	mlx5_mr_garbage_collect(dev);
+}
diff --git a/src/spdk/dpdk/drivers/net/mlx5/mlx5_mr.h b/src/spdk/dpdk/drivers/net/mlx5/mlx5_mr.h
new file mode 100644
index 00000000..a57003fe
--- /dev/null
+++ b/src/spdk/dpdk/drivers/net/mlx5/mlx5_mr.h
@@ -0,0 +1,120 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 6WIND S.A.
+ * Copyright 2018 Mellanox Technologies, Ltd
+ */
+
+#ifndef RTE_PMD_MLX5_MR_H_
+#define RTE_PMD_MLX5_MR_H_
+
+#include <stddef.h>
+#include <stdint.h>
+#include <sys/queue.h>
+
+/* Verbs header. */
+/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <infiniband/verbs.h>
+#include <infiniband/mlx5dv.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+#include <rte_eal_memconfig.h>
+#include <rte_ethdev.h>
+#include <rte_rwlock.h>
+#include <rte_bitmap.h>
+
+/* Memory Region object. */
+struct mlx5_mr {
+	LIST_ENTRY(mlx5_mr) mr; /**< Pointer to the prev/next entry. */
+	struct ibv_mr *ibv_mr; /* Verbs Memory Region. */
+	const struct rte_memseg_list *msl;
+	int ms_base_idx; /* Start index of msl->memseg_arr[]. */
+	int ms_n; /* Number of memsegs in use. */
+	uint32_t ms_bmp_n; /* Number of bits in memsegs bit-mask. */
+	struct rte_bitmap *ms_bmp; /* Bit-mask of memsegs belonged to MR. */
+};
+
+/* Cache entry for Memory Region. */
+struct mlx5_mr_cache {
+	uintptr_t start; /* Start address of MR. */
+	uintptr_t end; /* End address of MR. */
+	uint32_t lkey; /* rte_cpu_to_be_32(ibv_mr->lkey). */
+} __rte_packed;
+
+/* MR Cache table for Binary search. */
+struct mlx5_mr_btree {
+	uint16_t len; /* Number of entries. */
+	uint16_t size; /* Total number of entries. */
+	int overflow; /* Mark failure of table expansion. */
+	struct mlx5_mr_cache (*table)[];
+} __rte_packed;
+
+/* Per-queue MR control descriptor. */
+struct mlx5_mr_ctrl {
+	uint32_t *dev_gen_ptr; /* Generation number of device to poll. */
+	uint32_t cur_gen; /* Generation number saved to flush caches. */
+	uint16_t mru; /* Index of last hit entry in top-half cache. */
+	uint16_t head; /* Index of the oldest entry in top-half cache. */
+	struct mlx5_mr_cache cache[MLX5_MR_CACHE_N]; /* Cache for top-half. */
+	struct mlx5_mr_btree cache_bh; /* Cache for bottom-half. */
+} __rte_packed;
+
+extern struct mlx5_dev_list  mlx5_mem_event_cb_list;
+extern rte_rwlock_t mlx5_mem_event_rwlock;
+
+/* First entry must be NULL for comparison. */
+#define mlx5_mr_btree_len(bt) ((bt)->len - 1)
+
+int mlx5_mr_btree_init(struct mlx5_mr_btree *bt, int n, int socket);
+void mlx5_mr_btree_free(struct mlx5_mr_btree *bt);
+void mlx5_mr_mem_event_cb(enum rte_mem_event event_type, const void *addr,
+			  size_t len, void *arg);
+int mlx5_mr_update_mp(struct rte_eth_dev *dev, struct mlx5_mr_ctrl *mr_ctrl,
+		      struct rte_mempool *mp);
+void mlx5_mr_release(struct rte_eth_dev *dev);
+
+/* Debug purpose functions. */
+void mlx5_mr_btree_dump(struct mlx5_mr_btree *bt);
+void mlx5_mr_dump_dev(struct rte_eth_dev *dev);
+
+/**
+ * Look up LKey from given lookup table by linear search. Firstly look up the
+ * last-hit entry. If miss, the entire array is searched. If found, update the
+ * last-hit index and return LKey.
+ *
+ * @param lkp_tbl
+ *   Pointer to lookup table.
+ * @param[in,out] cached_idx
+ *   Pointer to last-hit index.
+ * @param n
+ *   Size of lookup table.
+ * @param addr
+ *   Search key.
+ *
+ * @return
+ *   Searched LKey on success, UINT32_MAX on no match.
+ */
+static __rte_always_inline uint32_t
+mlx5_mr_lookup_cache(struct mlx5_mr_cache *lkp_tbl, uint16_t *cached_idx,
+		     uint16_t n, uintptr_t addr)
+{
+	uint16_t idx;
+
+	if (likely(addr >= lkp_tbl[*cached_idx].start &&
+		   addr < lkp_tbl[*cached_idx].end))
+		return lkp_tbl[*cached_idx].lkey;
+	for (idx = 0; idx < n && lkp_tbl[idx].start != 0; ++idx) {
+		if (addr >= lkp_tbl[idx].start &&
+		    addr < lkp_tbl[idx].end) {
+			/* Found. */
+			*cached_idx = idx;
+			return lkp_tbl[idx].lkey;
+		}
+	}
+	return UINT32_MAX;
+}
+
+#endif /* RTE_PMD_MLX5_MR_H_ */
diff --git a/src/spdk/dpdk/drivers/net/mlx5/mlx5_nl.c b/src/spdk/dpdk/drivers/net/mlx5/mlx5_nl.c
new file mode 100644
index 00000000..d61826ae
--- /dev/null
+++ b/src/spdk/dpdk/drivers/net/mlx5/mlx5_nl.c
@@ -0,0 +1,916 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 6WIND S.A.
+ * Copyright 2018 Mellanox Technologies, Ltd
+ */
+
+#include <errno.h>
+#include <linux/if_link.h>
+#include <linux/netlink.h>
+#include <linux/rtnetlink.h>
+#include <net/if.h>
+#include <rdma/rdma_netlink.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/socket.h>
+#include <unistd.h>
+
+#include <rte_errno.h>
+
+#include "mlx5.h"
+#include "mlx5_utils.h"
+
+/* Size of the buffer to receive kernel messages */
+#define MLX5_NL_BUF_SIZE (32 * 1024)
+/* Send buffer size for the Netlink socket */
+#define MLX5_SEND_BUF_SIZE 32768
+/* Receive buffer size for the Netlink socket */
+#define MLX5_RECV_BUF_SIZE 32768
+
+/*
+ * Define NDA_RTA as defined in iproute2 sources.
+ *
+ * see in iproute2 sources file include/libnetlink.h
+ */
+#ifndef MLX5_NDA_RTA
+#define MLX5_NDA_RTA(r) \
+	((struct rtattr *)(((char *)(r)) + NLMSG_ALIGN(sizeof(struct ndmsg))))
+#endif
+
+/*
+ * The following definitions are normally found in rdma/rdma_netlink.h,
+ * however they are so recent that most systems do not expose them yet.
+ */
+#ifndef HAVE_RDMA_NL_NLDEV
+#define RDMA_NL_NLDEV 5
+#endif
+#ifndef HAVE_RDMA_NLDEV_CMD_GET
+#define RDMA_NLDEV_CMD_GET 1
+#endif
+#ifndef HAVE_RDMA_NLDEV_CMD_PORT_GET
+#define RDMA_NLDEV_CMD_PORT_GET 5
+#endif
+#ifndef HAVE_RDMA_NLDEV_ATTR_DEV_INDEX
+#define RDMA_NLDEV_ATTR_DEV_INDEX 1
+#endif
+#ifndef HAVE_RDMA_NLDEV_ATTR_DEV_NAME
+#define RDMA_NLDEV_ATTR_DEV_NAME 2
+#endif
+#ifndef HAVE_RDMA_NLDEV_ATTR_PORT_INDEX
+#define RDMA_NLDEV_ATTR_PORT_INDEX 3
+#endif
+#ifndef HAVE_RDMA_NLDEV_ATTR_NDEV_INDEX
+#define RDMA_NLDEV_ATTR_NDEV_INDEX 50
+#endif
+
+/* These are normally found in linux/if_link.h. */
+#ifndef HAVE_IFLA_PHYS_SWITCH_ID
+#define IFLA_PHYS_SWITCH_ID 36
+#endif
+#ifndef HAVE_IFLA_PHYS_PORT_NAME
+#define IFLA_PHYS_PORT_NAME 38
+#endif
+
+/* Add/remove MAC address through Netlink */
+struct mlx5_nl_mac_addr {
+	struct ether_addr (*mac)[];
+	/**< MAC address handled by the device. */
+	int mac_n; /**< Number of addresses in the array. */
+};
+
+/** Data structure used by mlx5_nl_ifindex_cb(). */
+struct mlx5_nl_ifindex_data {
+	const char *name; /**< IB device name (in). */
+	uint32_t ibindex; /**< IB device index (out). */
+	uint32_t ifindex; /**< Network interface index (out). */
+};
+
+/**
+ * Opens a Netlink socket.
+ *
+ * @param protocol
+ *   Netlink protocol (e.g. NETLINK_ROUTE, NETLINK_RDMA).
+ *
+ * @return
+ *   A file descriptor on success, a negative errno value otherwise and
+ *   rte_errno is set.
+ */
+int
+mlx5_nl_init(int protocol)
+{
+	int fd;
+	int sndbuf_size = MLX5_SEND_BUF_SIZE;
+	int rcvbuf_size = MLX5_RECV_BUF_SIZE;
+	struct sockaddr_nl local = {
+		.nl_family = AF_NETLINK,
+	};
+	int ret;
+
+	fd = socket(AF_NETLINK, SOCK_RAW | SOCK_CLOEXEC, protocol);
+	if (fd == -1) {
+		rte_errno = errno;
+		return -rte_errno;
+	}
+	ret = setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &sndbuf_size, sizeof(int));
+	if (ret == -1) {
+		rte_errno = errno;
+		goto error;
+	}
+	ret = setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &rcvbuf_size, sizeof(int));
+	if (ret == -1) {
+		rte_errno = errno;
+		goto error;
+	}
+	ret = bind(fd, (struct sockaddr *)&local, sizeof(local));
+	if (ret == -1) {
+		rte_errno = errno;
+		goto error;
+	}
+	return fd;
+error:
+	close(fd);
+	return -rte_errno;
+}
+
+/**
+ * Send a request message to the kernel on the Netlink socket.
+ *
+ * @param[in] nlsk_fd
+ *   Netlink socket file descriptor.
+ * @param[in] nh
+ *   The Netlink message send to the kernel.
+ * @param[in] ssn
+ *   Sequence number.
+ * @param[in] req
+ *   Pointer to the request structure.
+ * @param[in] len
+ *   Length of the request in bytes.
+ *
+ * @return
+ *   The number of sent bytes on success, a negative errno value otherwise and
+ *   rte_errno is set.
+ */
+static int
+mlx5_nl_request(int nlsk_fd, struct nlmsghdr *nh, uint32_t sn, void *req,
+		int len)
+{
+	struct sockaddr_nl sa = {
+		.nl_family = AF_NETLINK,
+	};
+	struct iovec iov[2] = {
+		{ .iov_base = nh, .iov_len = sizeof(*nh), },
+		{ .iov_base = req, .iov_len = len, },
+	};
+	struct msghdr msg = {
+		.msg_name = &sa,
+		.msg_namelen = sizeof(sa),
+		.msg_iov = iov,
+		.msg_iovlen = 2,
+	};
+	int send_bytes;
+
+	nh->nlmsg_pid = 0; /* communication with the kernel uses pid 0 */
+	nh->nlmsg_seq = sn;
+	send_bytes = sendmsg(nlsk_fd, &msg, 0);
+	if (send_bytes < 0) {
+		rte_errno = errno;
+		return -rte_errno;
+	}
+	return send_bytes;
+}
+
+/**
+ * Send a message to the kernel on the Netlink socket.
+ *
+ * @param[in] nlsk_fd
+ *   The Netlink socket file descriptor used for communication.
+ * @param[in] nh
+ *   The Netlink message send to the kernel.
+ * @param[in] sn
+ *   Sequence number.
+ *
+ * @return
+ *   The number of sent bytes on success, a negative errno value otherwise and
+ *   rte_errno is set.
+ */
+static int
+mlx5_nl_send(int nlsk_fd, struct nlmsghdr *nh, uint32_t sn)
+{
+	struct sockaddr_nl sa = {
+		.nl_family = AF_NETLINK,
+	};
+	struct iovec iov = {
+		.iov_base = nh,
+		.iov_len = nh->nlmsg_len,
+	};
+	struct msghdr msg = {
+		.msg_name = &sa,
+		.msg_namelen = sizeof(sa),
+		.msg_iov = &iov,
+		.msg_iovlen = 1,
+	};
+	int send_bytes;
+
+	nh->nlmsg_pid = 0; /* communication with the kernel uses pid 0 */
+	nh->nlmsg_seq = sn;
+	send_bytes = sendmsg(nlsk_fd, &msg, 0);
+	if (send_bytes < 0) {
+		rte_errno = errno;
+		return -rte_errno;
+	}
+	return send_bytes;
+}
+
+/**
+ * Receive a message from the kernel on the Netlink socket, following
+ * mlx5_nl_send().
+ *
+ * @param[in] nlsk_fd
+ *   The Netlink socket file descriptor used for communication.
+ * @param[in] sn
+ *   Sequence number.
+ * @param[in] cb
+ *   The callback function to call for each Netlink message received.
+ * @param[in, out] arg
+ *   Custom arguments for the callback.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_nl_recv(int nlsk_fd, uint32_t sn, int (*cb)(struct nlmsghdr *, void *arg),
+	     void *arg)
+{
+	struct sockaddr_nl sa;
+	char buf[MLX5_RECV_BUF_SIZE];
+	struct iovec iov = {
+		.iov_base = buf,
+		.iov_len = sizeof(buf),
+	};
+	struct msghdr msg = {
+		.msg_name = &sa,
+		.msg_namelen = sizeof(sa),
+		.msg_iov = &iov,
+		/* One message at a time */
+		.msg_iovlen = 1,
+	};
+	int multipart = 0;
+	int ret = 0;
+
+	do {
+		struct nlmsghdr *nh;
+		int recv_bytes = 0;
+
+		do {
+			recv_bytes = recvmsg(nlsk_fd, &msg, 0);
+			if (recv_bytes == -1) {
+				rte_errno = errno;
+				return -rte_errno;
+			}
+			nh = (struct nlmsghdr *)buf;
+		} while (nh->nlmsg_seq != sn);
+		for (;
+		     NLMSG_OK(nh, (unsigned int)recv_bytes);
+		     nh = NLMSG_NEXT(nh, recv_bytes)) {
+			if (nh->nlmsg_type == NLMSG_ERROR) {
+				struct nlmsgerr *err_data = NLMSG_DATA(nh);
+
+				if (err_data->error < 0) {
+					rte_errno = -err_data->error;
+					return -rte_errno;
+				}
+				/* Ack message. */
+				return 0;
+			}
+			/* Multi-part msgs and their trailing DONE message. */
+			if (nh->nlmsg_flags & NLM_F_MULTI) {
+				if (nh->nlmsg_type == NLMSG_DONE)
+					return 0;
+				multipart = 1;
+			}
+			if (cb) {
+				ret = cb(nh, arg);
+				if (ret < 0)
+					return ret;
+			}
+		}
+	} while (multipart);
+	return ret;
+}
+
+/**
+ * Parse Netlink message to retrieve the bridge MAC address.
+ *
+ * @param nh
+ *   Pointer to Netlink Message Header.
+ * @param arg
+ *   PMD data register with this callback.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_nl_mac_addr_cb(struct nlmsghdr *nh, void *arg)
+{
+	struct mlx5_nl_mac_addr *data = arg;
+	struct ndmsg *r = NLMSG_DATA(nh);
+	struct rtattr *attribute;
+	int len;
+
+	len = nh->nlmsg_len - NLMSG_LENGTH(sizeof(*r));
+	for (attribute = MLX5_NDA_RTA(r);
+	     RTA_OK(attribute, len);
+	     attribute = RTA_NEXT(attribute, len)) {
+		if (attribute->rta_type == NDA_LLADDR) {
+			if (data->mac_n == MLX5_MAX_MAC_ADDRESSES) {
+				DRV_LOG(WARNING,
+					"not enough room to finalize the"
+					" request");
+				rte_errno = ENOMEM;
+				return -rte_errno;
+			}
+#ifndef NDEBUG
+			char m[18];
+
+			ether_format_addr(m, 18, RTA_DATA(attribute));
+			DRV_LOG(DEBUG, "bridge MAC address %s", m);
+#endif
+			memcpy(&(*data->mac)[data->mac_n++],
+			       RTA_DATA(attribute), ETHER_ADDR_LEN);
+		}
+	}
+	return 0;
+}
+
+/**
+ * Get bridge MAC addresses.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param mac[out]
+ *   Pointer to the array table of MAC addresses to fill.
+ *   Its size should be of MLX5_MAX_MAC_ADDRESSES.
+ * @param mac_n[out]
+ *   Number of entries filled in MAC array.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_nl_mac_addr_list(struct rte_eth_dev *dev, struct ether_addr (*mac)[],
+		      int *mac_n)
+{
+	struct priv *priv = dev->data->dev_private;
+	unsigned int iface_idx = mlx5_ifindex(dev);
+	struct {
+		struct nlmsghdr	hdr;
+		struct ifinfomsg ifm;
+	} req = {
+		.hdr = {
+			.nlmsg_len = NLMSG_LENGTH(sizeof(struct ifinfomsg)),
+			.nlmsg_type = RTM_GETNEIGH,
+			.nlmsg_flags = NLM_F_DUMP | NLM_F_REQUEST,
+		},
+		.ifm = {
+			.ifi_family = PF_BRIDGE,
+			.ifi_index = iface_idx,
+		},
+	};
+	struct mlx5_nl_mac_addr data = {
+		.mac = mac,
+		.mac_n = 0,
+	};
+	int fd;
+	int ret;
+	uint32_t sn = priv->nl_sn++;
+
+	if (priv->nl_socket_route == -1)
+		return 0;
+	fd = priv->nl_socket_route;
+	ret = mlx5_nl_request(fd, &req.hdr, sn, &req.ifm,
+			      sizeof(struct ifinfomsg));
+	if (ret < 0)
+		goto error;
+	ret = mlx5_nl_recv(fd, sn, mlx5_nl_mac_addr_cb, &data);
+	if (ret < 0)
+		goto error;
+	*mac_n = data.mac_n;
+	return 0;
+error:
+	DRV_LOG(DEBUG, "port %u cannot retrieve MAC address list %s",
+		dev->data->port_id, strerror(rte_errno));
+	return -rte_errno;
+}
+
+/**
+ * Modify the MAC address neighbour table with Netlink.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param mac
+ *   MAC address to consider.
+ * @param add
+ *   1 to add the MAC address, 0 to remove the MAC address.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_nl_mac_addr_modify(struct rte_eth_dev *dev, struct ether_addr *mac,
+			int add)
+{
+	struct priv *priv = dev->data->dev_private;
+	unsigned int iface_idx = mlx5_ifindex(dev);
+	struct {
+		struct nlmsghdr hdr;
+		struct ndmsg ndm;
+		struct rtattr rta;
+		uint8_t buffer[ETHER_ADDR_LEN];
+	} req = {
+		.hdr = {
+			.nlmsg_len = NLMSG_LENGTH(sizeof(struct ndmsg)),
+			.nlmsg_flags = NLM_F_REQUEST | NLM_F_CREATE |
+				NLM_F_EXCL | NLM_F_ACK,
+			.nlmsg_type = add ? RTM_NEWNEIGH : RTM_DELNEIGH,
+		},
+		.ndm = {
+			.ndm_family = PF_BRIDGE,
+			.ndm_state = NUD_NOARP | NUD_PERMANENT,
+			.ndm_ifindex = iface_idx,
+			.ndm_flags = NTF_SELF,
+		},
+		.rta = {
+			.rta_type = NDA_LLADDR,
+			.rta_len = RTA_LENGTH(ETHER_ADDR_LEN),
+		},
+	};
+	int fd;
+	int ret;
+	uint32_t sn = priv->nl_sn++;
+
+	if (priv->nl_socket_route == -1)
+		return 0;
+	fd = priv->nl_socket_route;
+	memcpy(RTA_DATA(&req.rta), mac, ETHER_ADDR_LEN);
+	req.hdr.nlmsg_len = NLMSG_ALIGN(req.hdr.nlmsg_len) +
+		RTA_ALIGN(req.rta.rta_len);
+	ret = mlx5_nl_send(fd, &req.hdr, sn);
+	if (ret < 0)
+		goto error;
+	ret = mlx5_nl_recv(fd, sn, NULL, NULL);
+	if (ret < 0)
+		goto error;
+	return 0;
+error:
+	DRV_LOG(DEBUG,
+		"port %u cannot %s MAC address %02X:%02X:%02X:%02X:%02X:%02X"
+		" %s",
+		dev->data->port_id,
+		add ? "add" : "remove",
+		mac->addr_bytes[0], mac->addr_bytes[1],
+		mac->addr_bytes[2], mac->addr_bytes[3],
+		mac->addr_bytes[4], mac->addr_bytes[5],
+		strerror(rte_errno));
+	return -rte_errno;
+}
+
+/**
+ * Add a MAC address.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param mac
+ *   MAC address to register.
+ * @param index
+ *   MAC address index.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_nl_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac,
+		     uint32_t index)
+{
+	struct priv *priv = dev->data->dev_private;
+	int ret;
+
+	ret = mlx5_nl_mac_addr_modify(dev, mac, 1);
+	if (!ret)
+		BITFIELD_SET(priv->mac_own, index);
+	if (ret == -EEXIST)
+		return 0;
+	return ret;
+}
+
+/**
+ * Remove a MAC address.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param mac
+ *   MAC address to remove.
+ * @param index
+ *   MAC address index.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_nl_mac_addr_remove(struct rte_eth_dev *dev, struct ether_addr *mac,
+			uint32_t index)
+{
+	struct priv *priv = dev->data->dev_private;
+
+	BITFIELD_RESET(priv->mac_own, index);
+	return mlx5_nl_mac_addr_modify(dev, mac, 0);
+}
+
+/**
+ * Synchronize Netlink bridge table to the internal table.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ */
+void
+mlx5_nl_mac_addr_sync(struct rte_eth_dev *dev)
+{
+	struct ether_addr macs[MLX5_MAX_MAC_ADDRESSES];
+	int macs_n = 0;
+	int i;
+	int ret;
+
+	ret = mlx5_nl_mac_addr_list(dev, &macs, &macs_n);
+	if (ret)
+		return;
+	for (i = 0; i != macs_n; ++i) {
+		int j;
+
+		/* Verify the address is not in the array yet. */
+		for (j = 0; j != MLX5_MAX_MAC_ADDRESSES; ++j)
+			if (is_same_ether_addr(&macs[i],
+					       &dev->data->mac_addrs[j]))
+				break;
+		if (j != MLX5_MAX_MAC_ADDRESSES)
+			continue;
+		/* Find the first entry available. */
+		for (j = 0; j != MLX5_MAX_MAC_ADDRESSES; ++j) {
+			if (is_zero_ether_addr(&dev->data->mac_addrs[j])) {
+				dev->data->mac_addrs[j] = macs[i];
+				break;
+			}
+		}
+	}
+}
+
+/**
+ * Flush all added MAC addresses.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ */
+void
+mlx5_nl_mac_addr_flush(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	int i;
+
+	for (i = MLX5_MAX_MAC_ADDRESSES - 1; i >= 0; --i) {
+		struct ether_addr *m = &dev->data->mac_addrs[i];
+
+		if (BITFIELD_ISSET(priv->mac_own, i))
+			mlx5_nl_mac_addr_remove(dev, m, i);
+	}
+}
+
+/**
+ * Enable promiscuous / all multicast mode through Netlink.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param flags
+ *   IFF_PROMISC for promiscuous, IFF_ALLMULTI for allmulti.
+ * @param enable
+ *   Nonzero to enable, disable otherwise.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_nl_device_flags(struct rte_eth_dev *dev, uint32_t flags, int enable)
+{
+	struct priv *priv = dev->data->dev_private;
+	unsigned int iface_idx = mlx5_ifindex(dev);
+	struct {
+		struct nlmsghdr hdr;
+		struct ifinfomsg ifi;
+	} req = {
+		.hdr = {
+			.nlmsg_len = NLMSG_LENGTH(sizeof(struct ifinfomsg)),
+			.nlmsg_type = RTM_NEWLINK,
+			.nlmsg_flags = NLM_F_REQUEST,
+		},
+		.ifi = {
+			.ifi_flags = enable ? flags : 0,
+			.ifi_change = flags,
+			.ifi_index = iface_idx,
+		},
+	};
+	int fd;
+	int ret;
+
+	assert(!(flags & ~(IFF_PROMISC | IFF_ALLMULTI)));
+	if (priv->nl_socket_route < 0)
+		return 0;
+	fd = priv->nl_socket_route;
+	ret = mlx5_nl_send(fd, &req.hdr, priv->nl_sn++);
+	if (ret < 0)
+		return ret;
+	return 0;
+}
+
+/**
+ * Enable promiscuous mode through Netlink.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param enable
+ *   Nonzero to enable, disable otherwise.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_nl_promisc(struct rte_eth_dev *dev, int enable)
+{
+	int ret = mlx5_nl_device_flags(dev, IFF_PROMISC, enable);
+
+	if (ret)
+		DRV_LOG(DEBUG,
+			"port %u cannot %s promisc mode: Netlink error %s",
+			dev->data->port_id, enable ? "enable" : "disable",
+			strerror(rte_errno));
+	return ret;
+}
+
+/**
+ * Enable all multicast mode through Netlink.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param enable
+ *   Nonzero to enable, disable otherwise.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_nl_allmulti(struct rte_eth_dev *dev, int enable)
+{
+	int ret = mlx5_nl_device_flags(dev, IFF_ALLMULTI, enable);
+
+	if (ret)
+		DRV_LOG(DEBUG,
+			"port %u cannot %s allmulti mode: Netlink error %s",
+			dev->data->port_id, enable ? "enable" : "disable",
+			strerror(rte_errno));
+	return ret;
+}
+
+/**
+ * Process network interface information from Netlink message.
+ *
+ * @param nh
+ *   Pointer to Netlink message header.
+ * @param arg
+ *   Opaque data pointer for this callback.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_nl_ifindex_cb(struct nlmsghdr *nh, void *arg)
+{
+	struct mlx5_nl_ifindex_data *data = arg;
+	size_t off = NLMSG_HDRLEN;
+	uint32_t ibindex = 0;
+	uint32_t ifindex = 0;
+	int found = 0;
+
+	if (nh->nlmsg_type !=
+	    RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_GET) &&
+	    nh->nlmsg_type !=
+	    RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_PORT_GET))
+		goto error;
+	while (off < nh->nlmsg_len) {
+		struct nlattr *na = (void *)((uintptr_t)nh + off);
+		void *payload = (void *)((uintptr_t)na + NLA_HDRLEN);
+
+		if (na->nla_len > nh->nlmsg_len - off)
+			goto error;
+		switch (na->nla_type) {
+		case RDMA_NLDEV_ATTR_DEV_INDEX:
+			ibindex = *(uint32_t *)payload;
+			break;
+		case RDMA_NLDEV_ATTR_DEV_NAME:
+			if (!strcmp(payload, data->name))
+				found = 1;
+			break;
+		case RDMA_NLDEV_ATTR_NDEV_INDEX:
+			ifindex = *(uint32_t *)payload;
+			break;
+		default:
+			break;
+		}
+		off += NLA_ALIGN(na->nla_len);
+	}
+	if (found) {
+		data->ibindex = ibindex;
+		data->ifindex = ifindex;
+	}
+	return 0;
+error:
+	rte_errno = EINVAL;
+	return -rte_errno;
+}
+
+/**
+ * Get index of network interface associated with some IB device.
+ *
+ * This is the only somewhat safe method to avoid resorting to heuristics
+ * when faced with port representors. Unfortunately it requires at least
+ * Linux 4.17.
+ *
+ * @param nl
+ *   Netlink socket of the RDMA kind (NETLINK_RDMA).
+ * @param[in] name
+ *   IB device name.
+ *
+ * @return
+ *   A valid (nonzero) interface index on success, 0 otherwise and rte_errno
+ *   is set.
+ */
+unsigned int
+mlx5_nl_ifindex(int nl, const char *name)
+{
+	static const uint32_t pindex = 1;
+	uint32_t seq = random();
+	struct mlx5_nl_ifindex_data data = {
+		.name = name,
+		.ibindex = 0, /* Determined during first pass. */
+		.ifindex = 0, /* Determined during second pass. */
+	};
+	union {
+		struct nlmsghdr nh;
+		uint8_t buf[NLMSG_HDRLEN +
+			    NLA_HDRLEN + NLA_ALIGN(sizeof(data.ibindex)) +
+			    NLA_HDRLEN + NLA_ALIGN(sizeof(pindex))];
+	} req = {
+		.nh = {
+			.nlmsg_len = NLMSG_LENGTH(0),
+			.nlmsg_type = RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
+						       RDMA_NLDEV_CMD_GET),
+			.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_DUMP,
+		},
+	};
+	struct nlattr *na;
+	int ret;
+
+	ret = mlx5_nl_send(nl, &req.nh, seq);
+	if (ret < 0)
+		return 0;
+	ret = mlx5_nl_recv(nl, seq, mlx5_nl_ifindex_cb, &data);
+	if (ret < 0)
+		return 0;
+	if (!data.ibindex)
+		goto error;
+	++seq;
+	req.nh.nlmsg_type = RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
+					     RDMA_NLDEV_CMD_PORT_GET);
+	req.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK;
+	req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(req.buf) - NLMSG_HDRLEN);
+	na = (void *)((uintptr_t)req.buf + NLMSG_HDRLEN);
+	na->nla_len = NLA_HDRLEN + sizeof(data.ibindex);
+	na->nla_type = RDMA_NLDEV_ATTR_DEV_INDEX;
+	memcpy((void *)((uintptr_t)na + NLA_HDRLEN),
+	       &data.ibindex, sizeof(data.ibindex));
+	na = (void *)((uintptr_t)na + NLA_ALIGN(na->nla_len));
+	na->nla_len = NLA_HDRLEN + sizeof(pindex);
+	na->nla_type = RDMA_NLDEV_ATTR_PORT_INDEX;
+	memcpy((void *)((uintptr_t)na + NLA_HDRLEN),
+	       &pindex, sizeof(pindex));
+	ret = mlx5_nl_send(nl, &req.nh, seq);
+	if (ret < 0)
+		return 0;
+	ret = mlx5_nl_recv(nl, seq, mlx5_nl_ifindex_cb, &data);
+	if (ret < 0)
+		return 0;
+	if (!data.ifindex)
+		goto error;
+	return data.ifindex;
+error:
+	rte_errno = ENODEV;
+	return 0;
+}
+
+/**
+ * Process switch information from Netlink message.
+ *
+ * @param nh
+ *   Pointer to Netlink message header.
+ * @param arg
+ *   Opaque data pointer for this callback.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_nl_switch_info_cb(struct nlmsghdr *nh, void *arg)
+{
+	struct mlx5_switch_info info = {
+		.master = 0,
+		.representor = 0,
+		.port_name = 0,
+		.switch_id = 0,
+	};
+	size_t off = NLMSG_LENGTH(sizeof(struct ifinfomsg));
+	bool port_name_set = false;
+	bool switch_id_set = false;
+
+	if (nh->nlmsg_type != RTM_NEWLINK)
+		goto error;
+	while (off < nh->nlmsg_len) {
+		struct rtattr *ra = (void *)((uintptr_t)nh + off);
+		void *payload = RTA_DATA(ra);
+		char *end;
+		unsigned int i;
+
+		if (ra->rta_len > nh->nlmsg_len - off)
+			goto error;
+		switch (ra->rta_type) {
+		case IFLA_PHYS_PORT_NAME:
+			errno = 0;
+			info.port_name = strtol(payload, &end, 0);
+			if (errno ||
+			    (size_t)(end - (char *)payload) != strlen(payload))
+				goto error;
+			port_name_set = true;
+			break;
+		case IFLA_PHYS_SWITCH_ID:
+			info.switch_id = 0;
+			for (i = 0; i < RTA_PAYLOAD(ra); ++i) {
+				info.switch_id <<= 8;
+				info.switch_id |= ((uint8_t *)payload)[i];
+			}
+			switch_id_set = true;
+			break;
+		}
+		off += RTA_ALIGN(ra->rta_len);
+	}
+	info.master = switch_id_set && !port_name_set;
+	info.representor = switch_id_set && port_name_set;
+	memcpy(arg, &info, sizeof(info));
+	return 0;
+error:
+	rte_errno = EINVAL;
+	return -rte_errno;
+}
+
+/**
+ * Get switch information associated with network interface.
+ *
+ * @param nl
+ *   Netlink socket of the ROUTE kind (NETLINK_ROUTE).
+ * @param ifindex
+ *   Network interface index.
+ * @param[out] info
+ *   Switch information object, populated in case of success.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_nl_switch_info(int nl, unsigned int ifindex, struct mlx5_switch_info *info)
+{
+	uint32_t seq = random();
+	struct {
+		struct nlmsghdr nh;
+		struct ifinfomsg info;
+	} req = {
+		.nh = {
+			.nlmsg_len = NLMSG_LENGTH(sizeof(req.info)),
+			.nlmsg_type = RTM_GETLINK,
+			.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK,
+		},
+		.info = {
+			.ifi_family = AF_UNSPEC,
+			.ifi_index = ifindex,
+		},
+	};
+	int ret;
+
+	ret = mlx5_nl_send(nl, &req.nh, seq);
+	if (ret >= 0)
+		ret = mlx5_nl_recv(nl, seq, mlx5_nl_switch_info_cb, info);
+	return ret;
+}
diff --git a/src/spdk/dpdk/drivers/net/mlx5/mlx5_nl_flow.c b/src/spdk/dpdk/drivers/net/mlx5/mlx5_nl_flow.c
new file mode 100644
index 00000000..a1c8c340
--- /dev/null
+++ b/src/spdk/dpdk/drivers/net/mlx5/mlx5_nl_flow.c
@@ -0,0 +1,1248 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 6WIND S.A.
+ * Copyright 2018 Mellanox Technologies, Ltd
+ */
+
+#include <assert.h>
+#include <errno.h>
+#include <libmnl/libmnl.h>
+#include <linux/if_ether.h>
+#include <linux/netlink.h>
+#include <linux/pkt_cls.h>
+#include <linux/pkt_sched.h>
+#include <linux/rtnetlink.h>
+#include <linux/tc_act/tc_gact.h>
+#include <linux/tc_act/tc_mirred.h>
+#include <netinet/in.h>
+#include <stdalign.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <sys/socket.h>
+
+#include <rte_byteorder.h>
+#include <rte_errno.h>
+#include <rte_ether.h>
+#include <rte_flow.h>
+
+#include "mlx5.h"
+#include "mlx5_autoconf.h"
+
+#ifdef HAVE_TC_ACT_VLAN
+
+#include <linux/tc_act/tc_vlan.h>
+
+#else /* HAVE_TC_ACT_VLAN */
+
+#define TCA_VLAN_ACT_POP 1
+#define TCA_VLAN_ACT_PUSH 2
+#define TCA_VLAN_ACT_MODIFY 3
+#define TCA_VLAN_PARMS 2
+#define TCA_VLAN_PUSH_VLAN_ID 3
+#define TCA_VLAN_PUSH_VLAN_PROTOCOL 4
+#define TCA_VLAN_PAD 5
+#define TCA_VLAN_PUSH_VLAN_PRIORITY 6
+
+struct tc_vlan {
+	tc_gen;
+	int v_action;
+};
+
+#endif /* HAVE_TC_ACT_VLAN */
+
+/* Normally found in linux/netlink.h. */
+#ifndef NETLINK_CAP_ACK
+#define NETLINK_CAP_ACK 10
+#endif
+
+/* Normally found in linux/pkt_sched.h. */
+#ifndef TC_H_MIN_INGRESS
+#define TC_H_MIN_INGRESS 0xfff2u
+#endif
+
+/* Normally found in linux/pkt_cls.h. */
+#ifndef TCA_CLS_FLAGS_SKIP_SW
+#define TCA_CLS_FLAGS_SKIP_SW (1 << 1)
+#endif
+#ifndef HAVE_TCA_FLOWER_ACT
+#define TCA_FLOWER_ACT 3
+#endif
+#ifndef HAVE_TCA_FLOWER_FLAGS
+#define TCA_FLOWER_FLAGS 22
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_ETH_TYPE
+#define TCA_FLOWER_KEY_ETH_TYPE 8
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_ETH_DST
+#define TCA_FLOWER_KEY_ETH_DST 4
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_ETH_DST_MASK
+#define TCA_FLOWER_KEY_ETH_DST_MASK 5
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_ETH_SRC
+#define TCA_FLOWER_KEY_ETH_SRC 6
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_ETH_SRC_MASK
+#define TCA_FLOWER_KEY_ETH_SRC_MASK 7
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_IP_PROTO
+#define TCA_FLOWER_KEY_IP_PROTO 9
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_IPV4_SRC
+#define TCA_FLOWER_KEY_IPV4_SRC 10
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_IPV4_SRC_MASK
+#define TCA_FLOWER_KEY_IPV4_SRC_MASK 11
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_IPV4_DST
+#define TCA_FLOWER_KEY_IPV4_DST 12
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_IPV4_DST_MASK
+#define TCA_FLOWER_KEY_IPV4_DST_MASK 13
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_IPV6_SRC
+#define TCA_FLOWER_KEY_IPV6_SRC 14
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_IPV6_SRC_MASK
+#define TCA_FLOWER_KEY_IPV6_SRC_MASK 15
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_IPV6_DST
+#define TCA_FLOWER_KEY_IPV6_DST 16
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_IPV6_DST_MASK
+#define TCA_FLOWER_KEY_IPV6_DST_MASK 17
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_TCP_SRC
+#define TCA_FLOWER_KEY_TCP_SRC 18
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_TCP_SRC_MASK
+#define TCA_FLOWER_KEY_TCP_SRC_MASK 35
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_TCP_DST
+#define TCA_FLOWER_KEY_TCP_DST 19
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_TCP_DST_MASK
+#define TCA_FLOWER_KEY_TCP_DST_MASK 36
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_UDP_SRC
+#define TCA_FLOWER_KEY_UDP_SRC 20
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_UDP_SRC_MASK
+#define TCA_FLOWER_KEY_UDP_SRC_MASK 37
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_UDP_DST
+#define TCA_FLOWER_KEY_UDP_DST 21
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_UDP_DST_MASK
+#define TCA_FLOWER_KEY_UDP_DST_MASK 38
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_VLAN_ID
+#define TCA_FLOWER_KEY_VLAN_ID 23
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_VLAN_PRIO
+#define TCA_FLOWER_KEY_VLAN_PRIO 24
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_VLAN_ETH_TYPE
+#define TCA_FLOWER_KEY_VLAN_ETH_TYPE 25
+#endif
+
+/** Parser state definitions for mlx5_nl_flow_trans[]. */
+enum mlx5_nl_flow_trans {
+	INVALID,
+	BACK,
+	ATTR,
+	PATTERN,
+	ITEM_VOID,
+	ITEM_PORT_ID,
+	ITEM_ETH,
+	ITEM_VLAN,
+	ITEM_IPV4,
+	ITEM_IPV6,
+	ITEM_TCP,
+	ITEM_UDP,
+	ACTIONS,
+	ACTION_VOID,
+	ACTION_PORT_ID,
+	ACTION_DROP,
+	ACTION_OF_POP_VLAN,
+	ACTION_OF_PUSH_VLAN,
+	ACTION_OF_SET_VLAN_VID,
+	ACTION_OF_SET_VLAN_PCP,
+	END,
+};
+
+#define TRANS(...) (const enum mlx5_nl_flow_trans []){ __VA_ARGS__, INVALID, }
+
+#define PATTERN_COMMON \
+	ITEM_VOID, ITEM_PORT_ID, ACTIONS
+#define ACTIONS_COMMON \
+	ACTION_VOID, ACTION_OF_POP_VLAN, ACTION_OF_PUSH_VLAN, \
+	ACTION_OF_SET_VLAN_VID, ACTION_OF_SET_VLAN_PCP
+#define ACTIONS_FATE \
+	ACTION_PORT_ID, ACTION_DROP
+
+/** Parser state transitions used by mlx5_nl_flow_transpose(). */
+static const enum mlx5_nl_flow_trans *const mlx5_nl_flow_trans[] = {
+	[INVALID] = NULL,
+	[BACK] = NULL,
+	[ATTR] = TRANS(PATTERN),
+	[PATTERN] = TRANS(ITEM_ETH, PATTERN_COMMON),
+	[ITEM_VOID] = TRANS(BACK),
+	[ITEM_PORT_ID] = TRANS(BACK),
+	[ITEM_ETH] = TRANS(ITEM_IPV4, ITEM_IPV6, ITEM_VLAN, PATTERN_COMMON),
+	[ITEM_VLAN] = TRANS(ITEM_IPV4, ITEM_IPV6, PATTERN_COMMON),
+	[ITEM_IPV4] = TRANS(ITEM_TCP, ITEM_UDP, PATTERN_COMMON),
+	[ITEM_IPV6] = TRANS(ITEM_TCP, ITEM_UDP, PATTERN_COMMON),
+	[ITEM_TCP] = TRANS(PATTERN_COMMON),
+	[ITEM_UDP] = TRANS(PATTERN_COMMON),
+	[ACTIONS] = TRANS(ACTIONS_FATE, ACTIONS_COMMON),
+	[ACTION_VOID] = TRANS(BACK),
+	[ACTION_PORT_ID] = TRANS(ACTION_VOID, END),
+	[ACTION_DROP] = TRANS(ACTION_VOID, END),
+	[ACTION_OF_POP_VLAN] = TRANS(ACTIONS_FATE, ACTIONS_COMMON),
+	[ACTION_OF_PUSH_VLAN] = TRANS(ACTIONS_FATE, ACTIONS_COMMON),
+	[ACTION_OF_SET_VLAN_VID] = TRANS(ACTIONS_FATE, ACTIONS_COMMON),
+	[ACTION_OF_SET_VLAN_PCP] = TRANS(ACTIONS_FATE, ACTIONS_COMMON),
+	[END] = NULL,
+};
+
+/** Empty masks for known item types. */
+static const union {
+	struct rte_flow_item_port_id port_id;
+	struct rte_flow_item_eth eth;
+	struct rte_flow_item_vlan vlan;
+	struct rte_flow_item_ipv4 ipv4;
+	struct rte_flow_item_ipv6 ipv6;
+	struct rte_flow_item_tcp tcp;
+	struct rte_flow_item_udp udp;
+} mlx5_nl_flow_mask_empty;
+
+/** Supported masks for known item types. */
+static const struct {
+	struct rte_flow_item_port_id port_id;
+	struct rte_flow_item_eth eth;
+	struct rte_flow_item_vlan vlan;
+	struct rte_flow_item_ipv4 ipv4;
+	struct rte_flow_item_ipv6 ipv6;
+	struct rte_flow_item_tcp tcp;
+	struct rte_flow_item_udp udp;
+} mlx5_nl_flow_mask_supported = {
+	.port_id = {
+		.id = 0xffffffff,
+	},
+	.eth = {
+		.type = RTE_BE16(0xffff),
+		.dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
+		.src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
+	},
+	.vlan = {
+		/* PCP and VID only, no DEI. */
+		.tci = RTE_BE16(0xefff),
+		.inner_type = RTE_BE16(0xffff),
+	},
+	.ipv4.hdr = {
+		.next_proto_id = 0xff,
+		.src_addr = RTE_BE32(0xffffffff),
+		.dst_addr = RTE_BE32(0xffffffff),
+	},
+	.ipv6.hdr = {
+		.proto = 0xff,
+		.src_addr =
+			"\xff\xff\xff\xff\xff\xff\xff\xff"
+			"\xff\xff\xff\xff\xff\xff\xff\xff",
+		.dst_addr =
+			"\xff\xff\xff\xff\xff\xff\xff\xff"
+			"\xff\xff\xff\xff\xff\xff\xff\xff",
+	},
+	.tcp.hdr = {
+		.src_port = RTE_BE16(0xffff),
+		.dst_port = RTE_BE16(0xffff),
+	},
+	.udp.hdr = {
+		.src_port = RTE_BE16(0xffff),
+		.dst_port = RTE_BE16(0xffff),
+	},
+};
+
+/**
+ * Retrieve mask for pattern item.
+ *
+ * This function does basic sanity checks on a pattern item in order to
+ * return the most appropriate mask for it.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in] mask_default
+ *   Default mask for pattern item as specified by the flow API.
+ * @param[in] mask_supported
+ *   Mask fields supported by the implementation.
+ * @param[in] mask_empty
+ *   Empty mask to return when there is no specification.
+ * @param[out] error
+ *   Perform verbose error reporting if not NULL.
+ *
+ * @return
+ *   Either @p item->mask or one of the mask parameters on success, NULL
+ *   otherwise and rte_errno is set.
+ */
+static const void *
+mlx5_nl_flow_item_mask(const struct rte_flow_item *item,
+		       const void *mask_default,
+		       const void *mask_supported,
+		       const void *mask_empty,
+		       size_t mask_size,
+		       struct rte_flow_error *error)
+{
+	const uint8_t *mask;
+	size_t i;
+
+	/* item->last and item->mask cannot exist without item->spec. */
+	if (!item->spec && (item->mask || item->last)) {
+		rte_flow_error_set
+			(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
+			 "\"mask\" or \"last\" field provided without a"
+			 " corresponding \"spec\"");
+		return NULL;
+	}
+	/* No spec, no mask, no problem. */
+	if (!item->spec)
+		return mask_empty;
+	mask = item->mask ? item->mask : mask_default;
+	assert(mask);
+	/*
+	 * Single-pass check to make sure that:
+	 * - Mask is supported, no bits are set outside mask_supported.
+	 * - Both item->spec and item->last are included in mask.
+	 */
+	for (i = 0; i != mask_size; ++i) {
+		if (!mask[i])
+			continue;
+		if ((mask[i] | ((const uint8_t *)mask_supported)[i]) !=
+		    ((const uint8_t *)mask_supported)[i]) {
+			rte_flow_error_set
+				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM_MASK,
+				 mask, "unsupported field found in \"mask\"");
+			return NULL;
+		}
+		if (item->last &&
+		    (((const uint8_t *)item->spec)[i] & mask[i]) !=
+		    (((const uint8_t *)item->last)[i] & mask[i])) {
+			rte_flow_error_set
+				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM_LAST,
+				 item->last,
+				 "range between \"spec\" and \"last\" not"
+				 " comprised in \"mask\"");
+			return NULL;
+		}
+	}
+	return mask;
+}
+
+/**
+ * Transpose flow rule description to rtnetlink message.
+ *
+ * This function transposes a flow rule description to a traffic control
+ * (TC) filter creation message ready to be sent over Netlink.
+ *
+ * Target interface is specified as the first entry of the @p ptoi table.
+ * Subsequent entries enable this function to resolve other DPDK port IDs
+ * found in the flow rule.
+ *
+ * @param[out] buf
+ *   Output message buffer. May be NULL when @p size is 0.
+ * @param size
+ *   Size of @p buf. Message may be truncated if not large enough.
+ * @param[in] ptoi
+ *   DPDK port ID to network interface index translation table. This table
+ *   is terminated by an entry with a zero ifindex value.
+ * @param[in] attr
+ *   Flow rule attributes.
+ * @param[in] pattern
+ *   Pattern specification.
+ * @param[in] actions
+ *   Associated actions.
+ * @param[out] error
+ *   Perform verbose error reporting if not NULL.
+ *
+ * @return
+ *   A positive value representing the exact size of the message in bytes
+ *   regardless of the @p size parameter on success, a negative errno value
+ *   otherwise and rte_errno is set.
+ */
+int
+mlx5_nl_flow_transpose(void *buf,
+		       size_t size,
+		       const struct mlx5_nl_flow_ptoi *ptoi,
+		       const struct rte_flow_attr *attr,
+		       const struct rte_flow_item *pattern,
+		       const struct rte_flow_action *actions,
+		       struct rte_flow_error *error)
+{
+	alignas(struct nlmsghdr)
+	uint8_t buf_tmp[mnl_nlmsg_size(sizeof(struct tcmsg) + 1024)];
+	const struct rte_flow_item *item;
+	const struct rte_flow_action *action;
+	unsigned int n;
+	uint32_t act_index_cur;
+	bool in_port_id_set;
+	bool eth_type_set;
+	bool vlan_present;
+	bool vlan_eth_type_set;
+	bool ip_proto_set;
+	struct nlattr *na_flower;
+	struct nlattr *na_flower_act;
+	struct nlattr *na_vlan_id;
+	struct nlattr *na_vlan_priority;
+	const enum mlx5_nl_flow_trans *trans;
+	const enum mlx5_nl_flow_trans *back;
+
+	if (!size)
+		goto error_nobufs;
+init:
+	item = pattern;
+	action = actions;
+	n = 0;
+	act_index_cur = 0;
+	in_port_id_set = false;
+	eth_type_set = false;
+	vlan_present = false;
+	vlan_eth_type_set = false;
+	ip_proto_set = false;
+	na_flower = NULL;
+	na_flower_act = NULL;
+	na_vlan_id = NULL;
+	na_vlan_priority = NULL;
+	trans = TRANS(ATTR);
+	back = trans;
+trans:
+	switch (trans[n++]) {
+		union {
+			const struct rte_flow_item_port_id *port_id;
+			const struct rte_flow_item_eth *eth;
+			const struct rte_flow_item_vlan *vlan;
+			const struct rte_flow_item_ipv4 *ipv4;
+			const struct rte_flow_item_ipv6 *ipv6;
+			const struct rte_flow_item_tcp *tcp;
+			const struct rte_flow_item_udp *udp;
+		} spec, mask;
+		union {
+			const struct rte_flow_action_port_id *port_id;
+			const struct rte_flow_action_of_push_vlan *of_push_vlan;
+			const struct rte_flow_action_of_set_vlan_vid *
+				of_set_vlan_vid;
+			const struct rte_flow_action_of_set_vlan_pcp *
+				of_set_vlan_pcp;
+		} conf;
+		struct nlmsghdr *nlh;
+		struct tcmsg *tcm;
+		struct nlattr *act_index;
+		struct nlattr *act;
+		unsigned int i;
+
+	case INVALID:
+		if (item->type)
+			return rte_flow_error_set
+				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
+				 item, "unsupported pattern item combination");
+		else if (action->type)
+			return rte_flow_error_set
+				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
+				 action, "unsupported action combination");
+		return rte_flow_error_set
+			(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+			 "flow rule lacks some kind of fate action");
+	case BACK:
+		trans = back;
+		n = 0;
+		goto trans;
+	case ATTR:
+		/*
+		 * Supported attributes: no groups, some priorities and
+		 * ingress only. Don't care about transfer as it is the
+		 * caller's problem.
+		 */
+		if (attr->group)
+			return rte_flow_error_set
+				(error, ENOTSUP,
+				 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
+				 attr, "groups are not supported");
+		if (attr->priority > 0xfffe)
+			return rte_flow_error_set
+				(error, ENOTSUP,
+				 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
+				 attr, "lowest priority level is 0xfffe");
+		if (!attr->ingress)
+			return rte_flow_error_set
+				(error, ENOTSUP,
+				 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
+				 attr, "only ingress is supported");
+		if (attr->egress)
+			return rte_flow_error_set
+				(error, ENOTSUP,
+				 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
+				 attr, "egress is not supported");
+		if (size < mnl_nlmsg_size(sizeof(*tcm)))
+			goto error_nobufs;
+		nlh = mnl_nlmsg_put_header(buf);
+		nlh->nlmsg_type = 0;
+		nlh->nlmsg_flags = 0;
+		nlh->nlmsg_seq = 0;
+		tcm = mnl_nlmsg_put_extra_header(nlh, sizeof(*tcm));
+		tcm->tcm_family = AF_UNSPEC;
+		tcm->tcm_ifindex = ptoi[0].ifindex;
+		/*
+		 * Let kernel pick a handle by default. A predictable handle
+		 * can be set by the caller on the resulting buffer through
+		 * mlx5_nl_flow_brand().
+		 */
+		tcm->tcm_handle = 0;
+		tcm->tcm_parent = TC_H_MAKE(TC_H_INGRESS, TC_H_MIN_INGRESS);
+		/*
+		 * Priority cannot be zero to prevent the kernel from
+		 * picking one automatically.
+		 */
+		tcm->tcm_info = TC_H_MAKE((attr->priority + 1) << 16,
+					  RTE_BE16(ETH_P_ALL));
+		break;
+	case PATTERN:
+		if (!mnl_attr_put_strz_check(buf, size, TCA_KIND, "flower"))
+			goto error_nobufs;
+		na_flower = mnl_attr_nest_start_check(buf, size, TCA_OPTIONS);
+		if (!na_flower)
+			goto error_nobufs;
+		if (!mnl_attr_put_u32_check(buf, size, TCA_FLOWER_FLAGS,
+					    TCA_CLS_FLAGS_SKIP_SW))
+			goto error_nobufs;
+		break;
+	case ITEM_VOID:
+		if (item->type != RTE_FLOW_ITEM_TYPE_VOID)
+			goto trans;
+		++item;
+		break;
+	case ITEM_PORT_ID:
+		if (item->type != RTE_FLOW_ITEM_TYPE_PORT_ID)
+			goto trans;
+		mask.port_id = mlx5_nl_flow_item_mask
+			(item, &rte_flow_item_port_id_mask,
+			 &mlx5_nl_flow_mask_supported.port_id,
+			 &mlx5_nl_flow_mask_empty.port_id,
+			 sizeof(mlx5_nl_flow_mask_supported.port_id), error);
+		if (!mask.port_id)
+			return -rte_errno;
+		if (mask.port_id == &mlx5_nl_flow_mask_empty.port_id) {
+			in_port_id_set = 1;
+			++item;
+			break;
+		}
+		spec.port_id = item->spec;
+		if (mask.port_id->id && mask.port_id->id != 0xffffffff)
+			return rte_flow_error_set
+				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM_MASK,
+				 mask.port_id,
+				 "no support for partial mask on"
+				 " \"id\" field");
+		if (!mask.port_id->id)
+			i = 0;
+		else
+			for (i = 0; ptoi[i].ifindex; ++i)
+				if (ptoi[i].port_id == spec.port_id->id)
+					break;
+		if (!ptoi[i].ifindex)
+			return rte_flow_error_set
+				(error, ENODEV, RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
+				 spec.port_id,
+				 "missing data to convert port ID to ifindex");
+		tcm = mnl_nlmsg_get_payload(buf);
+		if (in_port_id_set &&
+		    ptoi[i].ifindex != (unsigned int)tcm->tcm_ifindex)
+			return rte_flow_error_set
+				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
+				 spec.port_id,
+				 "cannot match traffic for several port IDs"
+				 " through a single flow rule");
+		tcm->tcm_ifindex = ptoi[i].ifindex;
+		in_port_id_set = 1;
+		++item;
+		break;
+	case ITEM_ETH:
+		if (item->type != RTE_FLOW_ITEM_TYPE_ETH)
+			goto trans;
+		mask.eth = mlx5_nl_flow_item_mask
+			(item, &rte_flow_item_eth_mask,
+			 &mlx5_nl_flow_mask_supported.eth,
+			 &mlx5_nl_flow_mask_empty.eth,
+			 sizeof(mlx5_nl_flow_mask_supported.eth), error);
+		if (!mask.eth)
+			return -rte_errno;
+		if (mask.eth == &mlx5_nl_flow_mask_empty.eth) {
+			++item;
+			break;
+		}
+		spec.eth = item->spec;
+		if (mask.eth->type && mask.eth->type != RTE_BE16(0xffff))
+			return rte_flow_error_set
+				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM_MASK,
+				 mask.eth,
+				 "no support for partial mask on"
+				 " \"type\" field");
+		if (mask.eth->type) {
+			if (!mnl_attr_put_u16_check(buf, size,
+						    TCA_FLOWER_KEY_ETH_TYPE,
+						    spec.eth->type))
+				goto error_nobufs;
+			eth_type_set = 1;
+		}
+		if ((!is_zero_ether_addr(&mask.eth->dst) &&
+		     (!mnl_attr_put_check(buf, size,
+					  TCA_FLOWER_KEY_ETH_DST,
+					  ETHER_ADDR_LEN,
+					  spec.eth->dst.addr_bytes) ||
+		      !mnl_attr_put_check(buf, size,
+					  TCA_FLOWER_KEY_ETH_DST_MASK,
+					  ETHER_ADDR_LEN,
+					  mask.eth->dst.addr_bytes))) ||
+		    (!is_zero_ether_addr(&mask.eth->src) &&
+		     (!mnl_attr_put_check(buf, size,
+					  TCA_FLOWER_KEY_ETH_SRC,
+					  ETHER_ADDR_LEN,
+					  spec.eth->src.addr_bytes) ||
+		      !mnl_attr_put_check(buf, size,
+					  TCA_FLOWER_KEY_ETH_SRC_MASK,
+					  ETHER_ADDR_LEN,
+					  mask.eth->src.addr_bytes))))
+			goto error_nobufs;
+		++item;
+		break;
+	case ITEM_VLAN:
+		if (item->type != RTE_FLOW_ITEM_TYPE_VLAN)
+			goto trans;
+		mask.vlan = mlx5_nl_flow_item_mask
+			(item, &rte_flow_item_vlan_mask,
+			 &mlx5_nl_flow_mask_supported.vlan,
+			 &mlx5_nl_flow_mask_empty.vlan,
+			 sizeof(mlx5_nl_flow_mask_supported.vlan), error);
+		if (!mask.vlan)
+			return -rte_errno;
+		if (!eth_type_set &&
+		    !mnl_attr_put_u16_check(buf, size,
+					    TCA_FLOWER_KEY_ETH_TYPE,
+					    RTE_BE16(ETH_P_8021Q)))
+			goto error_nobufs;
+		eth_type_set = 1;
+		vlan_present = 1;
+		if (mask.vlan == &mlx5_nl_flow_mask_empty.vlan) {
+			++item;
+			break;
+		}
+		spec.vlan = item->spec;
+		if ((mask.vlan->tci & RTE_BE16(0xe000) &&
+		     (mask.vlan->tci & RTE_BE16(0xe000)) != RTE_BE16(0xe000)) ||
+		    (mask.vlan->tci & RTE_BE16(0x0fff) &&
+		     (mask.vlan->tci & RTE_BE16(0x0fff)) != RTE_BE16(0x0fff)) ||
+		    (mask.vlan->inner_type &&
+		     mask.vlan->inner_type != RTE_BE16(0xffff)))
+			return rte_flow_error_set
+				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM_MASK,
+				 mask.vlan,
+				 "no support for partial masks on"
+				 " \"tci\" (PCP and VID parts) and"
+				 " \"inner_type\" fields");
+		if (mask.vlan->inner_type) {
+			if (!mnl_attr_put_u16_check
+			    (buf, size, TCA_FLOWER_KEY_VLAN_ETH_TYPE,
+			     spec.vlan->inner_type))
+				goto error_nobufs;
+			vlan_eth_type_set = 1;
+		}
+		if ((mask.vlan->tci & RTE_BE16(0xe000) &&
+		     !mnl_attr_put_u8_check
+		     (buf, size, TCA_FLOWER_KEY_VLAN_PRIO,
+		      (rte_be_to_cpu_16(spec.vlan->tci) >> 13) & 0x7)) ||
+		    (mask.vlan->tci & RTE_BE16(0x0fff) &&
+		     !mnl_attr_put_u16_check
+		     (buf, size, TCA_FLOWER_KEY_VLAN_ID,
+		      rte_be_to_cpu_16(spec.vlan->tci & RTE_BE16(0x0fff)))))
+			goto error_nobufs;
+		++item;
+		break;
+	case ITEM_IPV4:
+		if (item->type != RTE_FLOW_ITEM_TYPE_IPV4)
+			goto trans;
+		mask.ipv4 = mlx5_nl_flow_item_mask
+			(item, &rte_flow_item_ipv4_mask,
+			 &mlx5_nl_flow_mask_supported.ipv4,
+			 &mlx5_nl_flow_mask_empty.ipv4,
+			 sizeof(mlx5_nl_flow_mask_supported.ipv4), error);
+		if (!mask.ipv4)
+			return -rte_errno;
+		if ((!eth_type_set || !vlan_eth_type_set) &&
+		    !mnl_attr_put_u16_check(buf, size,
+					    vlan_present ?
+					    TCA_FLOWER_KEY_VLAN_ETH_TYPE :
+					    TCA_FLOWER_KEY_ETH_TYPE,
+					    RTE_BE16(ETH_P_IP)))
+			goto error_nobufs;
+		eth_type_set = 1;
+		vlan_eth_type_set = 1;
+		if (mask.ipv4 == &mlx5_nl_flow_mask_empty.ipv4) {
+			++item;
+			break;
+		}
+		spec.ipv4 = item->spec;
+		if (mask.ipv4->hdr.next_proto_id &&
+		    mask.ipv4->hdr.next_proto_id != 0xff)
+			return rte_flow_error_set
+				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM_MASK,
+				 mask.ipv4,
+				 "no support for partial mask on"
+				 " \"hdr.next_proto_id\" field");
+		if (mask.ipv4->hdr.next_proto_id) {
+			if (!mnl_attr_put_u8_check
+			    (buf, size, TCA_FLOWER_KEY_IP_PROTO,
+			     spec.ipv4->hdr.next_proto_id))
+				goto error_nobufs;
+			ip_proto_set = 1;
+		}
+		if ((mask.ipv4->hdr.src_addr &&
+		     (!mnl_attr_put_u32_check(buf, size,
+					      TCA_FLOWER_KEY_IPV4_SRC,
+					      spec.ipv4->hdr.src_addr) ||
+		      !mnl_attr_put_u32_check(buf, size,
+					      TCA_FLOWER_KEY_IPV4_SRC_MASK,
+					      mask.ipv4->hdr.src_addr))) ||
+		    (mask.ipv4->hdr.dst_addr &&
+		     (!mnl_attr_put_u32_check(buf, size,
+					      TCA_FLOWER_KEY_IPV4_DST,
+					      spec.ipv4->hdr.dst_addr) ||
+		      !mnl_attr_put_u32_check(buf, size,
+					      TCA_FLOWER_KEY_IPV4_DST_MASK,
+					      mask.ipv4->hdr.dst_addr))))
+			goto error_nobufs;
+		++item;
+		break;
+	case ITEM_IPV6:
+		if (item->type != RTE_FLOW_ITEM_TYPE_IPV6)
+			goto trans;
+		mask.ipv6 = mlx5_nl_flow_item_mask
+			(item, &rte_flow_item_ipv6_mask,
+			 &mlx5_nl_flow_mask_supported.ipv6,
+			 &mlx5_nl_flow_mask_empty.ipv6,
+			 sizeof(mlx5_nl_flow_mask_supported.ipv6), error);
+		if (!mask.ipv6)
+			return -rte_errno;
+		if ((!eth_type_set || !vlan_eth_type_set) &&
+		    !mnl_attr_put_u16_check(buf, size,
+					    vlan_present ?
+					    TCA_FLOWER_KEY_VLAN_ETH_TYPE :
+					    TCA_FLOWER_KEY_ETH_TYPE,
+					    RTE_BE16(ETH_P_IPV6)))
+			goto error_nobufs;
+		eth_type_set = 1;
+		vlan_eth_type_set = 1;
+		if (mask.ipv6 == &mlx5_nl_flow_mask_empty.ipv6) {
+			++item;
+			break;
+		}
+		spec.ipv6 = item->spec;
+		if (mask.ipv6->hdr.proto && mask.ipv6->hdr.proto != 0xff)
+			return rte_flow_error_set
+				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM_MASK,
+				 mask.ipv6,
+				 "no support for partial mask on"
+				 " \"hdr.proto\" field");
+		if (mask.ipv6->hdr.proto) {
+			if (!mnl_attr_put_u8_check
+			    (buf, size, TCA_FLOWER_KEY_IP_PROTO,
+			     spec.ipv6->hdr.proto))
+				goto error_nobufs;
+			ip_proto_set = 1;
+		}
+		if ((!IN6_IS_ADDR_UNSPECIFIED(mask.ipv6->hdr.src_addr) &&
+		     (!mnl_attr_put_check(buf, size,
+					  TCA_FLOWER_KEY_IPV6_SRC,
+					  sizeof(spec.ipv6->hdr.src_addr),
+					  spec.ipv6->hdr.src_addr) ||
+		      !mnl_attr_put_check(buf, size,
+					  TCA_FLOWER_KEY_IPV6_SRC_MASK,
+					  sizeof(mask.ipv6->hdr.src_addr),
+					  mask.ipv6->hdr.src_addr))) ||
+		    (!IN6_IS_ADDR_UNSPECIFIED(mask.ipv6->hdr.dst_addr) &&
+		     (!mnl_attr_put_check(buf, size,
+					  TCA_FLOWER_KEY_IPV6_DST,
+					  sizeof(spec.ipv6->hdr.dst_addr),
+					  spec.ipv6->hdr.dst_addr) ||
+		      !mnl_attr_put_check(buf, size,
+					  TCA_FLOWER_KEY_IPV6_DST_MASK,
+					  sizeof(mask.ipv6->hdr.dst_addr),
+					  mask.ipv6->hdr.dst_addr))))
+			goto error_nobufs;
+		++item;
+		break;
+	case ITEM_TCP:
+		if (item->type != RTE_FLOW_ITEM_TYPE_TCP)
+			goto trans;
+		mask.tcp = mlx5_nl_flow_item_mask
+			(item, &rte_flow_item_tcp_mask,
+			 &mlx5_nl_flow_mask_supported.tcp,
+			 &mlx5_nl_flow_mask_empty.tcp,
+			 sizeof(mlx5_nl_flow_mask_supported.tcp), error);
+		if (!mask.tcp)
+			return -rte_errno;
+		if (!ip_proto_set &&
+		    !mnl_attr_put_u8_check(buf, size,
+					   TCA_FLOWER_KEY_IP_PROTO,
+					   IPPROTO_TCP))
+			goto error_nobufs;
+		if (mask.tcp == &mlx5_nl_flow_mask_empty.tcp) {
+			++item;
+			break;
+		}
+		spec.tcp = item->spec;
+		if ((mask.tcp->hdr.src_port &&
+		     mask.tcp->hdr.src_port != RTE_BE16(0xffff)) ||
+		    (mask.tcp->hdr.dst_port &&
+		     mask.tcp->hdr.dst_port != RTE_BE16(0xffff)))
+			return rte_flow_error_set
+				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM_MASK,
+				 mask.tcp,
+				 "no support for partial masks on"
+				 " \"hdr.src_port\" and \"hdr.dst_port\""
+				 " fields");
+		if ((mask.tcp->hdr.src_port &&
+		     (!mnl_attr_put_u16_check(buf, size,
+					      TCA_FLOWER_KEY_TCP_SRC,
+					      spec.tcp->hdr.src_port) ||
+		      !mnl_attr_put_u16_check(buf, size,
+					      TCA_FLOWER_KEY_TCP_SRC_MASK,
+					      mask.tcp->hdr.src_port))) ||
+		    (mask.tcp->hdr.dst_port &&
+		     (!mnl_attr_put_u16_check(buf, size,
+					      TCA_FLOWER_KEY_TCP_DST,
+					      spec.tcp->hdr.dst_port) ||
+		      !mnl_attr_put_u16_check(buf, size,
+					      TCA_FLOWER_KEY_TCP_DST_MASK,
+					      mask.tcp->hdr.dst_port))))
+			goto error_nobufs;
+		++item;
+		break;
+	case ITEM_UDP:
+		if (item->type != RTE_FLOW_ITEM_TYPE_UDP)
+			goto trans;
+		mask.udp = mlx5_nl_flow_item_mask
+			(item, &rte_flow_item_udp_mask,
+			 &mlx5_nl_flow_mask_supported.udp,
+			 &mlx5_nl_flow_mask_empty.udp,
+			 sizeof(mlx5_nl_flow_mask_supported.udp), error);
+		if (!mask.udp)
+			return -rte_errno;
+		if (!ip_proto_set &&
+		    !mnl_attr_put_u8_check(buf, size,
+					   TCA_FLOWER_KEY_IP_PROTO,
+					   IPPROTO_UDP))
+			goto error_nobufs;
+		if (mask.udp == &mlx5_nl_flow_mask_empty.udp) {
+			++item;
+			break;
+		}
+		spec.udp = item->spec;
+		if ((mask.udp->hdr.src_port &&
+		     mask.udp->hdr.src_port != RTE_BE16(0xffff)) ||
+		    (mask.udp->hdr.dst_port &&
+		     mask.udp->hdr.dst_port != RTE_BE16(0xffff)))
+			return rte_flow_error_set
+				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM_MASK,
+				 mask.udp,
+				 "no support for partial masks on"
+				 " \"hdr.src_port\" and \"hdr.dst_port\""
+				 " fields");
+		if ((mask.udp->hdr.src_port &&
+		     (!mnl_attr_put_u16_check(buf, size,
+					      TCA_FLOWER_KEY_UDP_SRC,
+					      spec.udp->hdr.src_port) ||
+		      !mnl_attr_put_u16_check(buf, size,
+					      TCA_FLOWER_KEY_UDP_SRC_MASK,
+					      mask.udp->hdr.src_port))) ||
+		    (mask.udp->hdr.dst_port &&
+		     (!mnl_attr_put_u16_check(buf, size,
+					      TCA_FLOWER_KEY_UDP_DST,
+					      spec.udp->hdr.dst_port) ||
+		      !mnl_attr_put_u16_check(buf, size,
+					      TCA_FLOWER_KEY_UDP_DST_MASK,
+					      mask.udp->hdr.dst_port))))
+			goto error_nobufs;
+		++item;
+		break;
+	case ACTIONS:
+		if (item->type != RTE_FLOW_ITEM_TYPE_END)
+			goto trans;
+		assert(na_flower);
+		assert(!na_flower_act);
+		na_flower_act =
+			mnl_attr_nest_start_check(buf, size, TCA_FLOWER_ACT);
+		if (!na_flower_act)
+			goto error_nobufs;
+		act_index_cur = 1;
+		break;
+	case ACTION_VOID:
+		if (action->type != RTE_FLOW_ACTION_TYPE_VOID)
+			goto trans;
+		++action;
+		break;
+	case ACTION_PORT_ID:
+		if (action->type != RTE_FLOW_ACTION_TYPE_PORT_ID)
+			goto trans;
+		conf.port_id = action->conf;
+		if (conf.port_id->original)
+			i = 0;
+		else
+			for (i = 0; ptoi[i].ifindex; ++i)
+				if (ptoi[i].port_id == conf.port_id->id)
+					break;
+		if (!ptoi[i].ifindex)
+			return rte_flow_error_set
+				(error, ENODEV, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+				 conf.port_id,
+				 "missing data to convert port ID to ifindex");
+		act_index =
+			mnl_attr_nest_start_check(buf, size, act_index_cur++);
+		if (!act_index ||
+		    !mnl_attr_put_strz_check(buf, size, TCA_ACT_KIND, "mirred"))
+			goto error_nobufs;
+		act = mnl_attr_nest_start_check(buf, size, TCA_ACT_OPTIONS);
+		if (!act)
+			goto error_nobufs;
+		if (!mnl_attr_put_check(buf, size, TCA_MIRRED_PARMS,
+					sizeof(struct tc_mirred),
+					&(struct tc_mirred){
+						.action = TC_ACT_STOLEN,
+						.eaction = TCA_EGRESS_REDIR,
+						.ifindex = ptoi[i].ifindex,
+					}))
+			goto error_nobufs;
+		mnl_attr_nest_end(buf, act);
+		mnl_attr_nest_end(buf, act_index);
+		++action;
+		break;
+	case ACTION_DROP:
+		if (action->type != RTE_FLOW_ACTION_TYPE_DROP)
+			goto trans;
+		act_index =
+			mnl_attr_nest_start_check(buf, size, act_index_cur++);
+		if (!act_index ||
+		    !mnl_attr_put_strz_check(buf, size, TCA_ACT_KIND, "gact"))
+			goto error_nobufs;
+		act = mnl_attr_nest_start_check(buf, size, TCA_ACT_OPTIONS);
+		if (!act)
+			goto error_nobufs;
+		if (!mnl_attr_put_check(buf, size, TCA_GACT_PARMS,
+					sizeof(struct tc_gact),
+					&(struct tc_gact){
+						.action = TC_ACT_SHOT,
+					}))
+			goto error_nobufs;
+		mnl_attr_nest_end(buf, act);
+		mnl_attr_nest_end(buf, act_index);
+		++action;
+		break;
+	case ACTION_OF_POP_VLAN:
+		if (action->type != RTE_FLOW_ACTION_TYPE_OF_POP_VLAN)
+			goto trans;
+		conf.of_push_vlan = NULL;
+		i = TCA_VLAN_ACT_POP;
+		goto action_of_vlan;
+	case ACTION_OF_PUSH_VLAN:
+		if (action->type != RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN)
+			goto trans;
+		conf.of_push_vlan = action->conf;
+		i = TCA_VLAN_ACT_PUSH;
+		goto action_of_vlan;
+	case ACTION_OF_SET_VLAN_VID:
+		if (action->type != RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)
+			goto trans;
+		conf.of_set_vlan_vid = action->conf;
+		if (na_vlan_id)
+			goto override_na_vlan_id;
+		i = TCA_VLAN_ACT_MODIFY;
+		goto action_of_vlan;
+	case ACTION_OF_SET_VLAN_PCP:
+		if (action->type != RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP)
+			goto trans;
+		conf.of_set_vlan_pcp = action->conf;
+		if (na_vlan_priority)
+			goto override_na_vlan_priority;
+		i = TCA_VLAN_ACT_MODIFY;
+		goto action_of_vlan;
+action_of_vlan:
+		act_index =
+			mnl_attr_nest_start_check(buf, size, act_index_cur++);
+		if (!act_index ||
+		    !mnl_attr_put_strz_check(buf, size, TCA_ACT_KIND, "vlan"))
+			goto error_nobufs;
+		act = mnl_attr_nest_start_check(buf, size, TCA_ACT_OPTIONS);
+		if (!act)
+			goto error_nobufs;
+		if (!mnl_attr_put_check(buf, size, TCA_VLAN_PARMS,
+					sizeof(struct tc_vlan),
+					&(struct tc_vlan){
+						.action = TC_ACT_PIPE,
+						.v_action = i,
+					}))
+			goto error_nobufs;
+		if (i == TCA_VLAN_ACT_POP) {
+			mnl_attr_nest_end(buf, act);
+			mnl_attr_nest_end(buf, act_index);
+			++action;
+			break;
+		}
+		if (i == TCA_VLAN_ACT_PUSH &&
+		    !mnl_attr_put_u16_check(buf, size,
+					    TCA_VLAN_PUSH_VLAN_PROTOCOL,
+					    conf.of_push_vlan->ethertype))
+			goto error_nobufs;
+		na_vlan_id = mnl_nlmsg_get_payload_tail(buf);
+		if (!mnl_attr_put_u16_check(buf, size, TCA_VLAN_PAD, 0))
+			goto error_nobufs;
+		na_vlan_priority = mnl_nlmsg_get_payload_tail(buf);
+		if (!mnl_attr_put_u8_check(buf, size, TCA_VLAN_PAD, 0))
+			goto error_nobufs;
+		mnl_attr_nest_end(buf, act);
+		mnl_attr_nest_end(buf, act_index);
+		if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
+override_na_vlan_id:
+			na_vlan_id->nla_type = TCA_VLAN_PUSH_VLAN_ID;
+			*(uint16_t *)mnl_attr_get_payload(na_vlan_id) =
+				rte_be_to_cpu_16
+				(conf.of_set_vlan_vid->vlan_vid);
+		} else if (action->type ==
+			   RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
+override_na_vlan_priority:
+			na_vlan_priority->nla_type =
+				TCA_VLAN_PUSH_VLAN_PRIORITY;
+			*(uint8_t *)mnl_attr_get_payload(na_vlan_priority) =
+				conf.of_set_vlan_pcp->vlan_pcp;
+		}
+		++action;
+		break;
+	case END:
+		if (item->type != RTE_FLOW_ITEM_TYPE_END ||
+		    action->type != RTE_FLOW_ACTION_TYPE_END)
+			goto trans;
+		if (na_flower_act)
+			mnl_attr_nest_end(buf, na_flower_act);
+		if (na_flower)
+			mnl_attr_nest_end(buf, na_flower);
+		nlh = buf;
+		return nlh->nlmsg_len;
+	}
+	back = trans;
+	trans = mlx5_nl_flow_trans[trans[n - 1]];
+	n = 0;
+	goto trans;
+error_nobufs:
+	if (buf != buf_tmp) {
+		buf = buf_tmp;
+		size = sizeof(buf_tmp);
+		goto init;
+	}
+	return rte_flow_error_set
+		(error, ENOBUFS, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+		 "generated TC message is too large");
+}
+
+/**
+ * Brand rtnetlink buffer with unique handle.
+ *
+ * This handle should be unique for a given network interface to avoid
+ * collisions.
+ *
+ * @param buf
+ *   Flow rule buffer previously initialized by mlx5_nl_flow_transpose().
+ * @param handle
+ *   Unique 32-bit handle to use.
+ */
+void
+mlx5_nl_flow_brand(void *buf, uint32_t handle)
+{
+	struct tcmsg *tcm = mnl_nlmsg_get_payload(buf);
+
+	tcm->tcm_handle = handle;
+}
+
+/**
+ * Send Netlink message with acknowledgment.
+ *
+ * @param nl
+ *   Libmnl socket to use.
+ * @param nlh
+ *   Message to send. This function always raises the NLM_F_ACK flag before
+ *   sending.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_nl_flow_nl_ack(struct mnl_socket *nl, struct nlmsghdr *nlh)
+{
+	alignas(struct nlmsghdr)
+	uint8_t ans[mnl_nlmsg_size(sizeof(struct nlmsgerr)) +
+		    nlh->nlmsg_len - sizeof(*nlh)];
+	uint32_t seq = random();
+	int ret;
+
+	nlh->nlmsg_flags |= NLM_F_ACK;
+	nlh->nlmsg_seq = seq;
+	ret = mnl_socket_sendto(nl, nlh, nlh->nlmsg_len);
+	if (ret != -1)
+		ret = mnl_socket_recvfrom(nl, ans, sizeof(ans));
+	if (ret != -1)
+		ret = mnl_cb_run
+			(ans, ret, seq, mnl_socket_get_portid(nl), NULL, NULL);
+	if (!ret)
+		return 0;
+	rte_errno = errno;
+	return -rte_errno;
+}
+
+/**
+ * Create a Netlink flow rule.
+ *
+ * @param nl
+ *   Libmnl socket to use.
+ * @param buf
+ *   Flow rule buffer previously initialized by mlx5_nl_flow_transpose().
+ * @param[out] error
+ *   Perform verbose error reporting if not NULL.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_nl_flow_create(struct mnl_socket *nl, void *buf,
+		    struct rte_flow_error *error)
+{
+	struct nlmsghdr *nlh = buf;
+
+	nlh->nlmsg_type = RTM_NEWTFILTER;
+	nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_CREATE | NLM_F_EXCL;
+	if (!mlx5_nl_flow_nl_ack(nl, nlh))
+		return 0;
+	return rte_flow_error_set
+		(error, rte_errno, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+		 "netlink: failed to create TC flow rule");
+}
+
+/**
+ * Destroy a Netlink flow rule.
+ *
+ * @param nl
+ *   Libmnl socket to use.
+ * @param buf
+ *   Flow rule buffer previously initialized by mlx5_nl_flow_transpose().
+ * @param[out] error
+ *   Perform verbose error reporting if not NULL.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_nl_flow_destroy(struct mnl_socket *nl, void *buf,
+		     struct rte_flow_error *error)
+{
+	struct nlmsghdr *nlh = buf;
+
+	nlh->nlmsg_type = RTM_DELTFILTER;
+	nlh->nlmsg_flags = NLM_F_REQUEST;
+	if (!mlx5_nl_flow_nl_ack(nl, nlh))
+		return 0;
+	return rte_flow_error_set
+		(error, errno, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+		 "netlink: failed to destroy TC flow rule");
+}
+
+/**
+ * Initialize ingress qdisc of a given network interface.
+ *
+ * @param nl
+ *   Libmnl socket of the @p NETLINK_ROUTE kind.
+ * @param ifindex
+ *   Index of network interface to initialize.
+ * @param[out] error
+ *   Perform verbose error reporting if not NULL.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_nl_flow_init(struct mnl_socket *nl, unsigned int ifindex,
+		  struct rte_flow_error *error)
+{
+	struct nlmsghdr *nlh;
+	struct tcmsg *tcm;
+	alignas(struct nlmsghdr)
+	uint8_t buf[mnl_nlmsg_size(sizeof(*tcm) + 128)];
+
+	/* Destroy existing ingress qdisc and everything attached to it. */
+	nlh = mnl_nlmsg_put_header(buf);
+	nlh->nlmsg_type = RTM_DELQDISC;
+	nlh->nlmsg_flags = NLM_F_REQUEST;
+	tcm = mnl_nlmsg_put_extra_header(nlh, sizeof(*tcm));
+	tcm->tcm_family = AF_UNSPEC;
+	tcm->tcm_ifindex = ifindex;
+	tcm->tcm_handle = TC_H_MAKE(TC_H_INGRESS, 0);
+	tcm->tcm_parent = TC_H_INGRESS;
+	/* Ignore errors when qdisc is already absent. */
+	if (mlx5_nl_flow_nl_ack(nl, nlh) &&
+	    rte_errno != EINVAL && rte_errno != ENOENT)
+		return rte_flow_error_set
+			(error, rte_errno, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+			 NULL, "netlink: failed to remove ingress qdisc");
+	/* Create fresh ingress qdisc. */
+	nlh = mnl_nlmsg_put_header(buf);
+	nlh->nlmsg_type = RTM_NEWQDISC;
+	nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_CREATE | NLM_F_EXCL;
+	tcm = mnl_nlmsg_put_extra_header(nlh, sizeof(*tcm));
+	tcm->tcm_family = AF_UNSPEC;
+	tcm->tcm_ifindex = ifindex;
+	tcm->tcm_handle = TC_H_MAKE(TC_H_INGRESS, 0);
+	tcm->tcm_parent = TC_H_INGRESS;
+	mnl_attr_put_strz_check(nlh, sizeof(buf), TCA_KIND, "ingress");
+	if (mlx5_nl_flow_nl_ack(nl, nlh))
+		return rte_flow_error_set
+			(error, rte_errno, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+			 NULL, "netlink: failed to create ingress qdisc");
+	return 0;
+}
+
+/**
+ * Create and configure a libmnl socket for Netlink flow rules.
+ *
+ * @return
+ *   A valid libmnl socket object pointer on success, NULL otherwise and
+ *   rte_errno is set.
+ */
+struct mnl_socket *
+mlx5_nl_flow_socket_create(void)
+{
+	struct mnl_socket *nl = mnl_socket_open(NETLINK_ROUTE);
+
+	if (nl) {
+		mnl_socket_setsockopt(nl, NETLINK_CAP_ACK, &(int){ 1 },
+				      sizeof(int));
+		if (!mnl_socket_bind(nl, 0, MNL_SOCKET_AUTOPID))
+			return nl;
+	}
+	rte_errno = errno;
+	if (nl)
+		mnl_socket_close(nl);
+	return NULL;
+}
+
+/**
+ * Destroy a libmnl socket.
+ */
+void
+mlx5_nl_flow_socket_destroy(struct mnl_socket *nl)
+{
+	mnl_socket_close(nl);
+}
diff --git a/src/spdk/dpdk/drivers/net/mlx5/mlx5_prm.h b/src/spdk/dpdk/drivers/net/mlx5/mlx5_prm.h
new file mode 100644
index 00000000..0870d32f
--- /dev/null
+++ b/src/spdk/dpdk/drivers/net/mlx5/mlx5_prm.h
@@ -0,0 +1,364 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2016 6WIND S.A.
+ * Copyright 2016 Mellanox Technologies, Ltd
+ */
+
+#ifndef RTE_PMD_MLX5_PRM_H_
+#define RTE_PMD_MLX5_PRM_H_
+
+#include <assert.h>
+
+/* Verbs header. */
+/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <infiniband/mlx5dv.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+#include <rte_vect.h>
+#include "mlx5_autoconf.h"
+
+/* RSS hash key size. */
+#define MLX5_RSS_HASH_KEY_LEN 40
+
+/* Get CQE owner bit. */
+#define MLX5_CQE_OWNER(op_own) ((op_own) & MLX5_CQE_OWNER_MASK)
+
+/* Get CQE format. */
+#define MLX5_CQE_FORMAT(op_own) (((op_own) & MLX5E_CQE_FORMAT_MASK) >> 2)
+
+/* Get CQE opcode. */
+#define MLX5_CQE_OPCODE(op_own) (((op_own) & 0xf0) >> 4)
+
+/* Get CQE solicited event. */
+#define MLX5_CQE_SE(op_own) (((op_own) >> 1) & 1)
+
+/* Invalidate a CQE. */
+#define MLX5_CQE_INVALIDATE (MLX5_CQE_INVALID << 4)
+
+/* Maximum number of packets a multi-packet WQE can handle. */
+#define MLX5_MPW_DSEG_MAX 5
+
+/* WQE DWORD size */
+#define MLX5_WQE_DWORD_SIZE 16
+
+/* WQE size */
+#define MLX5_WQE_SIZE (4 * MLX5_WQE_DWORD_SIZE)
+
+/* Max size of a WQE session. */
+#define MLX5_WQE_SIZE_MAX 960U
+
+/* Compute the number of DS. */
+#define MLX5_WQE_DS(n) \
+	(((n) + MLX5_WQE_DWORD_SIZE - 1) / MLX5_WQE_DWORD_SIZE)
+
+/* Room for inline data in multi-packet WQE. */
+#define MLX5_MWQE64_INL_DATA 28
+
+/* Default minimum number of Tx queues for inlining packets. */
+#define MLX5_EMPW_MIN_TXQS 8
+
+/* Default max packet length to be inlined. */
+#define MLX5_EMPW_MAX_INLINE_LEN (4U * MLX5_WQE_SIZE)
+
+
+#define MLX5_OPC_MOD_ENHANCED_MPSW 0
+#define MLX5_OPCODE_ENHANCED_MPSW 0x29
+
+/* CQE value to inform that VLAN is stripped. */
+#define MLX5_CQE_VLAN_STRIPPED (1u << 0)
+
+/* IPv4 options. */
+#define MLX5_CQE_RX_IP_EXT_OPTS_PACKET (1u << 1)
+
+/* IPv6 packet. */
+#define MLX5_CQE_RX_IPV6_PACKET (1u << 2)
+
+/* IPv4 packet. */
+#define MLX5_CQE_RX_IPV4_PACKET (1u << 3)
+
+/* TCP packet. */
+#define MLX5_CQE_RX_TCP_PACKET (1u << 4)
+
+/* UDP packet. */
+#define MLX5_CQE_RX_UDP_PACKET (1u << 5)
+
+/* IP is fragmented. */
+#define MLX5_CQE_RX_IP_FRAG_PACKET (1u << 7)
+
+/* L2 header is valid. */
+#define MLX5_CQE_RX_L2_HDR_VALID (1u << 8)
+
+/* L3 header is valid. */
+#define MLX5_CQE_RX_L3_HDR_VALID (1u << 9)
+
+/* L4 header is valid. */
+#define MLX5_CQE_RX_L4_HDR_VALID (1u << 10)
+
+/* Outer packet, 0 IPv4, 1 IPv6. */
+#define MLX5_CQE_RX_OUTER_PACKET (1u << 1)
+
+/* Tunnel packet bit in the CQE. */
+#define MLX5_CQE_RX_TUNNEL_PACKET (1u << 0)
+
+/* Inner L3 checksum offload (Tunneled packets only). */
+#define MLX5_ETH_WQE_L3_INNER_CSUM (1u << 4)
+
+/* Inner L4 checksum offload (Tunneled packets only). */
+#define MLX5_ETH_WQE_L4_INNER_CSUM (1u << 5)
+
+/* Outer L4 type is TCP. */
+#define MLX5_ETH_WQE_L4_OUTER_TCP  (0u << 5)
+
+/* Outer L4 type is UDP. */
+#define MLX5_ETH_WQE_L4_OUTER_UDP  (1u << 5)
+
+/* Outer L3 type is IPV4. */
+#define MLX5_ETH_WQE_L3_OUTER_IPV4 (0u << 4)
+
+/* Outer L3 type is IPV6. */
+#define MLX5_ETH_WQE_L3_OUTER_IPV6 (1u << 4)
+
+/* Inner L4 type is TCP. */
+#define MLX5_ETH_WQE_L4_INNER_TCP (0u << 1)
+
+/* Inner L4 type is UDP. */
+#define MLX5_ETH_WQE_L4_INNER_UDP (1u << 1)
+
+/* Inner L3 type is IPV4. */
+#define MLX5_ETH_WQE_L3_INNER_IPV4 (0u << 0)
+
+/* Inner L3 type is IPV6. */
+#define MLX5_ETH_WQE_L3_INNER_IPV6 (1u << 0)
+
+/* Is flow mark valid. */
+#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
+#define MLX5_FLOW_MARK_IS_VALID(val) ((val) & 0xffffff00)
+#else
+#define MLX5_FLOW_MARK_IS_VALID(val) ((val) & 0xffffff)
+#endif
+
+/* INVALID is used by packets matching no flow rules. */
+#define MLX5_FLOW_MARK_INVALID 0
+
+/* Maximum allowed value to mark a packet. */
+#define MLX5_FLOW_MARK_MAX 0xfffff0
+
+/* Default mark value used when none is provided. */
+#define MLX5_FLOW_MARK_DEFAULT 0xffffff
+
+/* Maximum number of DS in WQE. */
+#define MLX5_DSEG_MAX 63
+
+/* Subset of struct mlx5_wqe_eth_seg. */
+struct mlx5_wqe_eth_seg_small {
+	uint32_t rsvd0;
+	uint8_t	cs_flags;
+	uint8_t	rsvd1;
+	uint16_t mss;
+	uint32_t rsvd2;
+	uint16_t inline_hdr_sz;
+	uint8_t inline_hdr[2];
+} __rte_aligned(MLX5_WQE_DWORD_SIZE);
+
+struct mlx5_wqe_inl_small {
+	uint32_t byte_cnt;
+	uint8_t raw;
+} __rte_aligned(MLX5_WQE_DWORD_SIZE);
+
+struct mlx5_wqe_ctrl {
+	uint32_t ctrl0;
+	uint32_t ctrl1;
+	uint32_t ctrl2;
+	uint32_t ctrl3;
+} __rte_aligned(MLX5_WQE_DWORD_SIZE);
+
+/* Small common part of the WQE. */
+struct mlx5_wqe {
+	uint32_t ctrl[4];
+	struct mlx5_wqe_eth_seg_small eseg;
+};
+
+/* Vectorize WQE header. */
+struct mlx5_wqe_v {
+	rte_v128u32_t ctrl;
+	rte_v128u32_t eseg;
+};
+
+/* WQE. */
+struct mlx5_wqe64 {
+	struct mlx5_wqe hdr;
+	uint8_t raw[32];
+} __rte_aligned(MLX5_WQE_SIZE);
+
+/* MPW mode. */
+enum mlx5_mpw_mode {
+	MLX5_MPW_DISABLED,
+	MLX5_MPW,
+	MLX5_MPW_ENHANCED, /* Enhanced Multi-Packet Send WQE, a.k.a MPWv2. */
+};
+
+/* MPW session status. */
+enum mlx5_mpw_state {
+	MLX5_MPW_STATE_OPENED,
+	MLX5_MPW_INL_STATE_OPENED,
+	MLX5_MPW_ENHANCED_STATE_OPENED,
+	MLX5_MPW_STATE_CLOSED,
+};
+
+/* MPW session descriptor. */
+struct mlx5_mpw {
+	enum mlx5_mpw_state state;
+	unsigned int pkts_n;
+	unsigned int len;
+	unsigned int total_len;
+	volatile struct mlx5_wqe *wqe;
+	union {
+		volatile struct mlx5_wqe_data_seg *dseg[MLX5_MPW_DSEG_MAX];
+		volatile uint8_t *raw;
+	} data;
+};
+
+/* WQE for Multi-Packet RQ. */
+struct mlx5_wqe_mprq {
+	struct mlx5_wqe_srq_next_seg next_seg;
+	struct mlx5_wqe_data_seg dseg;
+};
+
+#define MLX5_MPRQ_LEN_MASK 0x000ffff
+#define MLX5_MPRQ_LEN_SHIFT 0
+#define MLX5_MPRQ_STRIDE_NUM_MASK 0x3fff0000
+#define MLX5_MPRQ_STRIDE_NUM_SHIFT 16
+#define MLX5_MPRQ_FILLER_MASK 0x80000000
+#define MLX5_MPRQ_FILLER_SHIFT 31
+
+#define MLX5_MPRQ_STRIDE_SHIFT_BYTE 2
+
+/* CQ element structure - should be equal to the cache line size */
+struct mlx5_cqe {
+#if (RTE_CACHE_LINE_SIZE == 128)
+	uint8_t padding[64];
+#endif
+	uint8_t pkt_info;
+	uint8_t rsvd0;
+	uint16_t wqe_id;
+	uint8_t rsvd3[8];
+	uint32_t rx_hash_res;
+	uint8_t rx_hash_type;
+	uint8_t rsvd1[11];
+	uint16_t hdr_type_etc;
+	uint16_t vlan_info;
+	uint8_t rsvd2[12];
+	uint32_t byte_cnt;
+	uint64_t timestamp;
+	uint32_t sop_drop_qpn;
+	uint16_t wqe_counter;
+	uint8_t rsvd4;
+	uint8_t op_own;
+};
+
+/* Adding direct verbs to data-path. */
+
+/* CQ sequence number mask. */
+#define MLX5_CQ_SQN_MASK 0x3
+
+/* CQ sequence number index. */
+#define MLX5_CQ_SQN_OFFSET 28
+
+/* CQ doorbell index mask. */
+#define MLX5_CI_MASK 0xffffff
+
+/* CQ doorbell offset. */
+#define MLX5_CQ_ARM_DB 1
+
+/* CQ doorbell offset*/
+#define MLX5_CQ_DOORBELL 0x20
+
+/* CQE format value. */
+#define MLX5_COMPRESSED 0x3
+
+/* CQE format mask. */
+#define MLX5E_CQE_FORMAT_MASK 0xc
+
+/* MPW opcode. */
+#define MLX5_OPC_MOD_MPW 0x01
+
+/* Compressed Rx CQE structure. */
+struct mlx5_mini_cqe8 {
+	union {
+		uint32_t rx_hash_result;
+		struct {
+			uint16_t checksum;
+			uint16_t stride_idx;
+		};
+		struct {
+			uint16_t wqe_counter;
+			uint8_t  s_wqe_opcode;
+			uint8_t  reserved;
+		} s_wqe_info;
+	};
+	uint32_t byte_cnt;
+};
+
+/**
+ * Convert a user mark to flow mark.
+ *
+ * @param val
+ *   Mark value to convert.
+ *
+ * @return
+ *   Converted mark value.
+ */
+static inline uint32_t
+mlx5_flow_mark_set(uint32_t val)
+{
+	uint32_t ret;
+
+	/*
+	 * Add one to the user value to differentiate un-marked flows from
+	 * marked flows, if the ID is equal to MLX5_FLOW_MARK_DEFAULT it
+	 * remains untouched.
+	 */
+	if (val != MLX5_FLOW_MARK_DEFAULT)
+		++val;
+#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
+	/*
+	 * Mark is 24 bits (minus reserved values) but is stored on a 32 bit
+	 * word, byte-swapped by the kernel on little-endian systems. In this
+	 * case, left-shifting the resulting big-endian value ensures the
+	 * least significant 24 bits are retained when converting it back.
+	 */
+	ret = rte_cpu_to_be_32(val) >> 8;
+#else
+	ret = val;
+#endif
+	return ret;
+}
+
+/**
+ * Convert a mark to user mark.
+ *
+ * @param val
+ *   Mark value to convert.
+ *
+ * @return
+ *   Converted mark value.
+ */
+static inline uint32_t
+mlx5_flow_mark_get(uint32_t val)
+{
+	/*
+	 * Subtract one from the retrieved value. It was added by
+	 * mlx5_flow_mark_set() to distinguish unmarked flows.
+	 */
+#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
+	return (val >> 8) - 1;
+#else
+	return val - 1;
+#endif
+}
+
+#endif /* RTE_PMD_MLX5_PRM_H_ */
diff --git a/src/spdk/dpdk/drivers/net/mlx5/mlx5_rss.c b/src/spdk/dpdk/drivers/net/mlx5/mlx5_rss.c
new file mode 100644
index 00000000..b95778a8
--- /dev/null
+++ b/src/spdk/dpdk/drivers/net/mlx5/mlx5_rss.c
@@ -0,0 +1,229 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2015 6WIND S.A.
+ * Copyright 2015 Mellanox Technologies, Ltd
+ */
+
+#include <stddef.h>
+#include <stdint.h>
+#include <errno.h>
+#include <string.h>
+#include <assert.h>
+
+/* Verbs header. */
+/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <infiniband/verbs.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+#include <rte_malloc.h>
+#include <rte_ethdev_driver.h>
+
+#include "mlx5.h"
+#include "mlx5_defs.h"
+#include "mlx5_rxtx.h"
+
+/**
+ * DPDK callback to update the RSS hash configuration.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param[in] rss_conf
+ *   RSS configuration data.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_rss_hash_update(struct rte_eth_dev *dev,
+		     struct rte_eth_rss_conf *rss_conf)
+{
+	struct priv *priv = dev->data->dev_private;
+	unsigned int i;
+	unsigned int idx;
+
+	if (rss_conf->rss_hf & MLX5_RSS_HF_MASK) {
+		rte_errno = EINVAL;
+		return -rte_errno;
+	}
+	if (rss_conf->rss_key && rss_conf->rss_key_len) {
+		if (rss_conf->rss_key_len != MLX5_RSS_HASH_KEY_LEN) {
+			DRV_LOG(ERR,
+				"port %u RSS key len must be %s Bytes long",
+				dev->data->port_id,
+				RTE_STR(MLX5_RSS_HASH_KEY_LEN));
+			rte_errno = EINVAL;
+			return -rte_errno;
+		}
+		priv->rss_conf.rss_key = rte_realloc(priv->rss_conf.rss_key,
+						     rss_conf->rss_key_len, 0);
+		if (!priv->rss_conf.rss_key) {
+			rte_errno = ENOMEM;
+			return -rte_errno;
+		}
+		memcpy(priv->rss_conf.rss_key, rss_conf->rss_key,
+		       rss_conf->rss_key_len);
+		priv->rss_conf.rss_key_len = rss_conf->rss_key_len;
+	}
+	priv->rss_conf.rss_hf = rss_conf->rss_hf;
+	/* Enable the RSS hash in all Rx queues. */
+	for (i = 0, idx = 0; idx != priv->rxqs_n; ++i) {
+		if (!(*priv->rxqs)[i])
+			continue;
+		(*priv->rxqs)[i]->rss_hash = !!rss_conf->rss_hf &&
+			!!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS);
+		++idx;
+	}
+	return 0;
+}
+
+/**
+ * DPDK callback to get the RSS hash configuration.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param[in, out] rss_conf
+ *   RSS configuration data.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_rss_hash_conf_get(struct rte_eth_dev *dev,
+		       struct rte_eth_rss_conf *rss_conf)
+{
+	struct priv *priv = dev->data->dev_private;
+
+	if (!rss_conf) {
+		rte_errno = EINVAL;
+		return -rte_errno;
+	}
+	if (rss_conf->rss_key &&
+	    (rss_conf->rss_key_len >= priv->rss_conf.rss_key_len)) {
+		memcpy(rss_conf->rss_key, priv->rss_conf.rss_key,
+		       priv->rss_conf.rss_key_len);
+	}
+	rss_conf->rss_key_len = priv->rss_conf.rss_key_len;
+	rss_conf->rss_hf = priv->rss_conf.rss_hf;
+	return 0;
+}
+
+/**
+ * Allocate/reallocate RETA index table.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @praram reta_size
+ *   The size of the array to allocate.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_rss_reta_index_resize(struct rte_eth_dev *dev, unsigned int reta_size)
+{
+	struct priv *priv = dev->data->dev_private;
+	void *mem;
+	unsigned int old_size = priv->reta_idx_n;
+
+	if (priv->reta_idx_n == reta_size)
+		return 0;
+
+	mem = rte_realloc(priv->reta_idx,
+			  reta_size * sizeof((*priv->reta_idx)[0]), 0);
+	if (!mem) {
+		rte_errno = ENOMEM;
+		return -rte_errno;
+	}
+	priv->reta_idx = mem;
+	priv->reta_idx_n = reta_size;
+	if (old_size < reta_size)
+		memset(&(*priv->reta_idx)[old_size], 0,
+		       (reta_size - old_size) *
+		       sizeof((*priv->reta_idx)[0]));
+	return 0;
+}
+
+/**
+ * DPDK callback to get the RETA indirection table.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param reta_conf
+ *   Pointer to RETA configuration structure array.
+ * @param reta_size
+ *   Size of the RETA table.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_dev_rss_reta_query(struct rte_eth_dev *dev,
+			struct rte_eth_rss_reta_entry64 *reta_conf,
+			uint16_t reta_size)
+{
+	struct priv *priv = dev->data->dev_private;
+	unsigned int idx;
+	unsigned int i;
+
+	if (!reta_size || reta_size > priv->reta_idx_n) {
+		rte_errno = EINVAL;
+		return -rte_errno;
+	}
+	/* Fill each entry of the table even if its bit is not set. */
+	for (idx = 0, i = 0; (i != reta_size); ++i) {
+		idx = i / RTE_RETA_GROUP_SIZE;
+		reta_conf[idx].reta[i % RTE_RETA_GROUP_SIZE] =
+			(*priv->reta_idx)[i];
+	}
+	return 0;
+}
+
+/**
+ * DPDK callback to update the RETA indirection table.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param reta_conf
+ *   Pointer to RETA configuration structure array.
+ * @param reta_size
+ *   Size of the RETA table.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_dev_rss_reta_update(struct rte_eth_dev *dev,
+			 struct rte_eth_rss_reta_entry64 *reta_conf,
+			 uint16_t reta_size)
+{
+	int ret;
+	struct priv *priv = dev->data->dev_private;
+	unsigned int idx;
+	unsigned int i;
+	unsigned int pos;
+
+	if (!reta_size) {
+		rte_errno = EINVAL;
+		return -rte_errno;
+	}
+	ret = mlx5_rss_reta_index_resize(dev, reta_size);
+	if (ret)
+		return ret;
+	for (idx = 0, i = 0; (i != reta_size); ++i) {
+		idx = i / RTE_RETA_GROUP_SIZE;
+		pos = i % RTE_RETA_GROUP_SIZE;
+		if (((reta_conf[idx].mask >> i) & 0x1) == 0)
+			continue;
+		assert(reta_conf[idx].reta[pos] < priv->rxqs_n);
+		(*priv->reta_idx)[i] = reta_conf[idx].reta[pos];
+	}
+	if (dev->data->dev_started) {
+		mlx5_dev_stop(dev);
+		return mlx5_dev_start(dev);
+	}
+	return 0;
+}
diff --git a/src/spdk/dpdk/drivers/net/mlx5/mlx5_rxmode.c b/src/spdk/dpdk/drivers/net/mlx5/mlx5_rxmode.c
new file mode 100644
index 00000000..e74fdef8
--- /dev/null
+++ b/src/spdk/dpdk/drivers/net/mlx5/mlx5_rxmode.c
@@ -0,0 +1,122 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2015 6WIND S.A.
+ * Copyright 2015 Mellanox Technologies, Ltd
+ */
+
+#include <stddef.h>
+#include <errno.h>
+#include <string.h>
+
+/* Verbs header. */
+/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <infiniband/verbs.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+#include <rte_ethdev_driver.h>
+
+#include "mlx5.h"
+#include "mlx5_rxtx.h"
+#include "mlx5_utils.h"
+
+/**
+ * DPDK callback to enable promiscuous mode.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ */
+void
+mlx5_promiscuous_enable(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	int ret;
+
+	dev->data->promiscuous = 1;
+	if (priv->isolated) {
+		DRV_LOG(WARNING,
+			"port %u cannot enable promiscuous mode"
+			" in flow isolation mode",
+			dev->data->port_id);
+		return;
+	}
+	if (priv->config.vf)
+		mlx5_nl_promisc(dev, 1);
+	ret = mlx5_traffic_restart(dev);
+	if (ret)
+		DRV_LOG(ERR, "port %u cannot enable promiscuous mode: %s",
+			dev->data->port_id, strerror(rte_errno));
+}
+
+/**
+ * DPDK callback to disable promiscuous mode.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ */
+void
+mlx5_promiscuous_disable(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	int ret;
+
+	dev->data->promiscuous = 0;
+	if (priv->config.vf)
+		mlx5_nl_promisc(dev, 0);
+	ret = mlx5_traffic_restart(dev);
+	if (ret)
+		DRV_LOG(ERR, "port %u cannot disable promiscuous mode: %s",
+			dev->data->port_id, strerror(rte_errno));
+}
+
+/**
+ * DPDK callback to enable allmulti mode.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ */
+void
+mlx5_allmulticast_enable(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	int ret;
+
+	dev->data->all_multicast = 1;
+	if (priv->isolated) {
+		DRV_LOG(WARNING,
+			"port %u cannot enable allmulticast mode"
+			" in flow isolation mode",
+			dev->data->port_id);
+		return;
+	}
+	if (priv->config.vf)
+		mlx5_nl_allmulti(dev, 1);
+	ret = mlx5_traffic_restart(dev);
+	if (ret)
+		DRV_LOG(ERR, "port %u cannot enable allmulicast mode: %s",
+			dev->data->port_id, strerror(rte_errno));
+}
+
+/**
+ * DPDK callback to disable allmulti mode.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ */
+void
+mlx5_allmulticast_disable(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	int ret;
+
+	dev->data->all_multicast = 0;
+	if (priv->config.vf)
+		mlx5_nl_allmulti(dev, 0);
+	ret = mlx5_traffic_restart(dev);
+	if (ret)
+		DRV_LOG(ERR, "port %u cannot disable allmulicast mode: %s",
+			dev->data->port_id, strerror(rte_errno));
+}
diff --git a/src/spdk/dpdk/drivers/net/mlx5/mlx5_rxq.c b/src/spdk/dpdk/drivers/net/mlx5/mlx5_rxq.c
new file mode 100644
index 00000000..1f7bfd44
--- /dev/null
+++ b/src/spdk/dpdk/drivers/net/mlx5/mlx5_rxq.c
@@ -0,0 +1,2191 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2015 6WIND S.A.
+ * Copyright 2015 Mellanox Technologies, Ltd
+ */
+
+#include <stddef.h>
+#include <assert.h>
+#include <errno.h>
+#include <string.h>
+#include <stdint.h>
+#include <fcntl.h>
+#include <sys/queue.h>
+
+/* Verbs header. */
+/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <infiniband/verbs.h>
+#include <infiniband/mlx5dv.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+#include <rte_mbuf.h>
+#include <rte_malloc.h>
+#include <rte_ethdev_driver.h>
+#include <rte_common.h>
+#include <rte_interrupts.h>
+#include <rte_debug.h>
+#include <rte_io.h>
+
+#include "mlx5.h"
+#include "mlx5_rxtx.h"
+#include "mlx5_utils.h"
+#include "mlx5_autoconf.h"
+#include "mlx5_defs.h"
+#include "mlx5_glue.h"
+
+/* Default RSS hash key also used for ConnectX-3. */
+uint8_t rss_hash_default_key[] = {
+	0x2c, 0xc6, 0x81, 0xd1,
+	0x5b, 0xdb, 0xf4, 0xf7,
+	0xfc, 0xa2, 0x83, 0x19,
+	0xdb, 0x1a, 0x3e, 0x94,
+	0x6b, 0x9e, 0x38, 0xd9,
+	0x2c, 0x9c, 0x03, 0xd1,
+	0xad, 0x99, 0x44, 0xa7,
+	0xd9, 0x56, 0x3d, 0x59,
+	0x06, 0x3c, 0x25, 0xf3,
+	0xfc, 0x1f, 0xdc, 0x2a,
+};
+
+/* Length of the default RSS hash key. */
+static_assert(MLX5_RSS_HASH_KEY_LEN ==
+	      (unsigned int)sizeof(rss_hash_default_key),
+	      "wrong RSS default key size.");
+
+/**
+ * Check whether Multi-Packet RQ can be enabled for the device.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   1 if supported, negative errno value if not.
+ */
+inline int
+mlx5_check_mprq_support(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+
+	if (priv->config.mprq.enabled &&
+	    priv->rxqs_n >= priv->config.mprq.min_rxqs_num)
+		return 1;
+	return -ENOTSUP;
+}
+
+/**
+ * Check whether Multi-Packet RQ is enabled for the Rx queue.
+ *
+ *  @param rxq
+ *     Pointer to receive queue structure.
+ *
+ * @return
+ *   0 if disabled, otherwise enabled.
+ */
+inline int
+mlx5_rxq_mprq_enabled(struct mlx5_rxq_data *rxq)
+{
+	return rxq->strd_num_n > 0;
+}
+
+/**
+ * Check whether Multi-Packet RQ is enabled for the device.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   0 if disabled, otherwise enabled.
+ */
+inline int
+mlx5_mprq_enabled(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	uint16_t i;
+	uint16_t n = 0;
+
+	if (mlx5_check_mprq_support(dev) < 0)
+		return 0;
+	/* All the configured queues should be enabled. */
+	for (i = 0; i < priv->rxqs_n; ++i) {
+		struct mlx5_rxq_data *rxq = (*priv->rxqs)[i];
+
+		if (!rxq)
+			continue;
+		if (mlx5_rxq_mprq_enabled(rxq))
+			++n;
+	}
+	/* Multi-Packet RQ can't be partially configured. */
+	assert(n == 0 || n == priv->rxqs_n);
+	return n == priv->rxqs_n;
+}
+
+/**
+ * Allocate RX queue elements for Multi-Packet RQ.
+ *
+ * @param rxq_ctrl
+ *   Pointer to RX queue structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+rxq_alloc_elts_mprq(struct mlx5_rxq_ctrl *rxq_ctrl)
+{
+	struct mlx5_rxq_data *rxq = &rxq_ctrl->rxq;
+	unsigned int wqe_n = 1 << rxq->elts_n;
+	unsigned int i;
+	int err;
+
+	/* Iterate on segments. */
+	for (i = 0; i <= wqe_n; ++i) {
+		struct mlx5_mprq_buf *buf;
+
+		if (rte_mempool_get(rxq->mprq_mp, (void **)&buf) < 0) {
+			DRV_LOG(ERR, "port %u empty mbuf pool", rxq->port_id);
+			rte_errno = ENOMEM;
+			goto error;
+		}
+		if (i < wqe_n)
+			(*rxq->mprq_bufs)[i] = buf;
+		else
+			rxq->mprq_repl = buf;
+	}
+	DRV_LOG(DEBUG,
+		"port %u Rx queue %u allocated and configured %u segments",
+		rxq->port_id, rxq_ctrl->idx, wqe_n);
+	return 0;
+error:
+	err = rte_errno; /* Save rte_errno before cleanup. */
+	wqe_n = i;
+	for (i = 0; (i != wqe_n); ++i) {
+		if ((*rxq->mprq_bufs)[i] != NULL)
+			rte_mempool_put(rxq->mprq_mp,
+					(*rxq->mprq_bufs)[i]);
+		(*rxq->mprq_bufs)[i] = NULL;
+	}
+	DRV_LOG(DEBUG, "port %u Rx queue %u failed, freed everything",
+		rxq->port_id, rxq_ctrl->idx);
+	rte_errno = err; /* Restore rte_errno. */
+	return -rte_errno;
+}
+
+/**
+ * Allocate RX queue elements for Single-Packet RQ.
+ *
+ * @param rxq_ctrl
+ *   Pointer to RX queue structure.
+ *
+ * @return
+ *   0 on success, errno value on failure.
+ */
+static int
+rxq_alloc_elts_sprq(struct mlx5_rxq_ctrl *rxq_ctrl)
+{
+	const unsigned int sges_n = 1 << rxq_ctrl->rxq.sges_n;
+	unsigned int elts_n = 1 << rxq_ctrl->rxq.elts_n;
+	unsigned int i;
+	int err;
+
+	/* Iterate on segments. */
+	for (i = 0; (i != elts_n); ++i) {
+		struct rte_mbuf *buf;
+
+		buf = rte_pktmbuf_alloc(rxq_ctrl->rxq.mp);
+		if (buf == NULL) {
+			DRV_LOG(ERR, "port %u empty mbuf pool",
+				PORT_ID(rxq_ctrl->priv));
+			rte_errno = ENOMEM;
+			goto error;
+		}
+		/* Headroom is reserved by rte_pktmbuf_alloc(). */
+		assert(DATA_OFF(buf) == RTE_PKTMBUF_HEADROOM);
+		/* Buffer is supposed to be empty. */
+		assert(rte_pktmbuf_data_len(buf) == 0);
+		assert(rte_pktmbuf_pkt_len(buf) == 0);
+		assert(!buf->next);
+		/* Only the first segment keeps headroom. */
+		if (i % sges_n)
+			SET_DATA_OFF(buf, 0);
+		PORT(buf) = rxq_ctrl->rxq.port_id;
+		DATA_LEN(buf) = rte_pktmbuf_tailroom(buf);
+		PKT_LEN(buf) = DATA_LEN(buf);
+		NB_SEGS(buf) = 1;
+		(*rxq_ctrl->rxq.elts)[i] = buf;
+	}
+	/* If Rx vector is activated. */
+	if (mlx5_rxq_check_vec_support(&rxq_ctrl->rxq) > 0) {
+		struct mlx5_rxq_data *rxq = &rxq_ctrl->rxq;
+		struct rte_mbuf *mbuf_init = &rxq->fake_mbuf;
+		int j;
+
+		/* Initialize default rearm_data for vPMD. */
+		mbuf_init->data_off = RTE_PKTMBUF_HEADROOM;
+		rte_mbuf_refcnt_set(mbuf_init, 1);
+		mbuf_init->nb_segs = 1;
+		mbuf_init->port = rxq->port_id;
+		/*
+		 * prevent compiler reordering:
+		 * rearm_data covers previous fields.
+		 */
+		rte_compiler_barrier();
+		rxq->mbuf_initializer =
+			*(uint64_t *)&mbuf_init->rearm_data;
+		/* Padding with a fake mbuf for vectorized Rx. */
+		for (j = 0; j < MLX5_VPMD_DESCS_PER_LOOP; ++j)
+			(*rxq->elts)[elts_n + j] = &rxq->fake_mbuf;
+	}
+	DRV_LOG(DEBUG,
+		"port %u Rx queue %u allocated and configured %u segments"
+		" (max %u packets)",
+		PORT_ID(rxq_ctrl->priv), rxq_ctrl->idx, elts_n,
+		elts_n / (1 << rxq_ctrl->rxq.sges_n));
+	return 0;
+error:
+	err = rte_errno; /* Save rte_errno before cleanup. */
+	elts_n = i;
+	for (i = 0; (i != elts_n); ++i) {
+		if ((*rxq_ctrl->rxq.elts)[i] != NULL)
+			rte_pktmbuf_free_seg((*rxq_ctrl->rxq.elts)[i]);
+		(*rxq_ctrl->rxq.elts)[i] = NULL;
+	}
+	DRV_LOG(DEBUG, "port %u Rx queue %u failed, freed everything",
+		PORT_ID(rxq_ctrl->priv), rxq_ctrl->idx);
+	rte_errno = err; /* Restore rte_errno. */
+	return -rte_errno;
+}
+
+/**
+ * Allocate RX queue elements.
+ *
+ * @param rxq_ctrl
+ *   Pointer to RX queue structure.
+ *
+ * @return
+ *   0 on success, errno value on failure.
+ */
+int
+rxq_alloc_elts(struct mlx5_rxq_ctrl *rxq_ctrl)
+{
+	return mlx5_rxq_mprq_enabled(&rxq_ctrl->rxq) ?
+	       rxq_alloc_elts_mprq(rxq_ctrl) : rxq_alloc_elts_sprq(rxq_ctrl);
+}
+
+/**
+ * Free RX queue elements for Multi-Packet RQ.
+ *
+ * @param rxq_ctrl
+ *   Pointer to RX queue structure.
+ */
+static void
+rxq_free_elts_mprq(struct mlx5_rxq_ctrl *rxq_ctrl)
+{
+	struct mlx5_rxq_data *rxq = &rxq_ctrl->rxq;
+	uint16_t i;
+
+	DRV_LOG(DEBUG, "port %u Multi-Packet Rx queue %u freeing WRs",
+		rxq->port_id, rxq_ctrl->idx);
+	if (rxq->mprq_bufs == NULL)
+		return;
+	assert(mlx5_rxq_check_vec_support(rxq) < 0);
+	for (i = 0; (i != (1u << rxq->elts_n)); ++i) {
+		if ((*rxq->mprq_bufs)[i] != NULL)
+			mlx5_mprq_buf_free((*rxq->mprq_bufs)[i]);
+		(*rxq->mprq_bufs)[i] = NULL;
+	}
+	if (rxq->mprq_repl != NULL) {
+		mlx5_mprq_buf_free(rxq->mprq_repl);
+		rxq->mprq_repl = NULL;
+	}
+}
+
+/**
+ * Free RX queue elements for Single-Packet RQ.
+ *
+ * @param rxq_ctrl
+ *   Pointer to RX queue structure.
+ */
+static void
+rxq_free_elts_sprq(struct mlx5_rxq_ctrl *rxq_ctrl)
+{
+	struct mlx5_rxq_data *rxq = &rxq_ctrl->rxq;
+	const uint16_t q_n = (1 << rxq->elts_n);
+	const uint16_t q_mask = q_n - 1;
+	uint16_t used = q_n - (rxq->rq_ci - rxq->rq_pi);
+	uint16_t i;
+
+	DRV_LOG(DEBUG, "port %u Rx queue %u freeing WRs",
+		PORT_ID(rxq_ctrl->priv), rxq_ctrl->idx);
+	if (rxq->elts == NULL)
+		return;
+	/**
+	 * Some mbuf in the Ring belongs to the application.  They cannot be
+	 * freed.
+	 */
+	if (mlx5_rxq_check_vec_support(rxq) > 0) {
+		for (i = 0; i < used; ++i)
+			(*rxq->elts)[(rxq->rq_ci + i) & q_mask] = NULL;
+		rxq->rq_pi = rxq->rq_ci;
+	}
+	for (i = 0; (i != (1u << rxq->elts_n)); ++i) {
+		if ((*rxq->elts)[i] != NULL)
+			rte_pktmbuf_free_seg((*rxq->elts)[i]);
+		(*rxq->elts)[i] = NULL;
+	}
+}
+
+/**
+ * Free RX queue elements.
+ *
+ * @param rxq_ctrl
+ *   Pointer to RX queue structure.
+ */
+static void
+rxq_free_elts(struct mlx5_rxq_ctrl *rxq_ctrl)
+{
+	if (mlx5_rxq_mprq_enabled(&rxq_ctrl->rxq))
+		rxq_free_elts_mprq(rxq_ctrl);
+	else
+		rxq_free_elts_sprq(rxq_ctrl);
+}
+
+/**
+ * Clean up a RX queue.
+ *
+ * Destroy objects, free allocated memory and reset the structure for reuse.
+ *
+ * @param rxq_ctrl
+ *   Pointer to RX queue structure.
+ */
+void
+mlx5_rxq_cleanup(struct mlx5_rxq_ctrl *rxq_ctrl)
+{
+	DRV_LOG(DEBUG, "port %u cleaning up Rx queue %u",
+		PORT_ID(rxq_ctrl->priv), rxq_ctrl->idx);
+	if (rxq_ctrl->ibv)
+		mlx5_rxq_ibv_release(rxq_ctrl->ibv);
+	memset(rxq_ctrl, 0, sizeof(*rxq_ctrl));
+}
+
+/**
+ * Returns the per-queue supported offloads.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   Supported Rx offloads.
+ */
+uint64_t
+mlx5_get_rx_queue_offloads(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_dev_config *config = &priv->config;
+	uint64_t offloads = (DEV_RX_OFFLOAD_SCATTER |
+			     DEV_RX_OFFLOAD_TIMESTAMP |
+			     DEV_RX_OFFLOAD_JUMBO_FRAME);
+
+	offloads |= DEV_RX_OFFLOAD_CRC_STRIP;
+	if (config->hw_fcs_strip)
+		offloads |= DEV_RX_OFFLOAD_KEEP_CRC;
+
+	if (config->hw_csum)
+		offloads |= (DEV_RX_OFFLOAD_IPV4_CKSUM |
+			     DEV_RX_OFFLOAD_UDP_CKSUM |
+			     DEV_RX_OFFLOAD_TCP_CKSUM);
+	if (config->hw_vlan_strip)
+		offloads |= DEV_RX_OFFLOAD_VLAN_STRIP;
+	return offloads;
+}
+
+
+/**
+ * Returns the per-port supported offloads.
+ *
+ * @return
+ *   Supported Rx offloads.
+ */
+uint64_t
+mlx5_get_rx_port_offloads(void)
+{
+	uint64_t offloads = DEV_RX_OFFLOAD_VLAN_FILTER;
+
+	return offloads;
+}
+
+/**
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param idx
+ *   RX queue index.
+ * @param desc
+ *   Number of descriptors to configure in queue.
+ * @param socket
+ *   NUMA socket on which memory must be allocated.
+ * @param[in] conf
+ *   Thresholds parameters.
+ * @param mp
+ *   Memory pool for buffer allocations.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
+		    unsigned int socket, const struct rte_eth_rxconf *conf,
+		    struct rte_mempool *mp)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_rxq_data *rxq = (*priv->rxqs)[idx];
+	struct mlx5_rxq_ctrl *rxq_ctrl =
+		container_of(rxq, struct mlx5_rxq_ctrl, rxq);
+
+	if (!rte_is_power_of_2(desc)) {
+		desc = 1 << log2above(desc);
+		DRV_LOG(WARNING,
+			"port %u increased number of descriptors in Rx queue %u"
+			" to the next power of two (%d)",
+			dev->data->port_id, idx, desc);
+	}
+	DRV_LOG(DEBUG, "port %u configuring Rx queue %u for %u descriptors",
+		dev->data->port_id, idx, desc);
+	if (idx >= priv->rxqs_n) {
+		DRV_LOG(ERR, "port %u Rx queue index out of range (%u >= %u)",
+			dev->data->port_id, idx, priv->rxqs_n);
+		rte_errno = EOVERFLOW;
+		return -rte_errno;
+	}
+	if (!mlx5_rxq_releasable(dev, idx)) {
+		DRV_LOG(ERR, "port %u unable to release queue index %u",
+			dev->data->port_id, idx);
+		rte_errno = EBUSY;
+		return -rte_errno;
+	}
+	mlx5_rxq_release(dev, idx);
+	rxq_ctrl = mlx5_rxq_new(dev, idx, desc, socket, conf, mp);
+	if (!rxq_ctrl) {
+		DRV_LOG(ERR, "port %u unable to allocate queue index %u",
+			dev->data->port_id, idx);
+		rte_errno = ENOMEM;
+		return -rte_errno;
+	}
+	DRV_LOG(DEBUG, "port %u adding Rx queue %u to list",
+		dev->data->port_id, idx);
+	(*priv->rxqs)[idx] = &rxq_ctrl->rxq;
+	return 0;
+}
+
+/**
+ * DPDK callback to release a RX queue.
+ *
+ * @param dpdk_rxq
+ *   Generic RX queue pointer.
+ */
+void
+mlx5_rx_queue_release(void *dpdk_rxq)
+{
+	struct mlx5_rxq_data *rxq = (struct mlx5_rxq_data *)dpdk_rxq;
+	struct mlx5_rxq_ctrl *rxq_ctrl;
+	struct priv *priv;
+
+	if (rxq == NULL)
+		return;
+	rxq_ctrl = container_of(rxq, struct mlx5_rxq_ctrl, rxq);
+	priv = rxq_ctrl->priv;
+	if (!mlx5_rxq_releasable(ETH_DEV(priv), rxq_ctrl->rxq.stats.idx))
+		rte_panic("port %u Rx queue %u is still used by a flow and"
+			  " cannot be removed\n",
+			  PORT_ID(priv), rxq_ctrl->idx);
+	mlx5_rxq_release(ETH_DEV(priv), rxq_ctrl->rxq.stats.idx);
+}
+
+/**
+ * Allocate queue vector and fill epoll fd list for Rx interrupts.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_rx_intr_vec_enable(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	unsigned int i;
+	unsigned int rxqs_n = priv->rxqs_n;
+	unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
+	unsigned int count = 0;
+	struct rte_intr_handle *intr_handle = dev->intr_handle;
+
+	if (!dev->data->dev_conf.intr_conf.rxq)
+		return 0;
+	mlx5_rx_intr_vec_disable(dev);
+	intr_handle->intr_vec = malloc(n * sizeof(intr_handle->intr_vec[0]));
+	if (intr_handle->intr_vec == NULL) {
+		DRV_LOG(ERR,
+			"port %u failed to allocate memory for interrupt"
+			" vector, Rx interrupts will not be supported",
+			dev->data->port_id);
+		rte_errno = ENOMEM;
+		return -rte_errno;
+	}
+	intr_handle->type = RTE_INTR_HANDLE_EXT;
+	for (i = 0; i != n; ++i) {
+		/* This rxq ibv must not be released in this function. */
+		struct mlx5_rxq_ibv *rxq_ibv = mlx5_rxq_ibv_get(dev, i);
+		int fd;
+		int flags;
+		int rc;
+
+		/* Skip queues that cannot request interrupts. */
+		if (!rxq_ibv || !rxq_ibv->channel) {
+			/* Use invalid intr_vec[] index to disable entry. */
+			intr_handle->intr_vec[i] =
+				RTE_INTR_VEC_RXTX_OFFSET +
+				RTE_MAX_RXTX_INTR_VEC_ID;
+			continue;
+		}
+		if (count >= RTE_MAX_RXTX_INTR_VEC_ID) {
+			DRV_LOG(ERR,
+				"port %u too many Rx queues for interrupt"
+				" vector size (%d), Rx interrupts cannot be"
+				" enabled",
+				dev->data->port_id, RTE_MAX_RXTX_INTR_VEC_ID);
+			mlx5_rx_intr_vec_disable(dev);
+			rte_errno = ENOMEM;
+			return -rte_errno;
+		}
+		fd = rxq_ibv->channel->fd;
+		flags = fcntl(fd, F_GETFL);
+		rc = fcntl(fd, F_SETFL, flags | O_NONBLOCK);
+		if (rc < 0) {
+			rte_errno = errno;
+			DRV_LOG(ERR,
+				"port %u failed to make Rx interrupt file"
+				" descriptor %d non-blocking for queue index"
+				" %d",
+				dev->data->port_id, fd, i);
+			mlx5_rx_intr_vec_disable(dev);
+			return -rte_errno;
+		}
+		intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + count;
+		intr_handle->efds[count] = fd;
+		count++;
+	}
+	if (!count)
+		mlx5_rx_intr_vec_disable(dev);
+	else
+		intr_handle->nb_efd = count;
+	return 0;
+}
+
+/**
+ * Clean up Rx interrupts handler.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ */
+void
+mlx5_rx_intr_vec_disable(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct rte_intr_handle *intr_handle = dev->intr_handle;
+	unsigned int i;
+	unsigned int rxqs_n = priv->rxqs_n;
+	unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
+
+	if (!dev->data->dev_conf.intr_conf.rxq)
+		return;
+	if (!intr_handle->intr_vec)
+		goto free;
+	for (i = 0; i != n; ++i) {
+		struct mlx5_rxq_ctrl *rxq_ctrl;
+		struct mlx5_rxq_data *rxq_data;
+
+		if (intr_handle->intr_vec[i] == RTE_INTR_VEC_RXTX_OFFSET +
+		    RTE_MAX_RXTX_INTR_VEC_ID)
+			continue;
+		/**
+		 * Need to access directly the queue to release the reference
+		 * kept in priv_rx_intr_vec_enable().
+		 */
+		rxq_data = (*priv->rxqs)[i];
+		rxq_ctrl = container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
+		mlx5_rxq_ibv_release(rxq_ctrl->ibv);
+	}
+free:
+	rte_intr_free_epoll_fd(intr_handle);
+	if (intr_handle->intr_vec)
+		free(intr_handle->intr_vec);
+	intr_handle->nb_efd = 0;
+	intr_handle->intr_vec = NULL;
+}
+
+/**
+ *  MLX5 CQ notification .
+ *
+ *  @param rxq
+ *     Pointer to receive queue structure.
+ *  @param sq_n_rxq
+ *     Sequence number per receive queue .
+ */
+static inline void
+mlx5_arm_cq(struct mlx5_rxq_data *rxq, int sq_n_rxq)
+{
+	int sq_n = 0;
+	uint32_t doorbell_hi;
+	uint64_t doorbell;
+	void *cq_db_reg = (char *)rxq->cq_uar + MLX5_CQ_DOORBELL;
+
+	sq_n = sq_n_rxq & MLX5_CQ_SQN_MASK;
+	doorbell_hi = sq_n << MLX5_CQ_SQN_OFFSET | (rxq->cq_ci & MLX5_CI_MASK);
+	doorbell = (uint64_t)doorbell_hi << 32;
+	doorbell |=  rxq->cqn;
+	rxq->cq_db[MLX5_CQ_ARM_DB] = rte_cpu_to_be_32(doorbell_hi);
+	mlx5_uar_write64(rte_cpu_to_be_64(doorbell),
+			 cq_db_reg, rxq->uar_lock_cq);
+}
+
+/**
+ * DPDK callback for Rx queue interrupt enable.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param rx_queue_id
+ *   Rx queue number.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_rxq_data *rxq_data;
+	struct mlx5_rxq_ctrl *rxq_ctrl;
+
+	rxq_data = (*priv->rxqs)[rx_queue_id];
+	if (!rxq_data) {
+		rte_errno = EINVAL;
+		return -rte_errno;
+	}
+	rxq_ctrl = container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
+	if (rxq_ctrl->irq) {
+		struct mlx5_rxq_ibv *rxq_ibv;
+
+		rxq_ibv = mlx5_rxq_ibv_get(dev, rx_queue_id);
+		if (!rxq_ibv) {
+			rte_errno = EINVAL;
+			return -rte_errno;
+		}
+		mlx5_arm_cq(rxq_data, rxq_data->cq_arm_sn);
+		mlx5_rxq_ibv_release(rxq_ibv);
+	}
+	return 0;
+}
+
+/**
+ * DPDK callback for Rx queue interrupt disable.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param rx_queue_id
+ *   Rx queue number.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_rxq_data *rxq_data;
+	struct mlx5_rxq_ctrl *rxq_ctrl;
+	struct mlx5_rxq_ibv *rxq_ibv = NULL;
+	struct ibv_cq *ev_cq;
+	void *ev_ctx;
+	int ret;
+
+	rxq_data = (*priv->rxqs)[rx_queue_id];
+	if (!rxq_data) {
+		rte_errno = EINVAL;
+		return -rte_errno;
+	}
+	rxq_ctrl = container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
+	if (!rxq_ctrl->irq)
+		return 0;
+	rxq_ibv = mlx5_rxq_ibv_get(dev, rx_queue_id);
+	if (!rxq_ibv) {
+		rte_errno = EINVAL;
+		return -rte_errno;
+	}
+	ret = mlx5_glue->get_cq_event(rxq_ibv->channel, &ev_cq, &ev_ctx);
+	if (ret || ev_cq != rxq_ibv->cq) {
+		rte_errno = EINVAL;
+		goto exit;
+	}
+	rxq_data->cq_arm_sn++;
+	mlx5_glue->ack_cq_events(rxq_ibv->cq, 1);
+	return 0;
+exit:
+	ret = rte_errno; /* Save rte_errno before cleanup. */
+	if (rxq_ibv)
+		mlx5_rxq_ibv_release(rxq_ibv);
+	DRV_LOG(WARNING, "port %u unable to disable interrupt on Rx queue %d",
+		dev->data->port_id, rx_queue_id);
+	rte_errno = ret; /* Restore rte_errno. */
+	return -rte_errno;
+}
+
+/**
+ * Create the Rx queue Verbs object.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param idx
+ *   Queue index in DPDK Rx queue array
+ *
+ * @return
+ *   The Verbs object initialised, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_rxq_ibv *
+mlx5_rxq_ibv_new(struct rte_eth_dev *dev, uint16_t idx)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx];
+	struct mlx5_rxq_ctrl *rxq_ctrl =
+		container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
+	struct ibv_wq_attr mod;
+	union {
+		struct {
+			struct ibv_cq_init_attr_ex ibv;
+			struct mlx5dv_cq_init_attr mlx5;
+		} cq;
+		struct {
+			struct ibv_wq_init_attr ibv;
+#ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
+			struct mlx5dv_wq_init_attr mlx5;
+#endif
+		} wq;
+		struct ibv_cq_ex cq_attr;
+	} attr;
+	unsigned int cqe_n;
+	unsigned int wqe_n = 1 << rxq_data->elts_n;
+	struct mlx5_rxq_ibv *tmpl;
+	struct mlx5dv_cq cq_info;
+	struct mlx5dv_rwq rwq;
+	unsigned int i;
+	int ret = 0;
+	struct mlx5dv_obj obj;
+	struct mlx5_dev_config *config = &priv->config;
+	const int mprq_en = mlx5_rxq_mprq_enabled(rxq_data);
+
+	assert(rxq_data);
+	assert(!rxq_ctrl->ibv);
+	priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_RX_QUEUE;
+	priv->verbs_alloc_ctx.obj = rxq_ctrl;
+	tmpl = rte_calloc_socket(__func__, 1, sizeof(*tmpl), 0,
+				 rxq_ctrl->socket);
+	if (!tmpl) {
+		DRV_LOG(ERR,
+			"port %u Rx queue %u cannot allocate verbs resources",
+			dev->data->port_id, rxq_ctrl->idx);
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	tmpl->rxq_ctrl = rxq_ctrl;
+	if (rxq_ctrl->irq) {
+		tmpl->channel = mlx5_glue->create_comp_channel(priv->ctx);
+		if (!tmpl->channel) {
+			DRV_LOG(ERR, "port %u: comp channel creation failure",
+				dev->data->port_id);
+			rte_errno = ENOMEM;
+			goto error;
+		}
+	}
+	if (mprq_en)
+		cqe_n = wqe_n * (1 << rxq_data->strd_num_n) - 1;
+	else
+		cqe_n = wqe_n  - 1;
+	attr.cq.ibv = (struct ibv_cq_init_attr_ex){
+		.cqe = cqe_n,
+		.channel = tmpl->channel,
+		.comp_mask = 0,
+	};
+	attr.cq.mlx5 = (struct mlx5dv_cq_init_attr){
+		.comp_mask = 0,
+	};
+	if (config->cqe_comp && !rxq_data->hw_timestamp) {
+		attr.cq.mlx5.comp_mask |=
+			MLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE;
+#ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
+		attr.cq.mlx5.cqe_comp_res_format =
+			mprq_en ? MLX5DV_CQE_RES_FORMAT_CSUM_STRIDX :
+				  MLX5DV_CQE_RES_FORMAT_HASH;
+#else
+		attr.cq.mlx5.cqe_comp_res_format = MLX5DV_CQE_RES_FORMAT_HASH;
+#endif
+		/*
+		 * For vectorized Rx, it must not be doubled in order to
+		 * make cq_ci and rq_ci aligned.
+		 */
+		if (mlx5_rxq_check_vec_support(rxq_data) < 0)
+			attr.cq.ibv.cqe *= 2;
+	} else if (config->cqe_comp && rxq_data->hw_timestamp) {
+		DRV_LOG(DEBUG,
+			"port %u Rx CQE compression is disabled for HW"
+			" timestamp",
+			dev->data->port_id);
+	}
+	tmpl->cq = mlx5_glue->cq_ex_to_cq
+		(mlx5_glue->dv_create_cq(priv->ctx, &attr.cq.ibv,
+					 &attr.cq.mlx5));
+	if (tmpl->cq == NULL) {
+		DRV_LOG(ERR, "port %u Rx queue %u CQ creation failure",
+			dev->data->port_id, idx);
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	DRV_LOG(DEBUG, "port %u priv->device_attr.max_qp_wr is %d",
+		dev->data->port_id, priv->device_attr.orig_attr.max_qp_wr);
+	DRV_LOG(DEBUG, "port %u priv->device_attr.max_sge is %d",
+		dev->data->port_id, priv->device_attr.orig_attr.max_sge);
+	attr.wq.ibv = (struct ibv_wq_init_attr){
+		.wq_context = NULL, /* Could be useful in the future. */
+		.wq_type = IBV_WQT_RQ,
+		/* Max number of outstanding WRs. */
+		.max_wr = wqe_n >> rxq_data->sges_n,
+		/* Max number of scatter/gather elements in a WR. */
+		.max_sge = 1 << rxq_data->sges_n,
+		.pd = priv->pd,
+		.cq = tmpl->cq,
+		.comp_mask =
+			IBV_WQ_FLAGS_CVLAN_STRIPPING |
+			0,
+		.create_flags = (rxq_data->vlan_strip ?
+				 IBV_WQ_FLAGS_CVLAN_STRIPPING :
+				 0),
+	};
+	/* By default, FCS (CRC) is stripped by hardware. */
+	if (rxq_data->crc_present) {
+		attr.wq.ibv.create_flags |= IBV_WQ_FLAGS_SCATTER_FCS;
+		attr.wq.ibv.comp_mask |= IBV_WQ_INIT_ATTR_FLAGS;
+	}
+#ifdef HAVE_IBV_WQ_FLAG_RX_END_PADDING
+	if (config->hw_padding) {
+		attr.wq.ibv.create_flags |= IBV_WQ_FLAG_RX_END_PADDING;
+		attr.wq.ibv.comp_mask |= IBV_WQ_INIT_ATTR_FLAGS;
+	}
+#endif
+#ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
+	attr.wq.mlx5 = (struct mlx5dv_wq_init_attr){
+		.comp_mask = 0,
+	};
+	if (mprq_en) {
+		struct mlx5dv_striding_rq_init_attr *mprq_attr =
+			&attr.wq.mlx5.striding_rq_attrs;
+
+		attr.wq.mlx5.comp_mask |= MLX5DV_WQ_INIT_ATTR_MASK_STRIDING_RQ;
+		*mprq_attr = (struct mlx5dv_striding_rq_init_attr){
+			.single_stride_log_num_of_bytes = rxq_data->strd_sz_n,
+			.single_wqe_log_num_of_strides = rxq_data->strd_num_n,
+			.two_byte_shift_en = MLX5_MPRQ_TWO_BYTE_SHIFT,
+		};
+	}
+	tmpl->wq = mlx5_glue->dv_create_wq(priv->ctx, &attr.wq.ibv,
+					   &attr.wq.mlx5);
+#else
+	tmpl->wq = mlx5_glue->create_wq(priv->ctx, &attr.wq.ibv);
+#endif
+	if (tmpl->wq == NULL) {
+		DRV_LOG(ERR, "port %u Rx queue %u WQ creation failure",
+			dev->data->port_id, idx);
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	/*
+	 * Make sure number of WRs*SGEs match expectations since a queue
+	 * cannot allocate more than "desc" buffers.
+	 */
+	if (attr.wq.ibv.max_wr != (wqe_n >> rxq_data->sges_n) ||
+	    attr.wq.ibv.max_sge != (1u << rxq_data->sges_n)) {
+		DRV_LOG(ERR,
+			"port %u Rx queue %u requested %u*%u but got %u*%u"
+			" WRs*SGEs",
+			dev->data->port_id, idx,
+			wqe_n >> rxq_data->sges_n, (1 << rxq_data->sges_n),
+			attr.wq.ibv.max_wr, attr.wq.ibv.max_sge);
+		rte_errno = EINVAL;
+		goto error;
+	}
+	/* Change queue state to ready. */
+	mod = (struct ibv_wq_attr){
+		.attr_mask = IBV_WQ_ATTR_STATE,
+		.wq_state = IBV_WQS_RDY,
+	};
+	ret = mlx5_glue->modify_wq(tmpl->wq, &mod);
+	if (ret) {
+		DRV_LOG(ERR,
+			"port %u Rx queue %u WQ state to IBV_WQS_RDY failed",
+			dev->data->port_id, idx);
+		rte_errno = ret;
+		goto error;
+	}
+	obj.cq.in = tmpl->cq;
+	obj.cq.out = &cq_info;
+	obj.rwq.in = tmpl->wq;
+	obj.rwq.out = &rwq;
+	ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_CQ | MLX5DV_OBJ_RWQ);
+	if (ret) {
+		rte_errno = ret;
+		goto error;
+	}
+	if (cq_info.cqe_size != RTE_CACHE_LINE_SIZE) {
+		DRV_LOG(ERR,
+			"port %u wrong MLX5_CQE_SIZE environment variable"
+			" value: it should be set to %u",
+			dev->data->port_id, RTE_CACHE_LINE_SIZE);
+		rte_errno = EINVAL;
+		goto error;
+	}
+	/* Fill the rings. */
+	rxq_data->wqes = rwq.buf;
+	for (i = 0; (i != wqe_n); ++i) {
+		volatile struct mlx5_wqe_data_seg *scat;
+		uintptr_t addr;
+		uint32_t byte_count;
+
+		if (mprq_en) {
+			struct mlx5_mprq_buf *buf = (*rxq_data->mprq_bufs)[i];
+
+			scat = &((volatile struct mlx5_wqe_mprq *)
+				 rxq_data->wqes)[i].dseg;
+			addr = (uintptr_t)mlx5_mprq_buf_addr(buf);
+			byte_count = (1 << rxq_data->strd_sz_n) *
+				     (1 << rxq_data->strd_num_n);
+		} else {
+			struct rte_mbuf *buf = (*rxq_data->elts)[i];
+
+			scat = &((volatile struct mlx5_wqe_data_seg *)
+				 rxq_data->wqes)[i];
+			addr = rte_pktmbuf_mtod(buf, uintptr_t);
+			byte_count = DATA_LEN(buf);
+		}
+		/* scat->addr must be able to store a pointer. */
+		assert(sizeof(scat->addr) >= sizeof(uintptr_t));
+		*scat = (struct mlx5_wqe_data_seg){
+			.addr = rte_cpu_to_be_64(addr),
+			.byte_count = rte_cpu_to_be_32(byte_count),
+			.lkey = mlx5_rx_addr2mr(rxq_data, addr),
+		};
+	}
+	rxq_data->rq_db = rwq.dbrec;
+	rxq_data->cqe_n = log2above(cq_info.cqe_cnt);
+	rxq_data->cq_ci = 0;
+	rxq_data->consumed_strd = 0;
+	rxq_data->rq_pi = 0;
+	rxq_data->zip = (struct rxq_zip){
+		.ai = 0,
+	};
+	rxq_data->cq_db = cq_info.dbrec;
+	rxq_data->cqes = (volatile struct mlx5_cqe (*)[])(uintptr_t)cq_info.buf;
+	rxq_data->cq_uar = cq_info.cq_uar;
+	rxq_data->cqn = cq_info.cqn;
+	rxq_data->cq_arm_sn = 0;
+	/* Update doorbell counter. */
+	rxq_data->rq_ci = wqe_n >> rxq_data->sges_n;
+	rte_wmb();
+	*rxq_data->rq_db = rte_cpu_to_be_32(rxq_data->rq_ci);
+	DRV_LOG(DEBUG, "port %u rxq %u updated with %p", dev->data->port_id,
+		idx, (void *)&tmpl);
+	rte_atomic32_inc(&tmpl->refcnt);
+	LIST_INSERT_HEAD(&priv->rxqsibv, tmpl, next);
+	priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE;
+	return tmpl;
+error:
+	ret = rte_errno; /* Save rte_errno before cleanup. */
+	if (tmpl->wq)
+		claim_zero(mlx5_glue->destroy_wq(tmpl->wq));
+	if (tmpl->cq)
+		claim_zero(mlx5_glue->destroy_cq(tmpl->cq));
+	if (tmpl->channel)
+		claim_zero(mlx5_glue->destroy_comp_channel(tmpl->channel));
+	priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE;
+	rte_errno = ret; /* Restore rte_errno. */
+	return NULL;
+}
+
+/**
+ * Get an Rx queue Verbs object.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param idx
+ *   Queue index in DPDK Rx queue array
+ *
+ * @return
+ *   The Verbs object if it exists.
+ */
+struct mlx5_rxq_ibv *
+mlx5_rxq_ibv_get(struct rte_eth_dev *dev, uint16_t idx)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx];
+	struct mlx5_rxq_ctrl *rxq_ctrl;
+
+	if (idx >= priv->rxqs_n)
+		return NULL;
+	if (!rxq_data)
+		return NULL;
+	rxq_ctrl = container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
+	if (rxq_ctrl->ibv) {
+		rte_atomic32_inc(&rxq_ctrl->ibv->refcnt);
+	}
+	return rxq_ctrl->ibv;
+}
+
+/**
+ * Release an Rx verbs queue object.
+ *
+ * @param rxq_ibv
+ *   Verbs Rx queue object.
+ *
+ * @return
+ *   1 while a reference on it exists, 0 when freed.
+ */
+int
+mlx5_rxq_ibv_release(struct mlx5_rxq_ibv *rxq_ibv)
+{
+	assert(rxq_ibv);
+	assert(rxq_ibv->wq);
+	assert(rxq_ibv->cq);
+	if (rte_atomic32_dec_and_test(&rxq_ibv->refcnt)) {
+		rxq_free_elts(rxq_ibv->rxq_ctrl);
+		claim_zero(mlx5_glue->destroy_wq(rxq_ibv->wq));
+		claim_zero(mlx5_glue->destroy_cq(rxq_ibv->cq));
+		if (rxq_ibv->channel)
+			claim_zero(mlx5_glue->destroy_comp_channel
+				   (rxq_ibv->channel));
+		LIST_REMOVE(rxq_ibv, next);
+		rte_free(rxq_ibv);
+		return 0;
+	}
+	return 1;
+}
+
+/**
+ * Verify the Verbs Rx queue list is empty
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   The number of object not released.
+ */
+int
+mlx5_rxq_ibv_verify(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	int ret = 0;
+	struct mlx5_rxq_ibv *rxq_ibv;
+
+	LIST_FOREACH(rxq_ibv, &priv->rxqsibv, next) {
+		DRV_LOG(DEBUG, "port %u Verbs Rx queue %u still referenced",
+			dev->data->port_id, rxq_ibv->rxq_ctrl->idx);
+		++ret;
+	}
+	return ret;
+}
+
+/**
+ * Return true if a single reference exists on the object.
+ *
+ * @param rxq_ibv
+ *   Verbs Rx queue object.
+ */
+int
+mlx5_rxq_ibv_releasable(struct mlx5_rxq_ibv *rxq_ibv)
+{
+	assert(rxq_ibv);
+	return (rte_atomic32_read(&rxq_ibv->refcnt) == 1);
+}
+
+/**
+ * Callback function to initialize mbufs for Multi-Packet RQ.
+ */
+static inline void
+mlx5_mprq_buf_init(struct rte_mempool *mp, void *opaque_arg __rte_unused,
+		    void *_m, unsigned int i __rte_unused)
+{
+	struct mlx5_mprq_buf *buf = _m;
+
+	memset(_m, 0, sizeof(*buf));
+	buf->mp = mp;
+	rte_atomic16_set(&buf->refcnt, 1);
+}
+
+/**
+ * Free mempool of Multi-Packet RQ.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   0 on success, negative errno value on failure.
+ */
+int
+mlx5_mprq_free_mp(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct rte_mempool *mp = priv->mprq_mp;
+	unsigned int i;
+
+	if (mp == NULL)
+		return 0;
+	DRV_LOG(DEBUG, "port %u freeing mempool (%s) for Multi-Packet RQ",
+		dev->data->port_id, mp->name);
+	/*
+	 * If a buffer in the pool has been externally attached to a mbuf and it
+	 * is still in use by application, destroying the Rx qeueue can spoil
+	 * the packet. It is unlikely to happen but if application dynamically
+	 * creates and destroys with holding Rx packets, this can happen.
+	 *
+	 * TODO: It is unavoidable for now because the mempool for Multi-Packet
+	 * RQ isn't provided by application but managed by PMD.
+	 */
+	if (!rte_mempool_full(mp)) {
+		DRV_LOG(ERR,
+			"port %u mempool for Multi-Packet RQ is still in use",
+			dev->data->port_id);
+		rte_errno = EBUSY;
+		return -rte_errno;
+	}
+	rte_mempool_free(mp);
+	/* Unset mempool for each Rx queue. */
+	for (i = 0; i != priv->rxqs_n; ++i) {
+		struct mlx5_rxq_data *rxq = (*priv->rxqs)[i];
+
+		if (rxq == NULL)
+			continue;
+		rxq->mprq_mp = NULL;
+	}
+	return 0;
+}
+
+/**
+ * Allocate a mempool for Multi-Packet RQ. All configured Rx queues share the
+ * mempool. If already allocated, reuse it if there're enough elements.
+ * Otherwise, resize it.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   0 on success, negative errno value on failure.
+ */
+int
+mlx5_mprq_alloc_mp(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct rte_mempool *mp = priv->mprq_mp;
+	char name[RTE_MEMPOOL_NAMESIZE];
+	unsigned int desc = 0;
+	unsigned int buf_len;
+	unsigned int obj_num;
+	unsigned int obj_size;
+	unsigned int strd_num_n = 0;
+	unsigned int strd_sz_n = 0;
+	unsigned int i;
+
+	if (!mlx5_mprq_enabled(dev))
+		return 0;
+	/* Count the total number of descriptors configured. */
+	for (i = 0; i != priv->rxqs_n; ++i) {
+		struct mlx5_rxq_data *rxq = (*priv->rxqs)[i];
+
+		if (rxq == NULL)
+			continue;
+		desc += 1 << rxq->elts_n;
+		/* Get the max number of strides. */
+		if (strd_num_n < rxq->strd_num_n)
+			strd_num_n = rxq->strd_num_n;
+		/* Get the max size of a stride. */
+		if (strd_sz_n < rxq->strd_sz_n)
+			strd_sz_n = rxq->strd_sz_n;
+	}
+	assert(strd_num_n && strd_sz_n);
+	buf_len = (1 << strd_num_n) * (1 << strd_sz_n);
+	obj_size = buf_len + sizeof(struct mlx5_mprq_buf);
+	/*
+	 * Received packets can be either memcpy'd or externally referenced. In
+	 * case that the packet is attached to an mbuf as an external buffer, as
+	 * it isn't possible to predict how the buffers will be queued by
+	 * application, there's no option to exactly pre-allocate needed buffers
+	 * in advance but to speculatively prepares enough buffers.
+	 *
+	 * In the data path, if this Mempool is depleted, PMD will try to memcpy
+	 * received packets to buffers provided by application (rxq->mp) until
+	 * this Mempool gets available again.
+	 */
+	desc *= 4;
+	obj_num = desc + MLX5_MPRQ_MP_CACHE_SZ * priv->rxqs_n;
+	/*
+	 * rte_mempool_create_empty() has sanity check to refuse large cache
+	 * size compared to the number of elements.
+	 * CACHE_FLUSHTHRESH_MULTIPLIER is defined in a C file, so using a
+	 * constant number 2 instead.
+	 */
+	obj_num = RTE_MAX(obj_num, MLX5_MPRQ_MP_CACHE_SZ * 2);
+	/* Check a mempool is already allocated and if it can be resued. */
+	if (mp != NULL && mp->elt_size >= obj_size && mp->size >= obj_num) {
+		DRV_LOG(DEBUG, "port %u mempool %s is being reused",
+			dev->data->port_id, mp->name);
+		/* Reuse. */
+		goto exit;
+	} else if (mp != NULL) {
+		DRV_LOG(DEBUG, "port %u mempool %s should be resized, freeing it",
+			dev->data->port_id, mp->name);
+		/*
+		 * If failed to free, which means it may be still in use, no way
+		 * but to keep using the existing one. On buffer underrun,
+		 * packets will be memcpy'd instead of external buffer
+		 * attachment.
+		 */
+		if (mlx5_mprq_free_mp(dev)) {
+			if (mp->elt_size >= obj_size)
+				goto exit;
+			else
+				return -rte_errno;
+		}
+	}
+	snprintf(name, sizeof(name), "%s-mprq", dev->device->name);
+	mp = rte_mempool_create(name, obj_num, obj_size, MLX5_MPRQ_MP_CACHE_SZ,
+				0, NULL, NULL, mlx5_mprq_buf_init, NULL,
+				dev->device->numa_node, 0);
+	if (mp == NULL) {
+		DRV_LOG(ERR,
+			"port %u failed to allocate a mempool for"
+			" Multi-Packet RQ, count=%u, size=%u",
+			dev->data->port_id, obj_num, obj_size);
+		rte_errno = ENOMEM;
+		return -rte_errno;
+	}
+	priv->mprq_mp = mp;
+exit:
+	/* Set mempool for each Rx queue. */
+	for (i = 0; i != priv->rxqs_n; ++i) {
+		struct mlx5_rxq_data *rxq = (*priv->rxqs)[i];
+
+		if (rxq == NULL)
+			continue;
+		rxq->mprq_mp = mp;
+	}
+	DRV_LOG(INFO, "port %u Multi-Packet RQ is configured",
+		dev->data->port_id);
+	return 0;
+}
+
+/**
+ * Create a DPDK Rx queue.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param idx
+ *   RX queue index.
+ * @param desc
+ *   Number of descriptors to configure in queue.
+ * @param socket
+ *   NUMA socket on which memory must be allocated.
+ *
+ * @return
+ *   A DPDK queue object on success, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_rxq_ctrl *
+mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
+	     unsigned int socket, const struct rte_eth_rxconf *conf,
+	     struct rte_mempool *mp)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_rxq_ctrl *tmpl;
+	unsigned int mb_len = rte_pktmbuf_data_room_size(mp);
+	unsigned int mprq_stride_size;
+	struct mlx5_dev_config *config = &priv->config;
+	/*
+	 * Always allocate extra slots, even if eventually
+	 * the vector Rx will not be used.
+	 */
+	uint16_t desc_n =
+		desc + config->rx_vec_en * MLX5_VPMD_DESCS_PER_LOOP;
+	uint64_t offloads = conf->offloads |
+			   dev->data->dev_conf.rxmode.offloads;
+	const int mprq_en = mlx5_check_mprq_support(dev) > 0;
+
+	tmpl = rte_calloc_socket("RXQ", 1,
+				 sizeof(*tmpl) +
+				 desc_n * sizeof(struct rte_mbuf *),
+				 0, socket);
+	if (!tmpl) {
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+	if (mlx5_mr_btree_init(&tmpl->rxq.mr_ctrl.cache_bh,
+			       MLX5_MR_BTREE_CACHE_N, socket)) {
+		/* rte_errno is already set. */
+		goto error;
+	}
+	tmpl->socket = socket;
+	if (dev->data->dev_conf.intr_conf.rxq)
+		tmpl->irq = 1;
+	/*
+	 * This Rx queue can be configured as a Multi-Packet RQ if all of the
+	 * following conditions are met:
+	 *  - MPRQ is enabled.
+	 *  - The number of descs is more than the number of strides.
+	 *  - max_rx_pkt_len plus overhead is less than the max size of a
+	 *    stride.
+	 *  Otherwise, enable Rx scatter if necessary.
+	 */
+	assert(mb_len >= RTE_PKTMBUF_HEADROOM);
+	mprq_stride_size =
+		dev->data->dev_conf.rxmode.max_rx_pkt_len +
+		sizeof(struct rte_mbuf_ext_shared_info) +
+		RTE_PKTMBUF_HEADROOM;
+	if (mprq_en &&
+	    desc > (1U << config->mprq.stride_num_n) &&
+	    mprq_stride_size <= (1U << config->mprq.max_stride_size_n)) {
+		/* TODO: Rx scatter isn't supported yet. */
+		tmpl->rxq.sges_n = 0;
+		/* Trim the number of descs needed. */
+		desc >>= config->mprq.stride_num_n;
+		tmpl->rxq.strd_num_n = config->mprq.stride_num_n;
+		tmpl->rxq.strd_sz_n = RTE_MAX(log2above(mprq_stride_size),
+					      config->mprq.min_stride_size_n);
+		tmpl->rxq.strd_shift_en = MLX5_MPRQ_TWO_BYTE_SHIFT;
+		tmpl->rxq.mprq_max_memcpy_len =
+			RTE_MIN(mb_len - RTE_PKTMBUF_HEADROOM,
+				config->mprq.max_memcpy_len);
+		DRV_LOG(DEBUG,
+			"port %u Rx queue %u: Multi-Packet RQ is enabled"
+			" strd_num_n = %u, strd_sz_n = %u",
+			dev->data->port_id, idx,
+			tmpl->rxq.strd_num_n, tmpl->rxq.strd_sz_n);
+	} else if (dev->data->dev_conf.rxmode.max_rx_pkt_len <=
+		   (mb_len - RTE_PKTMBUF_HEADROOM)) {
+		tmpl->rxq.sges_n = 0;
+	} else if (offloads & DEV_RX_OFFLOAD_SCATTER) {
+		unsigned int size =
+			RTE_PKTMBUF_HEADROOM +
+			dev->data->dev_conf.rxmode.max_rx_pkt_len;
+		unsigned int sges_n;
+
+		/*
+		 * Determine the number of SGEs needed for a full packet
+		 * and round it to the next power of two.
+		 */
+		sges_n = log2above((size / mb_len) + !!(size % mb_len));
+		tmpl->rxq.sges_n = sges_n;
+		/* Make sure rxq.sges_n did not overflow. */
+		size = mb_len * (1 << tmpl->rxq.sges_n);
+		size -= RTE_PKTMBUF_HEADROOM;
+		if (size < dev->data->dev_conf.rxmode.max_rx_pkt_len) {
+			DRV_LOG(ERR,
+				"port %u too many SGEs (%u) needed to handle"
+				" requested maximum packet size %u",
+				dev->data->port_id,
+				1 << sges_n,
+				dev->data->dev_conf.rxmode.max_rx_pkt_len);
+			rte_errno = EOVERFLOW;
+			goto error;
+		}
+	} else {
+		DRV_LOG(WARNING,
+			"port %u the requested maximum Rx packet size (%u) is"
+			" larger than a single mbuf (%u) and scattered mode has"
+			" not been requested",
+			dev->data->port_id,
+			dev->data->dev_conf.rxmode.max_rx_pkt_len,
+			mb_len - RTE_PKTMBUF_HEADROOM);
+	}
+	if (mprq_en && !mlx5_rxq_mprq_enabled(&tmpl->rxq))
+		DRV_LOG(WARNING,
+			"port %u MPRQ is requested but cannot be enabled"
+			" (requested: desc = %u, stride_sz = %u,"
+			" supported: min_stride_num = %u, max_stride_sz = %u).",
+			dev->data->port_id, desc, mprq_stride_size,
+			(1 << config->mprq.stride_num_n),
+			(1 << config->mprq.max_stride_size_n));
+	DRV_LOG(DEBUG, "port %u maximum number of segments per packet: %u",
+		dev->data->port_id, 1 << tmpl->rxq.sges_n);
+	if (desc % (1 << tmpl->rxq.sges_n)) {
+		DRV_LOG(ERR,
+			"port %u number of Rx queue descriptors (%u) is not a"
+			" multiple of SGEs per packet (%u)",
+			dev->data->port_id,
+			desc,
+			1 << tmpl->rxq.sges_n);
+		rte_errno = EINVAL;
+		goto error;
+	}
+	/* Toggle RX checksum offload if hardware supports it. */
+	tmpl->rxq.csum = !!(offloads & DEV_RX_OFFLOAD_CHECKSUM);
+	tmpl->rxq.hw_timestamp = !!(offloads & DEV_RX_OFFLOAD_TIMESTAMP);
+	/* Configure VLAN stripping. */
+	tmpl->rxq.vlan_strip = !!(offloads & DEV_RX_OFFLOAD_VLAN_STRIP);
+	/* By default, FCS (CRC) is stripped by hardware. */
+	tmpl->rxq.crc_present = 0;
+	if (rte_eth_dev_must_keep_crc(offloads)) {
+		if (config->hw_fcs_strip) {
+			tmpl->rxq.crc_present = 1;
+		} else {
+			DRV_LOG(WARNING,
+				"port %u CRC stripping has been disabled but will"
+				" still be performed by hardware, make sure MLNX_OFED"
+				" and firmware are up to date",
+				dev->data->port_id);
+		}
+	}
+	DRV_LOG(DEBUG,
+		"port %u CRC stripping is %s, %u bytes will be subtracted from"
+		" incoming frames to hide it",
+		dev->data->port_id,
+		tmpl->rxq.crc_present ? "disabled" : "enabled",
+		tmpl->rxq.crc_present << 2);
+	/* Save port ID. */
+	tmpl->rxq.rss_hash = !!priv->rss_conf.rss_hf &&
+		(!!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS));
+	tmpl->rxq.port_id = dev->data->port_id;
+	tmpl->priv = priv;
+	tmpl->rxq.mp = mp;
+	tmpl->rxq.stats.idx = idx;
+	tmpl->rxq.elts_n = log2above(desc);
+	tmpl->rxq.elts =
+		(struct rte_mbuf *(*)[1 << tmpl->rxq.elts_n])(tmpl + 1);
+#ifndef RTE_ARCH_64
+	tmpl->rxq.uar_lock_cq = &priv->uar_lock_cq;
+#endif
+	tmpl->idx = idx;
+	rte_atomic32_inc(&tmpl->refcnt);
+	LIST_INSERT_HEAD(&priv->rxqsctrl, tmpl, next);
+	return tmpl;
+error:
+	rte_free(tmpl);
+	return NULL;
+}
+
+/**
+ * Get a Rx queue.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param idx
+ *   TX queue index.
+ *
+ * @return
+ *   A pointer to the queue if it exists, NULL otherwise.
+ */
+struct mlx5_rxq_ctrl *
+mlx5_rxq_get(struct rte_eth_dev *dev, uint16_t idx)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_rxq_ctrl *rxq_ctrl = NULL;
+
+	if ((*priv->rxqs)[idx]) {
+		rxq_ctrl = container_of((*priv->rxqs)[idx],
+					struct mlx5_rxq_ctrl,
+					rxq);
+		mlx5_rxq_ibv_get(dev, idx);
+		rte_atomic32_inc(&rxq_ctrl->refcnt);
+	}
+	return rxq_ctrl;
+}
+
+/**
+ * Release a Rx queue.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param idx
+ *   TX queue index.
+ *
+ * @return
+ *   1 while a reference on it exists, 0 when freed.
+ */
+int
+mlx5_rxq_release(struct rte_eth_dev *dev, uint16_t idx)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_rxq_ctrl *rxq_ctrl;
+
+	if (!(*priv->rxqs)[idx])
+		return 0;
+	rxq_ctrl = container_of((*priv->rxqs)[idx], struct mlx5_rxq_ctrl, rxq);
+	assert(rxq_ctrl->priv);
+	if (rxq_ctrl->ibv && !mlx5_rxq_ibv_release(rxq_ctrl->ibv))
+		rxq_ctrl->ibv = NULL;
+	if (rte_atomic32_dec_and_test(&rxq_ctrl->refcnt)) {
+		mlx5_mr_btree_free(&rxq_ctrl->rxq.mr_ctrl.cache_bh);
+		LIST_REMOVE(rxq_ctrl, next);
+		rte_free(rxq_ctrl);
+		(*priv->rxqs)[idx] = NULL;
+		return 0;
+	}
+	return 1;
+}
+
+/**
+ * Verify if the queue can be released.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param idx
+ *   TX queue index.
+ *
+ * @return
+ *   1 if the queue can be released, negative errno otherwise and rte_errno is
+ *   set.
+ */
+int
+mlx5_rxq_releasable(struct rte_eth_dev *dev, uint16_t idx)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_rxq_ctrl *rxq_ctrl;
+
+	if (!(*priv->rxqs)[idx]) {
+		rte_errno = EINVAL;
+		return -rte_errno;
+	}
+	rxq_ctrl = container_of((*priv->rxqs)[idx], struct mlx5_rxq_ctrl, rxq);
+	return (rte_atomic32_read(&rxq_ctrl->refcnt) == 1);
+}
+
+/**
+ * Verify the Rx Queue list is empty
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   The number of object not released.
+ */
+int
+mlx5_rxq_verify(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_rxq_ctrl *rxq_ctrl;
+	int ret = 0;
+
+	LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) {
+		DRV_LOG(DEBUG, "port %u Rx Queue %u still referenced",
+			dev->data->port_id, rxq_ctrl->idx);
+		++ret;
+	}
+	return ret;
+}
+
+/**
+ * Create an indirection table.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param queues
+ *   Queues entering in the indirection table.
+ * @param queues_n
+ *   Number of queues in the array.
+ *
+ * @return
+ *   The Verbs object initialised, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_ind_table_ibv *
+mlx5_ind_table_ibv_new(struct rte_eth_dev *dev, const uint16_t *queues,
+		       uint32_t queues_n)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_ind_table_ibv *ind_tbl;
+	const unsigned int wq_n = rte_is_power_of_2(queues_n) ?
+		log2above(queues_n) :
+		log2above(priv->config.ind_table_max_size);
+	struct ibv_wq *wq[1 << wq_n];
+	unsigned int i;
+	unsigned int j;
+
+	ind_tbl = rte_calloc(__func__, 1, sizeof(*ind_tbl) +
+			     queues_n * sizeof(uint16_t), 0);
+	if (!ind_tbl) {
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+	for (i = 0; i != queues_n; ++i) {
+		struct mlx5_rxq_ctrl *rxq = mlx5_rxq_get(dev, queues[i]);
+
+		if (!rxq)
+			goto error;
+		wq[i] = rxq->ibv->wq;
+		ind_tbl->queues[i] = queues[i];
+	}
+	ind_tbl->queues_n = queues_n;
+	/* Finalise indirection table. */
+	for (j = 0; i != (unsigned int)(1 << wq_n); ++i, ++j)
+		wq[i] = wq[j];
+	ind_tbl->ind_table = mlx5_glue->create_rwq_ind_table
+		(priv->ctx,
+		 &(struct ibv_rwq_ind_table_init_attr){
+			.log_ind_tbl_size = wq_n,
+			.ind_tbl = wq,
+			.comp_mask = 0,
+		 });
+	if (!ind_tbl->ind_table) {
+		rte_errno = errno;
+		goto error;
+	}
+	rte_atomic32_inc(&ind_tbl->refcnt);
+	LIST_INSERT_HEAD(&priv->ind_tbls, ind_tbl, next);
+	return ind_tbl;
+error:
+	rte_free(ind_tbl);
+	DEBUG("port %u cannot create indirection table", dev->data->port_id);
+	return NULL;
+}
+
+/**
+ * Get an indirection table.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param queues
+ *   Queues entering in the indirection table.
+ * @param queues_n
+ *   Number of queues in the array.
+ *
+ * @return
+ *   An indirection table if found.
+ */
+struct mlx5_ind_table_ibv *
+mlx5_ind_table_ibv_get(struct rte_eth_dev *dev, const uint16_t *queues,
+		       uint32_t queues_n)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_ind_table_ibv *ind_tbl;
+
+	LIST_FOREACH(ind_tbl, &priv->ind_tbls, next) {
+		if ((ind_tbl->queues_n == queues_n) &&
+		    (memcmp(ind_tbl->queues, queues,
+			    ind_tbl->queues_n * sizeof(ind_tbl->queues[0]))
+		     == 0))
+			break;
+	}
+	if (ind_tbl) {
+		unsigned int i;
+
+		rte_atomic32_inc(&ind_tbl->refcnt);
+		for (i = 0; i != ind_tbl->queues_n; ++i)
+			mlx5_rxq_get(dev, ind_tbl->queues[i]);
+	}
+	return ind_tbl;
+}
+
+/**
+ * Release an indirection table.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param ind_table
+ *   Indirection table to release.
+ *
+ * @return
+ *   1 while a reference on it exists, 0 when freed.
+ */
+int
+mlx5_ind_table_ibv_release(struct rte_eth_dev *dev,
+			   struct mlx5_ind_table_ibv *ind_tbl)
+{
+	unsigned int i;
+
+	if (rte_atomic32_dec_and_test(&ind_tbl->refcnt))
+		claim_zero(mlx5_glue->destroy_rwq_ind_table
+			   (ind_tbl->ind_table));
+	for (i = 0; i != ind_tbl->queues_n; ++i)
+		claim_nonzero(mlx5_rxq_release(dev, ind_tbl->queues[i]));
+	if (!rte_atomic32_read(&ind_tbl->refcnt)) {
+		LIST_REMOVE(ind_tbl, next);
+		rte_free(ind_tbl);
+		return 0;
+	}
+	return 1;
+}
+
+/**
+ * Verify the Rx Queue list is empty
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   The number of object not released.
+ */
+int
+mlx5_ind_table_ibv_verify(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_ind_table_ibv *ind_tbl;
+	int ret = 0;
+
+	LIST_FOREACH(ind_tbl, &priv->ind_tbls, next) {
+		DRV_LOG(DEBUG,
+			"port %u Verbs indirection table %p still referenced",
+			dev->data->port_id, (void *)ind_tbl);
+		++ret;
+	}
+	return ret;
+}
+
+/**
+ * Create an Rx Hash queue.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param rss_key
+ *   RSS key for the Rx hash queue.
+ * @param rss_key_len
+ *   RSS key length.
+ * @param hash_fields
+ *   Verbs protocol hash field to make the RSS on.
+ * @param queues
+ *   Queues entering in hash queue. In case of empty hash_fields only the
+ *   first queue index will be taken for the indirection table.
+ * @param queues_n
+ *   Number of queues.
+ *
+ * @return
+ *   The Verbs object initialised, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_hrxq *
+mlx5_hrxq_new(struct rte_eth_dev *dev,
+	      const uint8_t *rss_key, uint32_t rss_key_len,
+	      uint64_t hash_fields,
+	      const uint16_t *queues, uint32_t queues_n,
+	      int tunnel __rte_unused)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_hrxq *hrxq;
+	struct mlx5_ind_table_ibv *ind_tbl;
+	struct ibv_qp *qp;
+	int err;
+
+	queues_n = hash_fields ? queues_n : 1;
+	ind_tbl = mlx5_ind_table_ibv_get(dev, queues, queues_n);
+	if (!ind_tbl)
+		ind_tbl = mlx5_ind_table_ibv_new(dev, queues, queues_n);
+	if (!ind_tbl) {
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+	if (!rss_key_len) {
+		rss_key_len = MLX5_RSS_HASH_KEY_LEN;
+		rss_key = rss_hash_default_key;
+	}
+#ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
+	qp = mlx5_glue->dv_create_qp
+		(priv->ctx,
+		 &(struct ibv_qp_init_attr_ex){
+			.qp_type = IBV_QPT_RAW_PACKET,
+			.comp_mask =
+				IBV_QP_INIT_ATTR_PD |
+				IBV_QP_INIT_ATTR_IND_TABLE |
+				IBV_QP_INIT_ATTR_RX_HASH,
+			.rx_hash_conf = (struct ibv_rx_hash_conf){
+				.rx_hash_function = IBV_RX_HASH_FUNC_TOEPLITZ,
+				.rx_hash_key_len = rss_key_len ? rss_key_len :
+						   MLX5_RSS_HASH_KEY_LEN,
+				.rx_hash_key = rss_key ?
+					       (void *)(uintptr_t)rss_key :
+					       rss_hash_default_key,
+				.rx_hash_fields_mask = hash_fields,
+			},
+			.rwq_ind_tbl = ind_tbl->ind_table,
+			.pd = priv->pd,
+		 },
+		 &(struct mlx5dv_qp_init_attr){
+			.comp_mask = tunnel ?
+				MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS : 0,
+			.create_flags = MLX5DV_QP_CREATE_TUNNEL_OFFLOADS,
+		 });
+#else
+	qp = mlx5_glue->create_qp_ex
+		(priv->ctx,
+		 &(struct ibv_qp_init_attr_ex){
+			.qp_type = IBV_QPT_RAW_PACKET,
+			.comp_mask =
+				IBV_QP_INIT_ATTR_PD |
+				IBV_QP_INIT_ATTR_IND_TABLE |
+				IBV_QP_INIT_ATTR_RX_HASH,
+			.rx_hash_conf = (struct ibv_rx_hash_conf){
+				.rx_hash_function = IBV_RX_HASH_FUNC_TOEPLITZ,
+				.rx_hash_key_len = rss_key_len ? rss_key_len :
+						   MLX5_RSS_HASH_KEY_LEN,
+				.rx_hash_key = rss_key ?
+					       (void *)(uintptr_t)rss_key :
+					       rss_hash_default_key,
+				.rx_hash_fields_mask = hash_fields,
+			},
+			.rwq_ind_tbl = ind_tbl->ind_table,
+			.pd = priv->pd,
+		 });
+#endif
+	if (!qp) {
+		rte_errno = errno;
+		goto error;
+	}
+	hrxq = rte_calloc(__func__, 1, sizeof(*hrxq) + rss_key_len, 0);
+	if (!hrxq)
+		goto error;
+	hrxq->ind_table = ind_tbl;
+	hrxq->qp = qp;
+	hrxq->rss_key_len = rss_key_len;
+	hrxq->hash_fields = hash_fields;
+	memcpy(hrxq->rss_key, rss_key, rss_key_len);
+	rte_atomic32_inc(&hrxq->refcnt);
+	LIST_INSERT_HEAD(&priv->hrxqs, hrxq, next);
+	return hrxq;
+error:
+	err = rte_errno; /* Save rte_errno before cleanup. */
+	mlx5_ind_table_ibv_release(dev, ind_tbl);
+	if (qp)
+		claim_zero(mlx5_glue->destroy_qp(qp));
+	rte_errno = err; /* Restore rte_errno. */
+	return NULL;
+}
+
+/**
+ * Get an Rx Hash queue.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param rss_conf
+ *   RSS configuration for the Rx hash queue.
+ * @param queues
+ *   Queues entering in hash queue. In case of empty hash_fields only the
+ *   first queue index will be taken for the indirection table.
+ * @param queues_n
+ *   Number of queues.
+ *
+ * @return
+ *   An hash Rx queue on success.
+ */
+struct mlx5_hrxq *
+mlx5_hrxq_get(struct rte_eth_dev *dev,
+	      const uint8_t *rss_key, uint32_t rss_key_len,
+	      uint64_t hash_fields,
+	      const uint16_t *queues, uint32_t queues_n)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_hrxq *hrxq;
+
+	queues_n = hash_fields ? queues_n : 1;
+	LIST_FOREACH(hrxq, &priv->hrxqs, next) {
+		struct mlx5_ind_table_ibv *ind_tbl;
+
+		if (hrxq->rss_key_len != rss_key_len)
+			continue;
+		if (memcmp(hrxq->rss_key, rss_key, rss_key_len))
+			continue;
+		if (hrxq->hash_fields != hash_fields)
+			continue;
+		ind_tbl = mlx5_ind_table_ibv_get(dev, queues, queues_n);
+		if (!ind_tbl)
+			continue;
+		if (ind_tbl != hrxq->ind_table) {
+			mlx5_ind_table_ibv_release(dev, ind_tbl);
+			continue;
+		}
+		rte_atomic32_inc(&hrxq->refcnt);
+		return hrxq;
+	}
+	return NULL;
+}
+
+/**
+ * Release the hash Rx queue.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param hrxq
+ *   Pointer to Hash Rx queue to release.
+ *
+ * @return
+ *   1 while a reference on it exists, 0 when freed.
+ */
+int
+mlx5_hrxq_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq)
+{
+	if (rte_atomic32_dec_and_test(&hrxq->refcnt)) {
+		claim_zero(mlx5_glue->destroy_qp(hrxq->qp));
+		mlx5_ind_table_ibv_release(dev, hrxq->ind_table);
+		LIST_REMOVE(hrxq, next);
+		rte_free(hrxq);
+		return 0;
+	}
+	claim_nonzero(mlx5_ind_table_ibv_release(dev, hrxq->ind_table));
+	return 1;
+}
+
+/**
+ * Verify the Rx Queue list is empty
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   The number of object not released.
+ */
+int
+mlx5_hrxq_ibv_verify(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_hrxq *hrxq;
+	int ret = 0;
+
+	LIST_FOREACH(hrxq, &priv->hrxqs, next) {
+		DRV_LOG(DEBUG,
+			"port %u Verbs hash Rx queue %p still referenced",
+			dev->data->port_id, (void *)hrxq);
+		++ret;
+	}
+	return ret;
+}
+
+/**
+ * Create a drop Rx queue Verbs object.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   The Verbs object initialised, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_rxq_ibv *
+mlx5_rxq_ibv_drop_new(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct ibv_cq *cq;
+	struct ibv_wq *wq = NULL;
+	struct mlx5_rxq_ibv *rxq;
+
+	if (priv->drop_queue.rxq)
+		return priv->drop_queue.rxq;
+	cq = mlx5_glue->create_cq(priv->ctx, 1, NULL, NULL, 0);
+	if (!cq) {
+		DEBUG("port %u cannot allocate CQ for drop queue",
+		      dev->data->port_id);
+		rte_errno = errno;
+		goto error;
+	}
+	wq = mlx5_glue->create_wq(priv->ctx,
+		 &(struct ibv_wq_init_attr){
+			.wq_type = IBV_WQT_RQ,
+			.max_wr = 1,
+			.max_sge = 1,
+			.pd = priv->pd,
+			.cq = cq,
+		 });
+	if (!wq) {
+		DEBUG("port %u cannot allocate WQ for drop queue",
+		      dev->data->port_id);
+		rte_errno = errno;
+		goto error;
+	}
+	rxq = rte_calloc(__func__, 1, sizeof(*rxq), 0);
+	if (!rxq) {
+		DEBUG("port %u cannot allocate drop Rx queue memory",
+		      dev->data->port_id);
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	rxq->cq = cq;
+	rxq->wq = wq;
+	priv->drop_queue.rxq = rxq;
+	return rxq;
+error:
+	if (wq)
+		claim_zero(mlx5_glue->destroy_wq(wq));
+	if (cq)
+		claim_zero(mlx5_glue->destroy_cq(cq));
+	return NULL;
+}
+
+/**
+ * Release a drop Rx queue Verbs object.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   The Verbs object initialised, NULL otherwise and rte_errno is set.
+ */
+void
+mlx5_rxq_ibv_drop_release(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_rxq_ibv *rxq = priv->drop_queue.rxq;
+
+	if (rxq->wq)
+		claim_zero(mlx5_glue->destroy_wq(rxq->wq));
+	if (rxq->cq)
+		claim_zero(mlx5_glue->destroy_cq(rxq->cq));
+	rte_free(rxq);
+	priv->drop_queue.rxq = NULL;
+}
+
+/**
+ * Create a drop indirection table.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   The Verbs object initialised, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_ind_table_ibv *
+mlx5_ind_table_ibv_drop_new(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_ind_table_ibv *ind_tbl;
+	struct mlx5_rxq_ibv *rxq;
+	struct mlx5_ind_table_ibv tmpl;
+
+	rxq = mlx5_rxq_ibv_drop_new(dev);
+	if (!rxq)
+		return NULL;
+	tmpl.ind_table = mlx5_glue->create_rwq_ind_table
+		(priv->ctx,
+		 &(struct ibv_rwq_ind_table_init_attr){
+			.log_ind_tbl_size = 0,
+			.ind_tbl = &rxq->wq,
+			.comp_mask = 0,
+		 });
+	if (!tmpl.ind_table) {
+		DEBUG("port %u cannot allocate indirection table for drop"
+		      " queue",
+		      dev->data->port_id);
+		rte_errno = errno;
+		goto error;
+	}
+	ind_tbl = rte_calloc(__func__, 1, sizeof(*ind_tbl), 0);
+	if (!ind_tbl) {
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	ind_tbl->ind_table = tmpl.ind_table;
+	return ind_tbl;
+error:
+	mlx5_rxq_ibv_drop_release(dev);
+	return NULL;
+}
+
+/**
+ * Release a drop indirection table.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ */
+void
+mlx5_ind_table_ibv_drop_release(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_ind_table_ibv *ind_tbl = priv->drop_queue.hrxq->ind_table;
+
+	claim_zero(mlx5_glue->destroy_rwq_ind_table(ind_tbl->ind_table));
+	mlx5_rxq_ibv_drop_release(dev);
+	rte_free(ind_tbl);
+	priv->drop_queue.hrxq->ind_table = NULL;
+}
+
+/**
+ * Create a drop Rx Hash queue.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   The Verbs object initialised, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_hrxq *
+mlx5_hrxq_drop_new(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_ind_table_ibv *ind_tbl;
+	struct ibv_qp *qp;
+	struct mlx5_hrxq *hrxq;
+
+	if (priv->drop_queue.hrxq) {
+		rte_atomic32_inc(&priv->drop_queue.hrxq->refcnt);
+		return priv->drop_queue.hrxq;
+	}
+	ind_tbl = mlx5_ind_table_ibv_drop_new(dev);
+	if (!ind_tbl)
+		return NULL;
+	qp = mlx5_glue->create_qp_ex(priv->ctx,
+		 &(struct ibv_qp_init_attr_ex){
+			.qp_type = IBV_QPT_RAW_PACKET,
+			.comp_mask =
+				IBV_QP_INIT_ATTR_PD |
+				IBV_QP_INIT_ATTR_IND_TABLE |
+				IBV_QP_INIT_ATTR_RX_HASH,
+			.rx_hash_conf = (struct ibv_rx_hash_conf){
+				.rx_hash_function =
+					IBV_RX_HASH_FUNC_TOEPLITZ,
+				.rx_hash_key_len = MLX5_RSS_HASH_KEY_LEN,
+				.rx_hash_key = rss_hash_default_key,
+				.rx_hash_fields_mask = 0,
+				},
+			.rwq_ind_tbl = ind_tbl->ind_table,
+			.pd = priv->pd
+		 });
+	if (!qp) {
+		DEBUG("port %u cannot allocate QP for drop queue",
+		      dev->data->port_id);
+		rte_errno = errno;
+		goto error;
+	}
+	hrxq = rte_calloc(__func__, 1, sizeof(*hrxq), 0);
+	if (!hrxq) {
+		DRV_LOG(WARNING,
+			"port %u cannot allocate memory for drop queue",
+			dev->data->port_id);
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	hrxq->ind_table = ind_tbl;
+	hrxq->qp = qp;
+	priv->drop_queue.hrxq = hrxq;
+	rte_atomic32_set(&hrxq->refcnt, 1);
+	return hrxq;
+error:
+	if (ind_tbl)
+		mlx5_ind_table_ibv_drop_release(dev);
+	return NULL;
+}
+
+/**
+ * Release a drop hash Rx queue.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ */
+void
+mlx5_hrxq_drop_release(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_hrxq *hrxq = priv->drop_queue.hrxq;
+
+	if (rte_atomic32_dec_and_test(&hrxq->refcnt)) {
+		claim_zero(mlx5_glue->destroy_qp(hrxq->qp));
+		mlx5_ind_table_ibv_drop_release(dev);
+		rte_free(hrxq);
+		priv->drop_queue.hrxq = NULL;
+	}
+}
diff --git a/src/spdk/dpdk/drivers/net/mlx5/mlx5_rxtx.c b/src/spdk/dpdk/drivers/net/mlx5/mlx5_rxtx.c
new file mode 100644
index 00000000..2d14f8a6
--- /dev/null
+++ b/src/spdk/dpdk/drivers/net/mlx5/mlx5_rxtx.c
@@ -0,0 +1,2373 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2015 6WIND S.A.
+ * Copyright 2015 Mellanox Technologies, Ltd
+ */
+
+#include <assert.h>
+#include <stdint.h>
+#include <string.h>
+#include <stdlib.h>
+
+/* Verbs header. */
+/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <infiniband/verbs.h>
+#include <infiniband/mlx5dv.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+#include <rte_mbuf.h>
+#include <rte_mempool.h>
+#include <rte_prefetch.h>
+#include <rte_common.h>
+#include <rte_branch_prediction.h>
+#include <rte_ether.h>
+
+#include "mlx5.h"
+#include "mlx5_utils.h"
+#include "mlx5_rxtx.h"
+#include "mlx5_autoconf.h"
+#include "mlx5_defs.h"
+#include "mlx5_prm.h"
+
+static __rte_always_inline uint32_t
+rxq_cq_to_pkt_type(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe);
+
+static __rte_always_inline int
+mlx5_rx_poll_len(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe,
+		 uint16_t cqe_cnt, volatile struct mlx5_mini_cqe8 **mcqe);
+
+static __rte_always_inline uint32_t
+rxq_cq_to_ol_flags(volatile struct mlx5_cqe *cqe);
+
+static __rte_always_inline void
+rxq_cq_to_mbuf(struct mlx5_rxq_data *rxq, struct rte_mbuf *pkt,
+	       volatile struct mlx5_cqe *cqe, uint32_t rss_hash_res);
+
+static __rte_always_inline void
+mprq_buf_replace(struct mlx5_rxq_data *rxq, uint16_t rq_idx);
+
+uint32_t mlx5_ptype_table[] __rte_cache_aligned = {
+	[0xff] = RTE_PTYPE_ALL_MASK, /* Last entry for errored packet. */
+};
+
+uint8_t mlx5_cksum_table[1 << 10] __rte_cache_aligned;
+uint8_t mlx5_swp_types_table[1 << 10] __rte_cache_aligned;
+
+/**
+ * Build a table to translate Rx completion flags to packet type.
+ *
+ * @note: fix mlx5_dev_supported_ptypes_get() if any change here.
+ */
+void
+mlx5_set_ptype_table(void)
+{
+	unsigned int i;
+	uint32_t (*p)[RTE_DIM(mlx5_ptype_table)] = &mlx5_ptype_table;
+
+	/* Last entry must not be overwritten, reserved for errored packet. */
+	for (i = 0; i < RTE_DIM(mlx5_ptype_table) - 1; ++i)
+		(*p)[i] = RTE_PTYPE_UNKNOWN;
+	/*
+	 * The index to the array should have:
+	 * bit[1:0] = l3_hdr_type
+	 * bit[4:2] = l4_hdr_type
+	 * bit[5] = ip_frag
+	 * bit[6] = tunneled
+	 * bit[7] = outer_l3_type
+	 */
+	/* L2 */
+	(*p)[0x00] = RTE_PTYPE_L2_ETHER;
+	/* L3 */
+	(*p)[0x01] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_L4_NONFRAG;
+	(*p)[0x02] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_L4_NONFRAG;
+	/* Fragmented */
+	(*p)[0x21] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_L4_FRAG;
+	(*p)[0x22] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_L4_FRAG;
+	/* TCP */
+	(*p)[0x05] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_L4_TCP;
+	(*p)[0x06] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_L4_TCP;
+	(*p)[0x0d] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_L4_TCP;
+	(*p)[0x0e] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_L4_TCP;
+	(*p)[0x11] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_L4_TCP;
+	(*p)[0x12] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_L4_TCP;
+	/* UDP */
+	(*p)[0x09] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_L4_UDP;
+	(*p)[0x0a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_L4_UDP;
+	/* Repeat with outer_l3_type being set. Just in case. */
+	(*p)[0x81] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_L4_NONFRAG;
+	(*p)[0x82] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_L4_NONFRAG;
+	(*p)[0xa1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_L4_FRAG;
+	(*p)[0xa2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_L4_FRAG;
+	(*p)[0x85] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_L4_TCP;
+	(*p)[0x86] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_L4_TCP;
+	(*p)[0x8d] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_L4_TCP;
+	(*p)[0x8e] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_L4_TCP;
+	(*p)[0x91] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_L4_TCP;
+	(*p)[0x92] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_L4_TCP;
+	(*p)[0x89] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_L4_UDP;
+	(*p)[0x8a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_L4_UDP;
+	/* Tunneled - L3 */
+	(*p)[0x40] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
+	(*p)[0x41] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L4_NONFRAG;
+	(*p)[0x42] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L4_NONFRAG;
+	(*p)[0xc0] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN;
+	(*p)[0xc1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L4_NONFRAG;
+	(*p)[0xc2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L4_NONFRAG;
+	/* Tunneled - Fragmented */
+	(*p)[0x61] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L4_FRAG;
+	(*p)[0x62] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L4_FRAG;
+	(*p)[0xe1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L4_FRAG;
+	(*p)[0xe2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L4_FRAG;
+	/* Tunneled - TCP */
+	(*p)[0x45] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L4_TCP;
+	(*p)[0x46] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L4_TCP;
+	(*p)[0x4d] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L4_TCP;
+	(*p)[0x4e] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L4_TCP;
+	(*p)[0x51] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L4_TCP;
+	(*p)[0x52] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L4_TCP;
+	(*p)[0xc5] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L4_TCP;
+	(*p)[0xc6] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L4_TCP;
+	(*p)[0xcd] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L4_TCP;
+	(*p)[0xce] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L4_TCP;
+	(*p)[0xd1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L4_TCP;
+	(*p)[0xd2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L4_TCP;
+	/* Tunneled - UDP */
+	(*p)[0x49] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L4_UDP;
+	(*p)[0x4a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L4_UDP;
+	(*p)[0xc9] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L4_UDP;
+	(*p)[0xca] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L4_UDP;
+}
+
+/**
+ * Build a table to translate packet to checksum type of Verbs.
+ */
+void
+mlx5_set_cksum_table(void)
+{
+	unsigned int i;
+	uint8_t v;
+
+	/*
+	 * The index should have:
+	 * bit[0] = PKT_TX_TCP_SEG
+	 * bit[2:3] = PKT_TX_UDP_CKSUM, PKT_TX_TCP_CKSUM
+	 * bit[4] = PKT_TX_IP_CKSUM
+	 * bit[8] = PKT_TX_OUTER_IP_CKSUM
+	 * bit[9] = tunnel
+	 */
+	for (i = 0; i < RTE_DIM(mlx5_cksum_table); ++i) {
+		v = 0;
+		if (i & (1 << 9)) {
+			/* Tunneled packet. */
+			if (i & (1 << 8)) /* Outer IP. */
+				v |= MLX5_ETH_WQE_L3_CSUM;
+			if (i & (1 << 4)) /* Inner IP. */
+				v |= MLX5_ETH_WQE_L3_INNER_CSUM;
+			if (i & (3 << 2 | 1 << 0)) /* L4 or TSO. */
+				v |= MLX5_ETH_WQE_L4_INNER_CSUM;
+		} else {
+			/* No tunnel. */
+			if (i & (1 << 4)) /* IP. */
+				v |= MLX5_ETH_WQE_L3_CSUM;
+			if (i & (3 << 2 | 1 << 0)) /* L4 or TSO. */
+				v |= MLX5_ETH_WQE_L4_CSUM;
+		}
+		mlx5_cksum_table[i] = v;
+	}
+}
+
+/**
+ * Build a table to translate packet type of mbuf to SWP type of Verbs.
+ */
+void
+mlx5_set_swp_types_table(void)
+{
+	unsigned int i;
+	uint8_t v;
+
+	/*
+	 * The index should have:
+	 * bit[0:1] = PKT_TX_L4_MASK
+	 * bit[4] = PKT_TX_IPV6
+	 * bit[8] = PKT_TX_OUTER_IPV6
+	 * bit[9] = PKT_TX_OUTER_UDP
+	 */
+	for (i = 0; i < RTE_DIM(mlx5_swp_types_table); ++i) {
+		v = 0;
+		if (i & (1 << 8))
+			v |= MLX5_ETH_WQE_L3_OUTER_IPV6;
+		if (i & (1 << 9))
+			v |= MLX5_ETH_WQE_L4_OUTER_UDP;
+		if (i & (1 << 4))
+			v |= MLX5_ETH_WQE_L3_INNER_IPV6;
+		if ((i & 3) == (PKT_TX_UDP_CKSUM >> 52))
+			v |= MLX5_ETH_WQE_L4_INNER_UDP;
+		mlx5_swp_types_table[i] = v;
+	}
+}
+
+/**
+ * Return the size of tailroom of WQ.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param addr
+ *   Pointer to tail of WQ.
+ *
+ * @return
+ *   Size of tailroom.
+ */
+static inline size_t
+tx_mlx5_wq_tailroom(struct mlx5_txq_data *txq, void *addr)
+{
+	size_t tailroom;
+	tailroom = (uintptr_t)(txq->wqes) +
+		   (1 << txq->wqe_n) * MLX5_WQE_SIZE -
+		   (uintptr_t)addr;
+	return tailroom;
+}
+
+/**
+ * Copy data to tailroom of circular queue.
+ *
+ * @param dst
+ *   Pointer to destination.
+ * @param src
+ *   Pointer to source.
+ * @param n
+ *   Number of bytes to copy.
+ * @param base
+ *   Pointer to head of queue.
+ * @param tailroom
+ *   Size of tailroom from dst.
+ *
+ * @return
+ *   Pointer after copied data.
+ */
+static inline void *
+mlx5_copy_to_wq(void *dst, const void *src, size_t n,
+		void *base, size_t tailroom)
+{
+	void *ret;
+
+	if (n > tailroom) {
+		rte_memcpy(dst, src, tailroom);
+		rte_memcpy(base, (void *)((uintptr_t)src + tailroom),
+			   n - tailroom);
+		ret = (uint8_t *)base + n - tailroom;
+	} else {
+		rte_memcpy(dst, src, n);
+		ret = (n == tailroom) ? base : (uint8_t *)dst + n;
+	}
+	return ret;
+}
+
+/**
+ * Inline TSO headers into WQE.
+ *
+ * @return
+ *   0 on success, negative errno value on failure.
+ */
+static int
+inline_tso(struct mlx5_txq_data *txq, struct rte_mbuf *buf,
+	   uint32_t *length,
+	   uintptr_t *addr,
+	   uint16_t *pkt_inline_sz,
+	   uint8_t **raw,
+	   uint16_t *max_wqe,
+	   uint16_t *tso_segsz,
+	   uint16_t *tso_header_sz)
+{
+	uintptr_t end = (uintptr_t)(((uintptr_t)txq->wqes) +
+				    (1 << txq->wqe_n) * MLX5_WQE_SIZE);
+	unsigned int copy_b;
+	uint8_t vlan_sz = (buf->ol_flags & PKT_TX_VLAN_PKT) ? 4 : 0;
+	const uint8_t tunneled = txq->tunnel_en && (buf->ol_flags &
+				 PKT_TX_TUNNEL_MASK);
+	uint16_t n_wqe;
+
+	*tso_segsz = buf->tso_segsz;
+	*tso_header_sz = buf->l2_len + vlan_sz + buf->l3_len + buf->l4_len;
+	if (unlikely(*tso_segsz == 0 || *tso_header_sz == 0)) {
+		txq->stats.oerrors++;
+		return -EINVAL;
+	}
+	if (tunneled)
+		*tso_header_sz += buf->outer_l2_len + buf->outer_l3_len;
+	/* First seg must contain all TSO headers. */
+	if (unlikely(*tso_header_sz > MLX5_MAX_TSO_HEADER) ||
+		     *tso_header_sz > DATA_LEN(buf)) {
+		txq->stats.oerrors++;
+		return -EINVAL;
+	}
+	copy_b = *tso_header_sz - *pkt_inline_sz;
+	if (!copy_b || ((end - (uintptr_t)*raw) < copy_b))
+		return -EAGAIN;
+	n_wqe = (MLX5_WQE_DS(copy_b) - 1 + 3) / 4;
+	if (unlikely(*max_wqe < n_wqe))
+		return -EINVAL;
+	*max_wqe -= n_wqe;
+	rte_memcpy((void *)*raw, (void *)*addr, copy_b);
+	*length -= copy_b;
+	*addr += copy_b;
+	copy_b = MLX5_WQE_DS(copy_b) * MLX5_WQE_DWORD_SIZE;
+	*pkt_inline_sz += copy_b;
+	*raw += copy_b;
+	return 0;
+}
+
+/**
+ * DPDK callback to check the status of a tx descriptor.
+ *
+ * @param tx_queue
+ *   The tx queue.
+ * @param[in] offset
+ *   The index of the descriptor in the ring.
+ *
+ * @return
+ *   The status of the tx descriptor.
+ */
+int
+mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset)
+{
+	struct mlx5_txq_data *txq = tx_queue;
+	uint16_t used;
+
+	mlx5_tx_complete(txq);
+	used = txq->elts_head - txq->elts_tail;
+	if (offset < used)
+		return RTE_ETH_TX_DESC_FULL;
+	return RTE_ETH_TX_DESC_DONE;
+}
+
+/**
+ * DPDK callback to check the status of a rx descriptor.
+ *
+ * @param rx_queue
+ *   The rx queue.
+ * @param[in] offset
+ *   The index of the descriptor in the ring.
+ *
+ * @return
+ *   The status of the tx descriptor.
+ */
+int
+mlx5_rx_descriptor_status(void *rx_queue, uint16_t offset)
+{
+	struct mlx5_rxq_data *rxq = rx_queue;
+	struct rxq_zip *zip = &rxq->zip;
+	volatile struct mlx5_cqe *cqe;
+	const unsigned int cqe_n = (1 << rxq->cqe_n);
+	const unsigned int cqe_cnt = cqe_n - 1;
+	unsigned int cq_ci;
+	unsigned int used;
+
+	/* if we are processing a compressed cqe */
+	if (zip->ai) {
+		used = zip->cqe_cnt - zip->ca;
+		cq_ci = zip->cq_ci;
+	} else {
+		used = 0;
+		cq_ci = rxq->cq_ci;
+	}
+	cqe = &(*rxq->cqes)[cq_ci & cqe_cnt];
+	while (check_cqe(cqe, cqe_n, cq_ci) == 0) {
+		int8_t op_own;
+		unsigned int n;
+
+		op_own = cqe->op_own;
+		if (MLX5_CQE_FORMAT(op_own) == MLX5_COMPRESSED)
+			n = rte_be_to_cpu_32(cqe->byte_cnt);
+		else
+			n = 1;
+		cq_ci += n;
+		used += n;
+		cqe = &(*rxq->cqes)[cq_ci & cqe_cnt];
+	}
+	used = RTE_MIN(used, (1U << rxq->elts_n) - 1);
+	if (offset < used)
+		return RTE_ETH_RX_DESC_DONE;
+	return RTE_ETH_RX_DESC_AVAIL;
+}
+
+/**
+ * DPDK callback for TX.
+ *
+ * @param dpdk_txq
+ *   Generic pointer to TX queue structure.
+ * @param[in] pkts
+ *   Packets to transmit.
+ * @param pkts_n
+ *   Number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully transmitted (<= pkts_n).
+ */
+uint16_t
+mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq;
+	uint16_t elts_head = txq->elts_head;
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
+	unsigned int i = 0;
+	unsigned int j = 0;
+	unsigned int k = 0;
+	uint16_t max_elts;
+	uint16_t max_wqe;
+	unsigned int comp;
+	volatile struct mlx5_wqe_ctrl *last_wqe = NULL;
+	unsigned int segs_n = 0;
+	const unsigned int max_inline = txq->max_inline;
+	uint64_t addr_64;
+
+	if (unlikely(!pkts_n))
+		return 0;
+	/* Prefetch first packet cacheline. */
+	rte_prefetch0(*pkts);
+	/* Start processing. */
+	mlx5_tx_complete(txq);
+	max_elts = (elts_n - (elts_head - txq->elts_tail));
+	max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
+	if (unlikely(!max_wqe))
+		return 0;
+	do {
+		struct rte_mbuf *buf = *pkts; /* First_seg. */
+		uint8_t *raw;
+		volatile struct mlx5_wqe_v *wqe = NULL;
+		volatile rte_v128u32_t *dseg = NULL;
+		uint32_t length;
+		unsigned int ds = 0;
+		unsigned int sg = 0; /* counter of additional segs attached. */
+		uintptr_t addr;
+		uint16_t pkt_inline_sz = MLX5_WQE_DWORD_SIZE + 2;
+		uint16_t tso_header_sz = 0;
+		uint16_t ehdr;
+		uint8_t cs_flags;
+		uint8_t tso = txq->tso_en && (buf->ol_flags & PKT_TX_TCP_SEG);
+		uint32_t swp_offsets = 0;
+		uint8_t swp_types = 0;
+		uint16_t tso_segsz = 0;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		uint32_t total_length = 0;
+#endif
+		int ret;
+
+		segs_n = buf->nb_segs;
+		/*
+		 * Make sure there is enough room to store this packet and
+		 * that one ring entry remains unused.
+		 */
+		assert(segs_n);
+		if (max_elts < segs_n)
+			break;
+		max_elts -= segs_n;
+		sg = --segs_n;
+		if (unlikely(--max_wqe == 0))
+			break;
+		wqe = (volatile struct mlx5_wqe_v *)
+			tx_mlx5_wqe(txq, txq->wqe_ci);
+		rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci + 1));
+		if (pkts_n - i > 1)
+			rte_prefetch0(*(pkts + 1));
+		addr = rte_pktmbuf_mtod(buf, uintptr_t);
+		length = DATA_LEN(buf);
+		ehdr = (((uint8_t *)addr)[1] << 8) |
+		       ((uint8_t *)addr)[0];
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		total_length = length;
+#endif
+		if (length < (MLX5_WQE_DWORD_SIZE + 2)) {
+			txq->stats.oerrors++;
+			break;
+		}
+		/* Update element. */
+		(*txq->elts)[elts_head & elts_m] = buf;
+		/* Prefetch next buffer data. */
+		if (pkts_n - i > 1)
+			rte_prefetch0(
+			    rte_pktmbuf_mtod(*(pkts + 1), volatile void *));
+		cs_flags = txq_ol_cksum_to_cs(buf);
+		txq_mbuf_to_swp(txq, buf, (uint8_t *)&swp_offsets, &swp_types);
+		raw = ((uint8_t *)(uintptr_t)wqe) + 2 * MLX5_WQE_DWORD_SIZE;
+		/* Replace the Ethernet type by the VLAN if necessary. */
+		if (buf->ol_flags & PKT_TX_VLAN_PKT) {
+			uint32_t vlan = rte_cpu_to_be_32(0x81000000 |
+							 buf->vlan_tci);
+			unsigned int len = 2 * ETHER_ADDR_LEN - 2;
+
+			addr += 2;
+			length -= 2;
+			/* Copy Destination and source mac address. */
+			memcpy((uint8_t *)raw, ((uint8_t *)addr), len);
+			/* Copy VLAN. */
+			memcpy((uint8_t *)raw + len, &vlan, sizeof(vlan));
+			/* Copy missing two bytes to end the DSeg. */
+			memcpy((uint8_t *)raw + len + sizeof(vlan),
+			       ((uint8_t *)addr) + len, 2);
+			addr += len + 2;
+			length -= (len + 2);
+		} else {
+			memcpy((uint8_t *)raw, ((uint8_t *)addr) + 2,
+			       MLX5_WQE_DWORD_SIZE);
+			length -= pkt_inline_sz;
+			addr += pkt_inline_sz;
+		}
+		raw += MLX5_WQE_DWORD_SIZE;
+		if (tso) {
+			ret = inline_tso(txq, buf, &length,
+					 &addr, &pkt_inline_sz,
+					 &raw, &max_wqe,
+					 &tso_segsz, &tso_header_sz);
+			if (ret == -EINVAL) {
+				break;
+			} else if (ret == -EAGAIN) {
+				/* NOP WQE. */
+				wqe->ctrl = (rte_v128u32_t){
+					rte_cpu_to_be_32(txq->wqe_ci << 8),
+					rte_cpu_to_be_32(txq->qp_num_8s | 1),
+					0,
+					0,
+				};
+				ds = 1;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+				total_length = 0;
+#endif
+				k++;
+				goto next_wqe;
+			}
+		}
+		/* Inline if enough room. */
+		if (max_inline || tso) {
+			uint32_t inl = 0;
+			uintptr_t end = (uintptr_t)
+				(((uintptr_t)txq->wqes) +
+				 (1 << txq->wqe_n) * MLX5_WQE_SIZE);
+			unsigned int inline_room = max_inline *
+						   RTE_CACHE_LINE_SIZE -
+						   (pkt_inline_sz - 2) -
+						   !!tso * sizeof(inl);
+			uintptr_t addr_end;
+			unsigned int copy_b;
+
+pkt_inline:
+			addr_end = RTE_ALIGN_FLOOR(addr + inline_room,
+						   RTE_CACHE_LINE_SIZE);
+			copy_b = (addr_end > addr) ?
+				 RTE_MIN((addr_end - addr), length) : 0;
+			if (copy_b && ((end - (uintptr_t)raw) > copy_b)) {
+				/*
+				 * One Dseg remains in the current WQE.  To
+				 * keep the computation positive, it is
+				 * removed after the bytes to Dseg conversion.
+				 */
+				uint16_t n = (MLX5_WQE_DS(copy_b) - 1 + 3) / 4;
+
+				if (unlikely(max_wqe < n))
+					break;
+				max_wqe -= n;
+				if (tso) {
+					assert(inl == 0);
+					inl = rte_cpu_to_be_32(copy_b |
+							       MLX5_INLINE_SEG);
+					rte_memcpy((void *)raw,
+						   (void *)&inl, sizeof(inl));
+					raw += sizeof(inl);
+					pkt_inline_sz += sizeof(inl);
+				}
+				rte_memcpy((void *)raw, (void *)addr, copy_b);
+				addr += copy_b;
+				length -= copy_b;
+				pkt_inline_sz += copy_b;
+			}
+			/*
+			 * 2 DWORDs consumed by the WQE header + ETH segment +
+			 * the size of the inline part of the packet.
+			 */
+			ds = 2 + MLX5_WQE_DS(pkt_inline_sz - 2);
+			if (length > 0) {
+				if (ds % (MLX5_WQE_SIZE /
+					  MLX5_WQE_DWORD_SIZE) == 0) {
+					if (unlikely(--max_wqe == 0))
+						break;
+					dseg = (volatile rte_v128u32_t *)
+					       tx_mlx5_wqe(txq, txq->wqe_ci +
+							   ds / 4);
+				} else {
+					dseg = (volatile rte_v128u32_t *)
+						((uintptr_t)wqe +
+						 (ds * MLX5_WQE_DWORD_SIZE));
+				}
+				goto use_dseg;
+			} else if (!segs_n) {
+				goto next_pkt;
+			} else {
+				/*
+				 * Further inline the next segment only for
+				 * non-TSO packets.
+				 */
+				if (!tso) {
+					raw += copy_b;
+					inline_room -= copy_b;
+				} else {
+					inline_room = 0;
+				}
+				/* Move to the next segment. */
+				--segs_n;
+				buf = buf->next;
+				assert(buf);
+				addr = rte_pktmbuf_mtod(buf, uintptr_t);
+				length = DATA_LEN(buf);
+#ifdef MLX5_PMD_SOFT_COUNTERS
+				total_length += length;
+#endif
+				(*txq->elts)[++elts_head & elts_m] = buf;
+				goto pkt_inline;
+			}
+		} else {
+			/*
+			 * No inline has been done in the packet, only the
+			 * Ethernet Header as been stored.
+			 */
+			dseg = (volatile rte_v128u32_t *)
+				((uintptr_t)wqe + (3 * MLX5_WQE_DWORD_SIZE));
+			ds = 3;
+use_dseg:
+			/* Add the remaining packet as a simple ds. */
+			addr_64 = rte_cpu_to_be_64(addr);
+			*dseg = (rte_v128u32_t){
+				rte_cpu_to_be_32(length),
+				mlx5_tx_mb2mr(txq, buf),
+				addr_64,
+				addr_64 >> 32,
+			};
+			++ds;
+			if (!segs_n)
+				goto next_pkt;
+		}
+next_seg:
+		assert(buf);
+		assert(ds);
+		assert(wqe);
+		/*
+		 * Spill on next WQE when the current one does not have
+		 * enough room left. Size of WQE must a be a multiple
+		 * of data segment size.
+		 */
+		assert(!(MLX5_WQE_SIZE % MLX5_WQE_DWORD_SIZE));
+		if (!(ds % (MLX5_WQE_SIZE / MLX5_WQE_DWORD_SIZE))) {
+			if (unlikely(--max_wqe == 0))
+				break;
+			dseg = (volatile rte_v128u32_t *)
+			       tx_mlx5_wqe(txq, txq->wqe_ci + ds / 4);
+			rte_prefetch0(tx_mlx5_wqe(txq,
+						  txq->wqe_ci + ds / 4 + 1));
+		} else {
+			++dseg;
+		}
+		++ds;
+		buf = buf->next;
+		assert(buf);
+		length = DATA_LEN(buf);
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		total_length += length;
+#endif
+		/* Store segment information. */
+		addr_64 = rte_cpu_to_be_64(rte_pktmbuf_mtod(buf, uintptr_t));
+		*dseg = (rte_v128u32_t){
+			rte_cpu_to_be_32(length),
+			mlx5_tx_mb2mr(txq, buf),
+			addr_64,
+			addr_64 >> 32,
+		};
+		(*txq->elts)[++elts_head & elts_m] = buf;
+		if (--segs_n)
+			goto next_seg;
+next_pkt:
+		if (ds > MLX5_DSEG_MAX) {
+			txq->stats.oerrors++;
+			break;
+		}
+		++elts_head;
+		++pkts;
+		++i;
+		j += sg;
+		/* Initialize known and common part of the WQE structure. */
+		if (tso) {
+			wqe->ctrl = (rte_v128u32_t){
+				rte_cpu_to_be_32((txq->wqe_ci << 8) |
+						 MLX5_OPCODE_TSO),
+				rte_cpu_to_be_32(txq->qp_num_8s | ds),
+				0,
+				0,
+			};
+			wqe->eseg = (rte_v128u32_t){
+				swp_offsets,
+				cs_flags | (swp_types << 8) |
+				(rte_cpu_to_be_16(tso_segsz) << 16),
+				0,
+				(ehdr << 16) | rte_cpu_to_be_16(tso_header_sz),
+			};
+		} else {
+			wqe->ctrl = (rte_v128u32_t){
+				rte_cpu_to_be_32((txq->wqe_ci << 8) |
+						 MLX5_OPCODE_SEND),
+				rte_cpu_to_be_32(txq->qp_num_8s | ds),
+				0,
+				0,
+			};
+			wqe->eseg = (rte_v128u32_t){
+				swp_offsets,
+				cs_flags | (swp_types << 8),
+				0,
+				(ehdr << 16) | rte_cpu_to_be_16(pkt_inline_sz),
+			};
+		}
+next_wqe:
+		txq->wqe_ci += (ds + 3) / 4;
+		/* Save the last successful WQE for completion request */
+		last_wqe = (volatile struct mlx5_wqe_ctrl *)wqe;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		/* Increment sent bytes counter. */
+		txq->stats.obytes += total_length;
+#endif
+	} while (i < pkts_n);
+	/* Take a shortcut if nothing must be sent. */
+	if (unlikely((i + k) == 0))
+		return 0;
+	txq->elts_head += (i + j);
+	/* Check whether completion threshold has been reached. */
+	comp = txq->elts_comp + i + j + k;
+	if (comp >= MLX5_TX_COMP_THRESH) {
+		/* A CQE slot must always be available. */
+		assert((1u << txq->cqe_n) - (txq->cq_pi++ - txq->cq_ci));
+		/* Request completion on last WQE. */
+		last_wqe->ctrl2 = rte_cpu_to_be_32(8);
+		/* Save elts_head in unused "immediate" field of WQE. */
+		last_wqe->ctrl3 = txq->elts_head;
+		txq->elts_comp = 0;
+	} else {
+		txq->elts_comp = comp;
+	}
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	/* Increment sent packets counter. */
+	txq->stats.opackets += i;
+#endif
+	/* Ring QP doorbell. */
+	mlx5_tx_dbrec(txq, (volatile struct mlx5_wqe *)last_wqe);
+	return i;
+}
+
+/**
+ * Open a MPW session.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param mpw
+ *   Pointer to MPW session structure.
+ * @param length
+ *   Packet length.
+ */
+static inline void
+mlx5_mpw_new(struct mlx5_txq_data *txq, struct mlx5_mpw *mpw, uint32_t length)
+{
+	uint16_t idx = txq->wqe_ci & ((1 << txq->wqe_n) - 1);
+	volatile struct mlx5_wqe_data_seg (*dseg)[MLX5_MPW_DSEG_MAX] =
+		(volatile struct mlx5_wqe_data_seg (*)[])
+		tx_mlx5_wqe(txq, idx + 1);
+
+	mpw->state = MLX5_MPW_STATE_OPENED;
+	mpw->pkts_n = 0;
+	mpw->len = length;
+	mpw->total_len = 0;
+	mpw->wqe = (volatile struct mlx5_wqe *)tx_mlx5_wqe(txq, idx);
+	mpw->wqe->eseg.mss = rte_cpu_to_be_16(length);
+	mpw->wqe->eseg.inline_hdr_sz = 0;
+	mpw->wqe->eseg.rsvd0 = 0;
+	mpw->wqe->eseg.rsvd1 = 0;
+	mpw->wqe->eseg.rsvd2 = 0;
+	mpw->wqe->ctrl[0] = rte_cpu_to_be_32((MLX5_OPC_MOD_MPW << 24) |
+					     (txq->wqe_ci << 8) |
+					     MLX5_OPCODE_TSO);
+	mpw->wqe->ctrl[2] = 0;
+	mpw->wqe->ctrl[3] = 0;
+	mpw->data.dseg[0] = (volatile struct mlx5_wqe_data_seg *)
+		(((uintptr_t)mpw->wqe) + (2 * MLX5_WQE_DWORD_SIZE));
+	mpw->data.dseg[1] = (volatile struct mlx5_wqe_data_seg *)
+		(((uintptr_t)mpw->wqe) + (3 * MLX5_WQE_DWORD_SIZE));
+	mpw->data.dseg[2] = &(*dseg)[0];
+	mpw->data.dseg[3] = &(*dseg)[1];
+	mpw->data.dseg[4] = &(*dseg)[2];
+}
+
+/**
+ * Close a MPW session.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param mpw
+ *   Pointer to MPW session structure.
+ */
+static inline void
+mlx5_mpw_close(struct mlx5_txq_data *txq, struct mlx5_mpw *mpw)
+{
+	unsigned int num = mpw->pkts_n;
+
+	/*
+	 * Store size in multiple of 16 bytes. Control and Ethernet segments
+	 * count as 2.
+	 */
+	mpw->wqe->ctrl[1] = rte_cpu_to_be_32(txq->qp_num_8s | (2 + num));
+	mpw->state = MLX5_MPW_STATE_CLOSED;
+	if (num < 3)
+		++txq->wqe_ci;
+	else
+		txq->wqe_ci += 2;
+	rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci));
+	rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci + 1));
+}
+
+/**
+ * DPDK callback for TX with MPW support.
+ *
+ * @param dpdk_txq
+ *   Generic pointer to TX queue structure.
+ * @param[in] pkts
+ *   Packets to transmit.
+ * @param pkts_n
+ *   Number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully transmitted (<= pkts_n).
+ */
+uint16_t
+mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq;
+	uint16_t elts_head = txq->elts_head;
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
+	unsigned int i = 0;
+	unsigned int j = 0;
+	uint16_t max_elts;
+	uint16_t max_wqe;
+	unsigned int comp;
+	struct mlx5_mpw mpw = {
+		.state = MLX5_MPW_STATE_CLOSED,
+	};
+
+	if (unlikely(!pkts_n))
+		return 0;
+	/* Prefetch first packet cacheline. */
+	rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci));
+	rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci + 1));
+	/* Start processing. */
+	mlx5_tx_complete(txq);
+	max_elts = (elts_n - (elts_head - txq->elts_tail));
+	max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
+	if (unlikely(!max_wqe))
+		return 0;
+	do {
+		struct rte_mbuf *buf = *(pkts++);
+		uint32_t length;
+		unsigned int segs_n = buf->nb_segs;
+		uint32_t cs_flags;
+
+		/*
+		 * Make sure there is enough room to store this packet and
+		 * that one ring entry remains unused.
+		 */
+		assert(segs_n);
+		if (max_elts < segs_n)
+			break;
+		/* Do not bother with large packets MPW cannot handle. */
+		if (segs_n > MLX5_MPW_DSEG_MAX) {
+			txq->stats.oerrors++;
+			break;
+		}
+		max_elts -= segs_n;
+		--pkts_n;
+		cs_flags = txq_ol_cksum_to_cs(buf);
+		/* Retrieve packet information. */
+		length = PKT_LEN(buf);
+		assert(length);
+		/* Start new session if packet differs. */
+		if ((mpw.state == MLX5_MPW_STATE_OPENED) &&
+		    ((mpw.len != length) ||
+		     (segs_n != 1) ||
+		     (mpw.wqe->eseg.cs_flags != cs_flags)))
+			mlx5_mpw_close(txq, &mpw);
+		if (mpw.state == MLX5_MPW_STATE_CLOSED) {
+			/*
+			 * Multi-Packet WQE consumes at most two WQE.
+			 * mlx5_mpw_new() expects to be able to use such
+			 * resources.
+			 */
+			if (unlikely(max_wqe < 2))
+				break;
+			max_wqe -= 2;
+			mlx5_mpw_new(txq, &mpw, length);
+			mpw.wqe->eseg.cs_flags = cs_flags;
+		}
+		/* Multi-segment packets must be alone in their MPW. */
+		assert((segs_n == 1) || (mpw.pkts_n == 0));
+#if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
+		length = 0;
+#endif
+		do {
+			volatile struct mlx5_wqe_data_seg *dseg;
+			uintptr_t addr;
+
+			assert(buf);
+			(*txq->elts)[elts_head++ & elts_m] = buf;
+			dseg = mpw.data.dseg[mpw.pkts_n];
+			addr = rte_pktmbuf_mtod(buf, uintptr_t);
+			*dseg = (struct mlx5_wqe_data_seg){
+				.byte_count = rte_cpu_to_be_32(DATA_LEN(buf)),
+				.lkey = mlx5_tx_mb2mr(txq, buf),
+				.addr = rte_cpu_to_be_64(addr),
+			};
+#if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
+			length += DATA_LEN(buf);
+#endif
+			buf = buf->next;
+			++mpw.pkts_n;
+			++j;
+		} while (--segs_n);
+		assert(length == mpw.len);
+		if (mpw.pkts_n == MLX5_MPW_DSEG_MAX)
+			mlx5_mpw_close(txq, &mpw);
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		/* Increment sent bytes counter. */
+		txq->stats.obytes += length;
+#endif
+		++i;
+	} while (pkts_n);
+	/* Take a shortcut if nothing must be sent. */
+	if (unlikely(i == 0))
+		return 0;
+	/* Check whether completion threshold has been reached. */
+	/* "j" includes both packets and segments. */
+	comp = txq->elts_comp + j;
+	if (comp >= MLX5_TX_COMP_THRESH) {
+		volatile struct mlx5_wqe *wqe = mpw.wqe;
+
+		/* A CQE slot must always be available. */
+		assert((1u << txq->cqe_n) - (txq->cq_pi++ - txq->cq_ci));
+		/* Request completion on last WQE. */
+		wqe->ctrl[2] = rte_cpu_to_be_32(8);
+		/* Save elts_head in unused "immediate" field of WQE. */
+		wqe->ctrl[3] = elts_head;
+		txq->elts_comp = 0;
+	} else {
+		txq->elts_comp = comp;
+	}
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	/* Increment sent packets counter. */
+	txq->stats.opackets += i;
+#endif
+	/* Ring QP doorbell. */
+	if (mpw.state == MLX5_MPW_STATE_OPENED)
+		mlx5_mpw_close(txq, &mpw);
+	mlx5_tx_dbrec(txq, mpw.wqe);
+	txq->elts_head = elts_head;
+	return i;
+}
+
+/**
+ * Open a MPW inline session.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param mpw
+ *   Pointer to MPW session structure.
+ * @param length
+ *   Packet length.
+ */
+static inline void
+mlx5_mpw_inline_new(struct mlx5_txq_data *txq, struct mlx5_mpw *mpw,
+		    uint32_t length)
+{
+	uint16_t idx = txq->wqe_ci & ((1 << txq->wqe_n) - 1);
+	struct mlx5_wqe_inl_small *inl;
+
+	mpw->state = MLX5_MPW_INL_STATE_OPENED;
+	mpw->pkts_n = 0;
+	mpw->len = length;
+	mpw->total_len = 0;
+	mpw->wqe = (volatile struct mlx5_wqe *)tx_mlx5_wqe(txq, idx);
+	mpw->wqe->ctrl[0] = rte_cpu_to_be_32((MLX5_OPC_MOD_MPW << 24) |
+					     (txq->wqe_ci << 8) |
+					     MLX5_OPCODE_TSO);
+	mpw->wqe->ctrl[2] = 0;
+	mpw->wqe->ctrl[3] = 0;
+	mpw->wqe->eseg.mss = rte_cpu_to_be_16(length);
+	mpw->wqe->eseg.inline_hdr_sz = 0;
+	mpw->wqe->eseg.cs_flags = 0;
+	mpw->wqe->eseg.rsvd0 = 0;
+	mpw->wqe->eseg.rsvd1 = 0;
+	mpw->wqe->eseg.rsvd2 = 0;
+	inl = (struct mlx5_wqe_inl_small *)
+		(((uintptr_t)mpw->wqe) + 2 * MLX5_WQE_DWORD_SIZE);
+	mpw->data.raw = (uint8_t *)&inl->raw;
+}
+
+/**
+ * Close a MPW inline session.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param mpw
+ *   Pointer to MPW session structure.
+ */
+static inline void
+mlx5_mpw_inline_close(struct mlx5_txq_data *txq, struct mlx5_mpw *mpw)
+{
+	unsigned int size;
+	struct mlx5_wqe_inl_small *inl = (struct mlx5_wqe_inl_small *)
+		(((uintptr_t)mpw->wqe) + (2 * MLX5_WQE_DWORD_SIZE));
+
+	size = MLX5_WQE_SIZE - MLX5_MWQE64_INL_DATA + mpw->total_len;
+	/*
+	 * Store size in multiple of 16 bytes. Control and Ethernet segments
+	 * count as 2.
+	 */
+	mpw->wqe->ctrl[1] = rte_cpu_to_be_32(txq->qp_num_8s |
+					     MLX5_WQE_DS(size));
+	mpw->state = MLX5_MPW_STATE_CLOSED;
+	inl->byte_cnt = rte_cpu_to_be_32(mpw->total_len | MLX5_INLINE_SEG);
+	txq->wqe_ci += (size + (MLX5_WQE_SIZE - 1)) / MLX5_WQE_SIZE;
+}
+
+/**
+ * DPDK callback for TX with MPW inline support.
+ *
+ * @param dpdk_txq
+ *   Generic pointer to TX queue structure.
+ * @param[in] pkts
+ *   Packets to transmit.
+ * @param pkts_n
+ *   Number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully transmitted (<= pkts_n).
+ */
+uint16_t
+mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
+			 uint16_t pkts_n)
+{
+	struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq;
+	uint16_t elts_head = txq->elts_head;
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
+	unsigned int i = 0;
+	unsigned int j = 0;
+	uint16_t max_elts;
+	uint16_t max_wqe;
+	unsigned int comp;
+	unsigned int inline_room = txq->max_inline * RTE_CACHE_LINE_SIZE;
+	struct mlx5_mpw mpw = {
+		.state = MLX5_MPW_STATE_CLOSED,
+	};
+	/*
+	 * Compute the maximum number of WQE which can be consumed by inline
+	 * code.
+	 * - 2 DSEG for:
+	 *   - 1 control segment,
+	 *   - 1 Ethernet segment,
+	 * - N Dseg from the inline request.
+	 */
+	const unsigned int wqe_inl_n =
+		((2 * MLX5_WQE_DWORD_SIZE +
+		  txq->max_inline * RTE_CACHE_LINE_SIZE) +
+		 RTE_CACHE_LINE_SIZE - 1) / RTE_CACHE_LINE_SIZE;
+
+	if (unlikely(!pkts_n))
+		return 0;
+	/* Prefetch first packet cacheline. */
+	rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci));
+	rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci + 1));
+	/* Start processing. */
+	mlx5_tx_complete(txq);
+	max_elts = (elts_n - (elts_head - txq->elts_tail));
+	do {
+		struct rte_mbuf *buf = *(pkts++);
+		uintptr_t addr;
+		uint32_t length;
+		unsigned int segs_n = buf->nb_segs;
+		uint8_t cs_flags;
+
+		/*
+		 * Make sure there is enough room to store this packet and
+		 * that one ring entry remains unused.
+		 */
+		assert(segs_n);
+		if (max_elts < segs_n)
+			break;
+		/* Do not bother with large packets MPW cannot handle. */
+		if (segs_n > MLX5_MPW_DSEG_MAX) {
+			txq->stats.oerrors++;
+			break;
+		}
+		max_elts -= segs_n;
+		--pkts_n;
+		/*
+		 * Compute max_wqe in case less WQE were consumed in previous
+		 * iteration.
+		 */
+		max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
+		cs_flags = txq_ol_cksum_to_cs(buf);
+		/* Retrieve packet information. */
+		length = PKT_LEN(buf);
+		/* Start new session if packet differs. */
+		if (mpw.state == MLX5_MPW_STATE_OPENED) {
+			if ((mpw.len != length) ||
+			    (segs_n != 1) ||
+			    (mpw.wqe->eseg.cs_flags != cs_flags))
+				mlx5_mpw_close(txq, &mpw);
+		} else if (mpw.state == MLX5_MPW_INL_STATE_OPENED) {
+			if ((mpw.len != length) ||
+			    (segs_n != 1) ||
+			    (length > inline_room) ||
+			    (mpw.wqe->eseg.cs_flags != cs_flags)) {
+				mlx5_mpw_inline_close(txq, &mpw);
+				inline_room =
+					txq->max_inline * RTE_CACHE_LINE_SIZE;
+			}
+		}
+		if (mpw.state == MLX5_MPW_STATE_CLOSED) {
+			if ((segs_n != 1) ||
+			    (length > inline_room)) {
+				/*
+				 * Multi-Packet WQE consumes at most two WQE.
+				 * mlx5_mpw_new() expects to be able to use
+				 * such resources.
+				 */
+				if (unlikely(max_wqe < 2))
+					break;
+				max_wqe -= 2;
+				mlx5_mpw_new(txq, &mpw, length);
+				mpw.wqe->eseg.cs_flags = cs_flags;
+			} else {
+				if (unlikely(max_wqe < wqe_inl_n))
+					break;
+				max_wqe -= wqe_inl_n;
+				mlx5_mpw_inline_new(txq, &mpw, length);
+				mpw.wqe->eseg.cs_flags = cs_flags;
+			}
+		}
+		/* Multi-segment packets must be alone in their MPW. */
+		assert((segs_n == 1) || (mpw.pkts_n == 0));
+		if (mpw.state == MLX5_MPW_STATE_OPENED) {
+			assert(inline_room ==
+			       txq->max_inline * RTE_CACHE_LINE_SIZE);
+#if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
+			length = 0;
+#endif
+			do {
+				volatile struct mlx5_wqe_data_seg *dseg;
+
+				assert(buf);
+				(*txq->elts)[elts_head++ & elts_m] = buf;
+				dseg = mpw.data.dseg[mpw.pkts_n];
+				addr = rte_pktmbuf_mtod(buf, uintptr_t);
+				*dseg = (struct mlx5_wqe_data_seg){
+					.byte_count =
+					       rte_cpu_to_be_32(DATA_LEN(buf)),
+					.lkey = mlx5_tx_mb2mr(txq, buf),
+					.addr = rte_cpu_to_be_64(addr),
+				};
+#if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
+				length += DATA_LEN(buf);
+#endif
+				buf = buf->next;
+				++mpw.pkts_n;
+				++j;
+			} while (--segs_n);
+			assert(length == mpw.len);
+			if (mpw.pkts_n == MLX5_MPW_DSEG_MAX)
+				mlx5_mpw_close(txq, &mpw);
+		} else {
+			unsigned int max;
+
+			assert(mpw.state == MLX5_MPW_INL_STATE_OPENED);
+			assert(length <= inline_room);
+			assert(length == DATA_LEN(buf));
+			addr = rte_pktmbuf_mtod(buf, uintptr_t);
+			(*txq->elts)[elts_head++ & elts_m] = buf;
+			/* Maximum number of bytes before wrapping. */
+			max = ((((uintptr_t)(txq->wqes)) +
+				(1 << txq->wqe_n) *
+				MLX5_WQE_SIZE) -
+			       (uintptr_t)mpw.data.raw);
+			if (length > max) {
+				rte_memcpy((void *)(uintptr_t)mpw.data.raw,
+					   (void *)addr,
+					   max);
+				mpw.data.raw = (volatile void *)txq->wqes;
+				rte_memcpy((void *)(uintptr_t)mpw.data.raw,
+					   (void *)(addr + max),
+					   length - max);
+				mpw.data.raw += length - max;
+			} else {
+				rte_memcpy((void *)(uintptr_t)mpw.data.raw,
+					   (void *)addr,
+					   length);
+
+				if (length == max)
+					mpw.data.raw =
+						(volatile void *)txq->wqes;
+				else
+					mpw.data.raw += length;
+			}
+			++mpw.pkts_n;
+			mpw.total_len += length;
+			++j;
+			if (mpw.pkts_n == MLX5_MPW_DSEG_MAX) {
+				mlx5_mpw_inline_close(txq, &mpw);
+				inline_room =
+					txq->max_inline * RTE_CACHE_LINE_SIZE;
+			} else {
+				inline_room -= length;
+			}
+		}
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		/* Increment sent bytes counter. */
+		txq->stats.obytes += length;
+#endif
+		++i;
+	} while (pkts_n);
+	/* Take a shortcut if nothing must be sent. */
+	if (unlikely(i == 0))
+		return 0;
+	/* Check whether completion threshold has been reached. */
+	/* "j" includes both packets and segments. */
+	comp = txq->elts_comp + j;
+	if (comp >= MLX5_TX_COMP_THRESH) {
+		volatile struct mlx5_wqe *wqe = mpw.wqe;
+
+		/* A CQE slot must always be available. */
+		assert((1u << txq->cqe_n) - (txq->cq_pi++ - txq->cq_ci));
+		/* Request completion on last WQE. */
+		wqe->ctrl[2] = rte_cpu_to_be_32(8);
+		/* Save elts_head in unused "immediate" field of WQE. */
+		wqe->ctrl[3] = elts_head;
+		txq->elts_comp = 0;
+	} else {
+		txq->elts_comp = comp;
+	}
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	/* Increment sent packets counter. */
+	txq->stats.opackets += i;
+#endif
+	/* Ring QP doorbell. */
+	if (mpw.state == MLX5_MPW_INL_STATE_OPENED)
+		mlx5_mpw_inline_close(txq, &mpw);
+	else if (mpw.state == MLX5_MPW_STATE_OPENED)
+		mlx5_mpw_close(txq, &mpw);
+	mlx5_tx_dbrec(txq, mpw.wqe);
+	txq->elts_head = elts_head;
+	return i;
+}
+
+/**
+ * Open an Enhanced MPW session.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param mpw
+ *   Pointer to MPW session structure.
+ * @param length
+ *   Packet length.
+ */
+static inline void
+mlx5_empw_new(struct mlx5_txq_data *txq, struct mlx5_mpw *mpw, int padding)
+{
+	uint16_t idx = txq->wqe_ci & ((1 << txq->wqe_n) - 1);
+
+	mpw->state = MLX5_MPW_ENHANCED_STATE_OPENED;
+	mpw->pkts_n = 0;
+	mpw->total_len = sizeof(struct mlx5_wqe);
+	mpw->wqe = (volatile struct mlx5_wqe *)tx_mlx5_wqe(txq, idx);
+	mpw->wqe->ctrl[0] =
+		rte_cpu_to_be_32((MLX5_OPC_MOD_ENHANCED_MPSW << 24) |
+				 (txq->wqe_ci << 8) |
+				 MLX5_OPCODE_ENHANCED_MPSW);
+	mpw->wqe->ctrl[2] = 0;
+	mpw->wqe->ctrl[3] = 0;
+	memset((void *)(uintptr_t)&mpw->wqe->eseg, 0, MLX5_WQE_DWORD_SIZE);
+	if (unlikely(padding)) {
+		uintptr_t addr = (uintptr_t)(mpw->wqe + 1);
+
+		/* Pad the first 2 DWORDs with zero-length inline header. */
+		*(volatile uint32_t *)addr = rte_cpu_to_be_32(MLX5_INLINE_SEG);
+		*(volatile uint32_t *)(addr + MLX5_WQE_DWORD_SIZE) =
+			rte_cpu_to_be_32(MLX5_INLINE_SEG);
+		mpw->total_len += 2 * MLX5_WQE_DWORD_SIZE;
+		/* Start from the next WQEBB. */
+		mpw->data.raw = (volatile void *)(tx_mlx5_wqe(txq, idx + 1));
+	} else {
+		mpw->data.raw = (volatile void *)(mpw->wqe + 1);
+	}
+}
+
+/**
+ * Close an Enhanced MPW session.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param mpw
+ *   Pointer to MPW session structure.
+ *
+ * @return
+ *   Number of consumed WQEs.
+ */
+static inline uint16_t
+mlx5_empw_close(struct mlx5_txq_data *txq, struct mlx5_mpw *mpw)
+{
+	uint16_t ret;
+
+	/* Store size in multiple of 16 bytes. Control and Ethernet segments
+	 * count as 2.
+	 */
+	mpw->wqe->ctrl[1] = rte_cpu_to_be_32(txq->qp_num_8s |
+					     MLX5_WQE_DS(mpw->total_len));
+	mpw->state = MLX5_MPW_STATE_CLOSED;
+	ret = (mpw->total_len + (MLX5_WQE_SIZE - 1)) / MLX5_WQE_SIZE;
+	txq->wqe_ci += ret;
+	return ret;
+}
+
+/**
+ * TX with Enhanced MPW support.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param[in] pkts
+ *   Packets to transmit.
+ * @param pkts_n
+ *   Number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully transmitted (<= pkts_n).
+ */
+static inline uint16_t
+txq_burst_empw(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
+	       uint16_t pkts_n)
+{
+	uint16_t elts_head = txq->elts_head;
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
+	unsigned int i = 0;
+	unsigned int j = 0;
+	uint16_t max_elts;
+	uint16_t max_wqe;
+	unsigned int max_inline = txq->max_inline * RTE_CACHE_LINE_SIZE;
+	unsigned int mpw_room = 0;
+	unsigned int inl_pad = 0;
+	uint32_t inl_hdr;
+	uint64_t addr_64;
+	struct mlx5_mpw mpw = {
+		.state = MLX5_MPW_STATE_CLOSED,
+	};
+
+	if (unlikely(!pkts_n))
+		return 0;
+	/* Start processing. */
+	mlx5_tx_complete(txq);
+	max_elts = (elts_n - (elts_head - txq->elts_tail));
+	max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
+	if (unlikely(!max_wqe))
+		return 0;
+	do {
+		struct rte_mbuf *buf = *(pkts++);
+		uintptr_t addr;
+		unsigned int do_inline = 0; /* Whether inline is possible. */
+		uint32_t length;
+		uint8_t cs_flags;
+
+		/* Multi-segmented packet is handled in slow-path outside. */
+		assert(NB_SEGS(buf) == 1);
+		/* Make sure there is enough room to store this packet. */
+		if (max_elts - j == 0)
+			break;
+		cs_flags = txq_ol_cksum_to_cs(buf);
+		/* Retrieve packet information. */
+		length = PKT_LEN(buf);
+		/* Start new session if:
+		 * - multi-segment packet
+		 * - no space left even for a dseg
+		 * - next packet can be inlined with a new WQE
+		 * - cs_flag differs
+		 */
+		if (mpw.state == MLX5_MPW_ENHANCED_STATE_OPENED) {
+			if ((inl_pad + sizeof(struct mlx5_wqe_data_seg) >
+			     mpw_room) ||
+			    (length <= txq->inline_max_packet_sz &&
+			     inl_pad + sizeof(inl_hdr) + length >
+			     mpw_room) ||
+			    (mpw.wqe->eseg.cs_flags != cs_flags))
+				max_wqe -= mlx5_empw_close(txq, &mpw);
+		}
+		if (unlikely(mpw.state == MLX5_MPW_STATE_CLOSED)) {
+			/* In Enhanced MPW, inline as much as the budget is
+			 * allowed. The remaining space is to be filled with
+			 * dsegs. If the title WQEBB isn't padded, it will have
+			 * 2 dsegs there.
+			 */
+			mpw_room = RTE_MIN(MLX5_WQE_SIZE_MAX,
+					   (max_inline ? max_inline :
+					    pkts_n * MLX5_WQE_DWORD_SIZE) +
+					   MLX5_WQE_SIZE);
+			if (unlikely(max_wqe * MLX5_WQE_SIZE < mpw_room))
+				break;
+			/* Don't pad the title WQEBB to not waste WQ. */
+			mlx5_empw_new(txq, &mpw, 0);
+			mpw_room -= mpw.total_len;
+			inl_pad = 0;
+			do_inline = length <= txq->inline_max_packet_sz &&
+				    sizeof(inl_hdr) + length <= mpw_room &&
+				    !txq->mpw_hdr_dseg;
+			mpw.wqe->eseg.cs_flags = cs_flags;
+		} else {
+			/* Evaluate whether the next packet can be inlined.
+			 * Inlininig is possible when:
+			 * - length is less than configured value
+			 * - length fits for remaining space
+			 * - not required to fill the title WQEBB with dsegs
+			 */
+			do_inline =
+				length <= txq->inline_max_packet_sz &&
+				inl_pad + sizeof(inl_hdr) + length <=
+				 mpw_room &&
+				(!txq->mpw_hdr_dseg ||
+				 mpw.total_len >= MLX5_WQE_SIZE);
+		}
+		if (max_inline && do_inline) {
+			/* Inline packet into WQE. */
+			unsigned int max;
+
+			assert(mpw.state == MLX5_MPW_ENHANCED_STATE_OPENED);
+			assert(length == DATA_LEN(buf));
+			inl_hdr = rte_cpu_to_be_32(length | MLX5_INLINE_SEG);
+			addr = rte_pktmbuf_mtod(buf, uintptr_t);
+			mpw.data.raw = (volatile void *)
+				((uintptr_t)mpw.data.raw + inl_pad);
+			max = tx_mlx5_wq_tailroom(txq,
+					(void *)(uintptr_t)mpw.data.raw);
+			/* Copy inline header. */
+			mpw.data.raw = (volatile void *)
+				mlx5_copy_to_wq(
+					  (void *)(uintptr_t)mpw.data.raw,
+					  &inl_hdr,
+					  sizeof(inl_hdr),
+					  (void *)(uintptr_t)txq->wqes,
+					  max);
+			max = tx_mlx5_wq_tailroom(txq,
+					(void *)(uintptr_t)mpw.data.raw);
+			/* Copy packet data. */
+			mpw.data.raw = (volatile void *)
+				mlx5_copy_to_wq(
+					  (void *)(uintptr_t)mpw.data.raw,
+					  (void *)addr,
+					  length,
+					  (void *)(uintptr_t)txq->wqes,
+					  max);
+			++mpw.pkts_n;
+			mpw.total_len += (inl_pad + sizeof(inl_hdr) + length);
+			/* No need to get completion as the entire packet is
+			 * copied to WQ. Free the buf right away.
+			 */
+			rte_pktmbuf_free_seg(buf);
+			mpw_room -= (inl_pad + sizeof(inl_hdr) + length);
+			/* Add pad in the next packet if any. */
+			inl_pad = (((uintptr_t)mpw.data.raw +
+					(MLX5_WQE_DWORD_SIZE - 1)) &
+					~(MLX5_WQE_DWORD_SIZE - 1)) -
+				  (uintptr_t)mpw.data.raw;
+		} else {
+			/* No inline. Load a dseg of packet pointer. */
+			volatile rte_v128u32_t *dseg;
+
+			assert(mpw.state == MLX5_MPW_ENHANCED_STATE_OPENED);
+			assert((inl_pad + sizeof(*dseg)) <= mpw_room);
+			assert(length == DATA_LEN(buf));
+			if (!tx_mlx5_wq_tailroom(txq,
+					(void *)((uintptr_t)mpw.data.raw
+						+ inl_pad)))
+				dseg = (volatile void *)txq->wqes;
+			else
+				dseg = (volatile void *)
+					((uintptr_t)mpw.data.raw +
+					 inl_pad);
+			(*txq->elts)[elts_head++ & elts_m] = buf;
+			addr_64 = rte_cpu_to_be_64(rte_pktmbuf_mtod(buf,
+								    uintptr_t));
+			*dseg = (rte_v128u32_t) {
+				rte_cpu_to_be_32(length),
+				mlx5_tx_mb2mr(txq, buf),
+				addr_64,
+				addr_64 >> 32,
+			};
+			mpw.data.raw = (volatile void *)(dseg + 1);
+			mpw.total_len += (inl_pad + sizeof(*dseg));
+			++j;
+			++mpw.pkts_n;
+			mpw_room -= (inl_pad + sizeof(*dseg));
+			inl_pad = 0;
+		}
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		/* Increment sent bytes counter. */
+		txq->stats.obytes += length;
+#endif
+		++i;
+	} while (i < pkts_n);
+	/* Take a shortcut if nothing must be sent. */
+	if (unlikely(i == 0))
+		return 0;
+	/* Check whether completion threshold has been reached. */
+	if (txq->elts_comp + j >= MLX5_TX_COMP_THRESH ||
+			(uint16_t)(txq->wqe_ci - txq->mpw_comp) >=
+			 (1 << txq->wqe_n) / MLX5_TX_COMP_THRESH_INLINE_DIV) {
+		volatile struct mlx5_wqe *wqe = mpw.wqe;
+
+		/* A CQE slot must always be available. */
+		assert((1u << txq->cqe_n) - (txq->cq_pi++ - txq->cq_ci));
+		/* Request completion on last WQE. */
+		wqe->ctrl[2] = rte_cpu_to_be_32(8);
+		/* Save elts_head in unused "immediate" field of WQE. */
+		wqe->ctrl[3] = elts_head;
+		txq->elts_comp = 0;
+		txq->mpw_comp = txq->wqe_ci;
+	} else {
+		txq->elts_comp += j;
+	}
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	/* Increment sent packets counter. */
+	txq->stats.opackets += i;
+#endif
+	if (mpw.state == MLX5_MPW_ENHANCED_STATE_OPENED)
+		mlx5_empw_close(txq, &mpw);
+	/* Ring QP doorbell. */
+	mlx5_tx_dbrec(txq, mpw.wqe);
+	txq->elts_head = elts_head;
+	return i;
+}
+
+/**
+ * DPDK callback for TX with Enhanced MPW support.
+ *
+ * @param dpdk_txq
+ *   Generic pointer to TX queue structure.
+ * @param[in] pkts
+ *   Packets to transmit.
+ * @param pkts_n
+ *   Number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully transmitted (<= pkts_n).
+ */
+uint16_t
+mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq;
+	uint16_t nb_tx = 0;
+
+	while (pkts_n > nb_tx) {
+		uint16_t n;
+		uint16_t ret;
+
+		n = txq_count_contig_multi_seg(&pkts[nb_tx], pkts_n - nb_tx);
+		if (n) {
+			ret = mlx5_tx_burst(dpdk_txq, &pkts[nb_tx], n);
+			if (!ret)
+				break;
+			nb_tx += ret;
+		}
+		n = txq_count_contig_single_seg(&pkts[nb_tx], pkts_n - nb_tx);
+		if (n) {
+			ret = txq_burst_empw(txq, &pkts[nb_tx], n);
+			if (!ret)
+				break;
+			nb_tx += ret;
+		}
+	}
+	return nb_tx;
+}
+
+/**
+ * Translate RX completion flags to packet type.
+ *
+ * @param[in] rxq
+ *   Pointer to RX queue structure.
+ * @param[in] cqe
+ *   Pointer to CQE.
+ *
+ * @note: fix mlx5_dev_supported_ptypes_get() if any change here.
+ *
+ * @return
+ *   Packet type for struct rte_mbuf.
+ */
+static inline uint32_t
+rxq_cq_to_pkt_type(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe)
+{
+	uint8_t idx;
+	uint8_t pinfo = cqe->pkt_info;
+	uint16_t ptype = cqe->hdr_type_etc;
+
+	/*
+	 * The index to the array should have:
+	 * bit[1:0] = l3_hdr_type
+	 * bit[4:2] = l4_hdr_type
+	 * bit[5] = ip_frag
+	 * bit[6] = tunneled
+	 * bit[7] = outer_l3_type
+	 */
+	idx = ((pinfo & 0x3) << 6) | ((ptype & 0xfc00) >> 10);
+	return mlx5_ptype_table[idx] | rxq->tunnel * !!(idx & (1 << 6));
+}
+
+/**
+ * Get size of the next packet for a given CQE. For compressed CQEs, the
+ * consumer index is updated only once all packets of the current one have
+ * been processed.
+ *
+ * @param rxq
+ *   Pointer to RX queue.
+ * @param cqe
+ *   CQE to process.
+ * @param[out] mcqe
+ *   Store pointer to mini-CQE if compressed. Otherwise, the pointer is not
+ *   written.
+ *
+ * @return
+ *   Packet size in bytes (0 if there is none), -1 in case of completion
+ *   with error.
+ */
+static inline int
+mlx5_rx_poll_len(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe,
+		 uint16_t cqe_cnt, volatile struct mlx5_mini_cqe8 **mcqe)
+{
+	struct rxq_zip *zip = &rxq->zip;
+	uint16_t cqe_n = cqe_cnt + 1;
+	int len = 0;
+	uint16_t idx, end;
+
+	/* Process compressed data in the CQE and mini arrays. */
+	if (zip->ai) {
+		volatile struct mlx5_mini_cqe8 (*mc)[8] =
+			(volatile struct mlx5_mini_cqe8 (*)[8])
+			(uintptr_t)(&(*rxq->cqes)[zip->ca & cqe_cnt].pkt_info);
+
+		len = rte_be_to_cpu_32((*mc)[zip->ai & 7].byte_cnt);
+		*mcqe = &(*mc)[zip->ai & 7];
+		if ((++zip->ai & 7) == 0) {
+			/* Invalidate consumed CQEs */
+			idx = zip->ca;
+			end = zip->na;
+			while (idx != end) {
+				(*rxq->cqes)[idx & cqe_cnt].op_own =
+					MLX5_CQE_INVALIDATE;
+				++idx;
+			}
+			/*
+			 * Increment consumer index to skip the number of
+			 * CQEs consumed. Hardware leaves holes in the CQ
+			 * ring for software use.
+			 */
+			zip->ca = zip->na;
+			zip->na += 8;
+		}
+		if (unlikely(rxq->zip.ai == rxq->zip.cqe_cnt)) {
+			/* Invalidate the rest */
+			idx = zip->ca;
+			end = zip->cq_ci;
+
+			while (idx != end) {
+				(*rxq->cqes)[idx & cqe_cnt].op_own =
+					MLX5_CQE_INVALIDATE;
+				++idx;
+			}
+			rxq->cq_ci = zip->cq_ci;
+			zip->ai = 0;
+		}
+	/* No compressed data, get next CQE and verify if it is compressed. */
+	} else {
+		int ret;
+		int8_t op_own;
+
+		ret = check_cqe(cqe, cqe_n, rxq->cq_ci);
+		if (unlikely(ret == 1))
+			return 0;
+		++rxq->cq_ci;
+		op_own = cqe->op_own;
+		rte_cio_rmb();
+		if (MLX5_CQE_FORMAT(op_own) == MLX5_COMPRESSED) {
+			volatile struct mlx5_mini_cqe8 (*mc)[8] =
+				(volatile struct mlx5_mini_cqe8 (*)[8])
+				(uintptr_t)(&(*rxq->cqes)[rxq->cq_ci &
+							  cqe_cnt].pkt_info);
+
+			/* Fix endianness. */
+			zip->cqe_cnt = rte_be_to_cpu_32(cqe->byte_cnt);
+			/*
+			 * Current mini array position is the one returned by
+			 * check_cqe64().
+			 *
+			 * If completion comprises several mini arrays, as a
+			 * special case the second one is located 7 CQEs after
+			 * the initial CQE instead of 8 for subsequent ones.
+			 */
+			zip->ca = rxq->cq_ci;
+			zip->na = zip->ca + 7;
+			/* Compute the next non compressed CQE. */
+			--rxq->cq_ci;
+			zip->cq_ci = rxq->cq_ci + zip->cqe_cnt;
+			/* Get packet size to return. */
+			len = rte_be_to_cpu_32((*mc)[0].byte_cnt);
+			*mcqe = &(*mc)[0];
+			zip->ai = 1;
+			/* Prefetch all the entries to be invalidated */
+			idx = zip->ca;
+			end = zip->cq_ci;
+			while (idx != end) {
+				rte_prefetch0(&(*rxq->cqes)[(idx) & cqe_cnt]);
+				++idx;
+			}
+		} else {
+			len = rte_be_to_cpu_32(cqe->byte_cnt);
+		}
+		/* Error while receiving packet. */
+		if (unlikely(MLX5_CQE_OPCODE(op_own) == MLX5_CQE_RESP_ERR))
+			return -1;
+	}
+	return len;
+}
+
+/**
+ * Translate RX completion flags to offload flags.
+ *
+ * @param[in] cqe
+ *   Pointer to CQE.
+ *
+ * @return
+ *   Offload flags (ol_flags) for struct rte_mbuf.
+ */
+static inline uint32_t
+rxq_cq_to_ol_flags(volatile struct mlx5_cqe *cqe)
+{
+	uint32_t ol_flags = 0;
+	uint16_t flags = rte_be_to_cpu_16(cqe->hdr_type_etc);
+
+	ol_flags =
+		TRANSPOSE(flags,
+			  MLX5_CQE_RX_L3_HDR_VALID,
+			  PKT_RX_IP_CKSUM_GOOD) |
+		TRANSPOSE(flags,
+			  MLX5_CQE_RX_L4_HDR_VALID,
+			  PKT_RX_L4_CKSUM_GOOD);
+	return ol_flags;
+}
+
+/**
+ * Fill in mbuf fields from RX completion flags.
+ * Note that pkt->ol_flags should be initialized outside of this function.
+ *
+ * @param rxq
+ *   Pointer to RX queue.
+ * @param pkt
+ *   mbuf to fill.
+ * @param cqe
+ *   CQE to process.
+ * @param rss_hash_res
+ *   Packet RSS Hash result.
+ */
+static inline void
+rxq_cq_to_mbuf(struct mlx5_rxq_data *rxq, struct rte_mbuf *pkt,
+	       volatile struct mlx5_cqe *cqe, uint32_t rss_hash_res)
+{
+	/* Update packet information. */
+	pkt->packet_type = rxq_cq_to_pkt_type(rxq, cqe);
+	if (rss_hash_res && rxq->rss_hash) {
+		pkt->hash.rss = rss_hash_res;
+		pkt->ol_flags |= PKT_RX_RSS_HASH;
+	}
+	if (rxq->mark && MLX5_FLOW_MARK_IS_VALID(cqe->sop_drop_qpn)) {
+		pkt->ol_flags |= PKT_RX_FDIR;
+		if (cqe->sop_drop_qpn !=
+		    rte_cpu_to_be_32(MLX5_FLOW_MARK_DEFAULT)) {
+			uint32_t mark = cqe->sop_drop_qpn;
+
+			pkt->ol_flags |= PKT_RX_FDIR_ID;
+			pkt->hash.fdir.hi = mlx5_flow_mark_get(mark);
+		}
+	}
+	if (rxq->csum)
+		pkt->ol_flags |= rxq_cq_to_ol_flags(cqe);
+	if (rxq->vlan_strip &&
+	    (cqe->hdr_type_etc & rte_cpu_to_be_16(MLX5_CQE_VLAN_STRIPPED))) {
+		pkt->ol_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED;
+		pkt->vlan_tci = rte_be_to_cpu_16(cqe->vlan_info);
+	}
+	if (rxq->hw_timestamp) {
+		pkt->timestamp = rte_be_to_cpu_64(cqe->timestamp);
+		pkt->ol_flags |= PKT_RX_TIMESTAMP;
+	}
+}
+
+/**
+ * DPDK callback for RX.
+ *
+ * @param dpdk_rxq
+ *   Generic pointer to RX queue structure.
+ * @param[out] pkts
+ *   Array to store received packets.
+ * @param pkts_n
+ *   Maximum number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully received (<= pkts_n).
+ */
+uint16_t
+mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	struct mlx5_rxq_data *rxq = dpdk_rxq;
+	const unsigned int wqe_cnt = (1 << rxq->elts_n) - 1;
+	const unsigned int cqe_cnt = (1 << rxq->cqe_n) - 1;
+	const unsigned int sges_n = rxq->sges_n;
+	struct rte_mbuf *pkt = NULL;
+	struct rte_mbuf *seg = NULL;
+	volatile struct mlx5_cqe *cqe =
+		&(*rxq->cqes)[rxq->cq_ci & cqe_cnt];
+	unsigned int i = 0;
+	unsigned int rq_ci = rxq->rq_ci << sges_n;
+	int len = 0; /* keep its value across iterations. */
+
+	while (pkts_n) {
+		unsigned int idx = rq_ci & wqe_cnt;
+		volatile struct mlx5_wqe_data_seg *wqe =
+			&((volatile struct mlx5_wqe_data_seg *)rxq->wqes)[idx];
+		struct rte_mbuf *rep = (*rxq->elts)[idx];
+		volatile struct mlx5_mini_cqe8 *mcqe = NULL;
+		uint32_t rss_hash_res;
+
+		if (pkt)
+			NEXT(seg) = rep;
+		seg = rep;
+		rte_prefetch0(seg);
+		rte_prefetch0(cqe);
+		rte_prefetch0(wqe);
+		rep = rte_mbuf_raw_alloc(rxq->mp);
+		if (unlikely(rep == NULL)) {
+			++rxq->stats.rx_nombuf;
+			if (!pkt) {
+				/*
+				 * no buffers before we even started,
+				 * bail out silently.
+				 */
+				break;
+			}
+			while (pkt != seg) {
+				assert(pkt != (*rxq->elts)[idx]);
+				rep = NEXT(pkt);
+				NEXT(pkt) = NULL;
+				NB_SEGS(pkt) = 1;
+				rte_mbuf_raw_free(pkt);
+				pkt = rep;
+			}
+			break;
+		}
+		if (!pkt) {
+			cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_cnt];
+			len = mlx5_rx_poll_len(rxq, cqe, cqe_cnt, &mcqe);
+			if (!len) {
+				rte_mbuf_raw_free(rep);
+				break;
+			}
+			if (unlikely(len == -1)) {
+				/* RX error, packet is likely too large. */
+				rte_mbuf_raw_free(rep);
+				++rxq->stats.idropped;
+				goto skip;
+			}
+			pkt = seg;
+			assert(len >= (rxq->crc_present << 2));
+			pkt->ol_flags = 0;
+			/* If compressed, take hash result from mini-CQE. */
+			rss_hash_res = rte_be_to_cpu_32(mcqe == NULL ?
+							cqe->rx_hash_res :
+							mcqe->rx_hash_result);
+			rxq_cq_to_mbuf(rxq, pkt, cqe, rss_hash_res);
+			if (rxq->crc_present)
+				len -= ETHER_CRC_LEN;
+			PKT_LEN(pkt) = len;
+		}
+		DATA_LEN(rep) = DATA_LEN(seg);
+		PKT_LEN(rep) = PKT_LEN(seg);
+		SET_DATA_OFF(rep, DATA_OFF(seg));
+		PORT(rep) = PORT(seg);
+		(*rxq->elts)[idx] = rep;
+		/*
+		 * Fill NIC descriptor with the new buffer.  The lkey and size
+		 * of the buffers are already known, only the buffer address
+		 * changes.
+		 */
+		wqe->addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(rep, uintptr_t));
+		/* If there's only one MR, no need to replace LKey in WQE. */
+		if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) > 1))
+			wqe->lkey = mlx5_rx_mb2mr(rxq, rep);
+		if (len > DATA_LEN(seg)) {
+			len -= DATA_LEN(seg);
+			++NB_SEGS(pkt);
+			++rq_ci;
+			continue;
+		}
+		DATA_LEN(seg) = len;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		/* Increment bytes counter. */
+		rxq->stats.ibytes += PKT_LEN(pkt);
+#endif
+		/* Return packet. */
+		*(pkts++) = pkt;
+		pkt = NULL;
+		--pkts_n;
+		++i;
+skip:
+		/* Align consumer index to the next stride. */
+		rq_ci >>= sges_n;
+		++rq_ci;
+		rq_ci <<= sges_n;
+	}
+	if (unlikely((i == 0) && ((rq_ci >> sges_n) == rxq->rq_ci)))
+		return 0;
+	/* Update the consumer index. */
+	rxq->rq_ci = rq_ci >> sges_n;
+	rte_cio_wmb();
+	*rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci);
+	rte_cio_wmb();
+	*rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci);
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	/* Increment packets counter. */
+	rxq->stats.ipackets += i;
+#endif
+	return i;
+}
+
+void
+mlx5_mprq_buf_free_cb(void *addr __rte_unused, void *opaque)
+{
+	struct mlx5_mprq_buf *buf = opaque;
+
+	if (rte_atomic16_read(&buf->refcnt) == 1) {
+		rte_mempool_put(buf->mp, buf);
+	} else if (rte_atomic16_add_return(&buf->refcnt, -1) == 0) {
+		rte_atomic16_set(&buf->refcnt, 1);
+		rte_mempool_put(buf->mp, buf);
+	}
+}
+
+void
+mlx5_mprq_buf_free(struct mlx5_mprq_buf *buf)
+{
+	mlx5_mprq_buf_free_cb(NULL, buf);
+}
+
+static inline void
+mprq_buf_replace(struct mlx5_rxq_data *rxq, uint16_t rq_idx)
+{
+	struct mlx5_mprq_buf *rep = rxq->mprq_repl;
+	volatile struct mlx5_wqe_data_seg *wqe =
+		&((volatile struct mlx5_wqe_mprq *)rxq->wqes)[rq_idx].dseg;
+	void *addr;
+
+	assert(rep != NULL);
+	/* Replace MPRQ buf. */
+	(*rxq->mprq_bufs)[rq_idx] = rep;
+	/* Replace WQE. */
+	addr = mlx5_mprq_buf_addr(rep);
+	wqe->addr = rte_cpu_to_be_64((uintptr_t)addr);
+	/* If there's only one MR, no need to replace LKey in WQE. */
+	if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) > 1))
+		wqe->lkey = mlx5_rx_addr2mr(rxq, (uintptr_t)addr);
+	/* Stash a mbuf for next replacement. */
+	if (likely(!rte_mempool_get(rxq->mprq_mp, (void **)&rep)))
+		rxq->mprq_repl = rep;
+	else
+		rxq->mprq_repl = NULL;
+}
+
+/**
+ * DPDK callback for RX with Multi-Packet RQ support.
+ *
+ * @param dpdk_rxq
+ *   Generic pointer to RX queue structure.
+ * @param[out] pkts
+ *   Array to store received packets.
+ * @param pkts_n
+ *   Maximum number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully received (<= pkts_n).
+ */
+uint16_t
+mlx5_rx_burst_mprq(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	struct mlx5_rxq_data *rxq = dpdk_rxq;
+	const unsigned int strd_n = 1 << rxq->strd_num_n;
+	const unsigned int strd_sz = 1 << rxq->strd_sz_n;
+	const unsigned int strd_shift =
+		MLX5_MPRQ_STRIDE_SHIFT_BYTE * rxq->strd_shift_en;
+	const unsigned int cq_mask = (1 << rxq->cqe_n) - 1;
+	const unsigned int wq_mask = (1 << rxq->elts_n) - 1;
+	volatile struct mlx5_cqe *cqe = &(*rxq->cqes)[rxq->cq_ci & cq_mask];
+	unsigned int i = 0;
+	uint16_t rq_ci = rxq->rq_ci;
+	uint16_t consumed_strd = rxq->consumed_strd;
+	struct mlx5_mprq_buf *buf = (*rxq->mprq_bufs)[rq_ci & wq_mask];
+
+	while (i < pkts_n) {
+		struct rte_mbuf *pkt;
+		void *addr;
+		int ret;
+		unsigned int len;
+		uint16_t strd_cnt;
+		uint16_t strd_idx;
+		uint32_t offset;
+		uint32_t byte_cnt;
+		volatile struct mlx5_mini_cqe8 *mcqe = NULL;
+		uint32_t rss_hash_res = 0;
+
+		if (consumed_strd == strd_n) {
+			/* Replace WQE only if the buffer is still in use. */
+			if (rte_atomic16_read(&buf->refcnt) > 1) {
+				mprq_buf_replace(rxq, rq_ci & wq_mask);
+				/* Release the old buffer. */
+				mlx5_mprq_buf_free(buf);
+			} else if (unlikely(rxq->mprq_repl == NULL)) {
+				struct mlx5_mprq_buf *rep;
+
+				/*
+				 * Currently, the MPRQ mempool is out of buffer
+				 * and doing memcpy regardless of the size of Rx
+				 * packet. Retry allocation to get back to
+				 * normal.
+				 */
+				if (!rte_mempool_get(rxq->mprq_mp,
+						     (void **)&rep))
+					rxq->mprq_repl = rep;
+			}
+			/* Advance to the next WQE. */
+			consumed_strd = 0;
+			++rq_ci;
+			buf = (*rxq->mprq_bufs)[rq_ci & wq_mask];
+		}
+		cqe = &(*rxq->cqes)[rxq->cq_ci & cq_mask];
+		ret = mlx5_rx_poll_len(rxq, cqe, cq_mask, &mcqe);
+		if (!ret)
+			break;
+		if (unlikely(ret == -1)) {
+			/* RX error, packet is likely too large. */
+			++rxq->stats.idropped;
+			continue;
+		}
+		byte_cnt = ret;
+		strd_cnt = (byte_cnt & MLX5_MPRQ_STRIDE_NUM_MASK) >>
+			   MLX5_MPRQ_STRIDE_NUM_SHIFT;
+		assert(strd_cnt);
+		consumed_strd += strd_cnt;
+		if (byte_cnt & MLX5_MPRQ_FILLER_MASK)
+			continue;
+		if (mcqe == NULL) {
+			rss_hash_res = rte_be_to_cpu_32(cqe->rx_hash_res);
+			strd_idx = rte_be_to_cpu_16(cqe->wqe_counter);
+		} else {
+			/* mini-CQE for MPRQ doesn't have hash result. */
+			strd_idx = rte_be_to_cpu_16(mcqe->stride_idx);
+		}
+		assert(strd_idx < strd_n);
+		assert(!((rte_be_to_cpu_16(cqe->wqe_id) ^ rq_ci) & wq_mask));
+		/*
+		 * Currently configured to receive a packet per a stride. But if
+		 * MTU is adjusted through kernel interface, device could
+		 * consume multiple strides without raising an error. In this
+		 * case, the packet should be dropped because it is bigger than
+		 * the max_rx_pkt_len.
+		 */
+		if (unlikely(strd_cnt > 1)) {
+			++rxq->stats.idropped;
+			continue;
+		}
+		pkt = rte_pktmbuf_alloc(rxq->mp);
+		if (unlikely(pkt == NULL)) {
+			++rxq->stats.rx_nombuf;
+			break;
+		}
+		len = (byte_cnt & MLX5_MPRQ_LEN_MASK) >> MLX5_MPRQ_LEN_SHIFT;
+		assert((int)len >= (rxq->crc_present << 2));
+		if (rxq->crc_present)
+			len -= ETHER_CRC_LEN;
+		offset = strd_idx * strd_sz + strd_shift;
+		addr = RTE_PTR_ADD(mlx5_mprq_buf_addr(buf), offset);
+		/* Initialize the offload flag. */
+		pkt->ol_flags = 0;
+		/*
+		 * Memcpy packets to the target mbuf if:
+		 * - The size of packet is smaller than mprq_max_memcpy_len.
+		 * - Out of buffer in the Mempool for Multi-Packet RQ.
+		 */
+		if (len <= rxq->mprq_max_memcpy_len || rxq->mprq_repl == NULL) {
+			/*
+			 * When memcpy'ing packet due to out-of-buffer, the
+			 * packet must be smaller than the target mbuf.
+			 */
+			if (unlikely(rte_pktmbuf_tailroom(pkt) < len)) {
+				rte_pktmbuf_free_seg(pkt);
+				++rxq->stats.idropped;
+				continue;
+			}
+			rte_memcpy(rte_pktmbuf_mtod(pkt, void *), addr, len);
+		} else {
+			rte_iova_t buf_iova;
+			struct rte_mbuf_ext_shared_info *shinfo;
+			uint16_t buf_len = strd_cnt * strd_sz;
+
+			/* Increment the refcnt of the whole chunk. */
+			rte_atomic16_add_return(&buf->refcnt, 1);
+			assert((uint16_t)rte_atomic16_read(&buf->refcnt) <=
+			       strd_n + 1);
+			addr = RTE_PTR_SUB(addr, RTE_PKTMBUF_HEADROOM);
+			/*
+			 * MLX5 device doesn't use iova but it is necessary in a
+			 * case where the Rx packet is transmitted via a
+			 * different PMD.
+			 */
+			buf_iova = rte_mempool_virt2iova(buf) +
+				   RTE_PTR_DIFF(addr, buf);
+			shinfo = rte_pktmbuf_ext_shinfo_init_helper(addr,
+					&buf_len, mlx5_mprq_buf_free_cb, buf);
+			/*
+			 * EXT_ATTACHED_MBUF will be set to pkt->ol_flags when
+			 * attaching the stride to mbuf and more offload flags
+			 * will be added below by calling rxq_cq_to_mbuf().
+			 * Other fields will be overwritten.
+			 */
+			rte_pktmbuf_attach_extbuf(pkt, addr, buf_iova, buf_len,
+						  shinfo);
+			rte_pktmbuf_reset_headroom(pkt);
+			assert(pkt->ol_flags == EXT_ATTACHED_MBUF);
+			/*
+			 * Prevent potential overflow due to MTU change through
+			 * kernel interface.
+			 */
+			if (unlikely(rte_pktmbuf_tailroom(pkt) < len)) {
+				rte_pktmbuf_free_seg(pkt);
+				++rxq->stats.idropped;
+				continue;
+			}
+		}
+		rxq_cq_to_mbuf(rxq, pkt, cqe, rss_hash_res);
+		PKT_LEN(pkt) = len;
+		DATA_LEN(pkt) = len;
+		PORT(pkt) = rxq->port_id;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		/* Increment bytes counter. */
+		rxq->stats.ibytes += PKT_LEN(pkt);
+#endif
+		/* Return packet. */
+		*(pkts++) = pkt;
+		++i;
+	}
+	/* Update the consumer indexes. */
+	rxq->consumed_strd = consumed_strd;
+	rte_cio_wmb();
+	*rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci);
+	if (rq_ci != rxq->rq_ci) {
+		rxq->rq_ci = rq_ci;
+		rte_cio_wmb();
+		*rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci);
+	}
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	/* Increment packets counter. */
+	rxq->stats.ipackets += i;
+#endif
+	return i;
+}
+
+/**
+ * Dummy DPDK callback for TX.
+ *
+ * This function is used to temporarily replace the real callback during
+ * unsafe control operations on the queue, or in case of error.
+ *
+ * @param dpdk_txq
+ *   Generic pointer to TX queue structure.
+ * @param[in] pkts
+ *   Packets to transmit.
+ * @param pkts_n
+ *   Number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully transmitted (<= pkts_n).
+ */
+uint16_t
+removed_tx_burst(void *dpdk_txq __rte_unused,
+		 struct rte_mbuf **pkts __rte_unused,
+		 uint16_t pkts_n __rte_unused)
+{
+	return 0;
+}
+
+/**
+ * Dummy DPDK callback for RX.
+ *
+ * This function is used to temporarily replace the real callback during
+ * unsafe control operations on the queue, or in case of error.
+ *
+ * @param dpdk_rxq
+ *   Generic pointer to RX queue structure.
+ * @param[out] pkts
+ *   Array to store received packets.
+ * @param pkts_n
+ *   Maximum number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully received (<= pkts_n).
+ */
+uint16_t
+removed_rx_burst(void *dpdk_txq __rte_unused,
+		 struct rte_mbuf **pkts __rte_unused,
+		 uint16_t pkts_n __rte_unused)
+{
+	return 0;
+}
+
+/*
+ * Vectorized Rx/Tx routines are not compiled in when required vector
+ * instructions are not supported on a target architecture. The following null
+ * stubs are needed for linkage when those are not included outside of this file
+ * (e.g.  mlx5_rxtx_vec_sse.c for x86).
+ */
+
+uint16_t __attribute__((weak))
+mlx5_tx_burst_raw_vec(void *dpdk_txq __rte_unused,
+		      struct rte_mbuf **pkts __rte_unused,
+		      uint16_t pkts_n __rte_unused)
+{
+	return 0;
+}
+
+uint16_t __attribute__((weak))
+mlx5_tx_burst_vec(void *dpdk_txq __rte_unused,
+		  struct rte_mbuf **pkts __rte_unused,
+		  uint16_t pkts_n __rte_unused)
+{
+	return 0;
+}
+
+uint16_t __attribute__((weak))
+mlx5_rx_burst_vec(void *dpdk_txq __rte_unused,
+		  struct rte_mbuf **pkts __rte_unused,
+		  uint16_t pkts_n __rte_unused)
+{
+	return 0;
+}
+
+int __attribute__((weak))
+mlx5_check_raw_vec_tx_support(struct rte_eth_dev *dev __rte_unused)
+{
+	return -ENOTSUP;
+}
+
+int __attribute__((weak))
+mlx5_check_vec_tx_support(struct rte_eth_dev *dev __rte_unused)
+{
+	return -ENOTSUP;
+}
+
+int __attribute__((weak))
+mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq __rte_unused)
+{
+	return -ENOTSUP;
+}
+
+int __attribute__((weak))
+mlx5_check_vec_rx_support(struct rte_eth_dev *dev __rte_unused)
+{
+	return -ENOTSUP;
+}
diff --git a/src/spdk/dpdk/drivers/net/mlx5/mlx5_rxtx.h b/src/spdk/dpdk/drivers/net/mlx5/mlx5_rxtx.h
new file mode 100644
index 00000000..48ed2b20
--- /dev/null
+++ b/src/spdk/dpdk/drivers/net/mlx5/mlx5_rxtx.h
@@ -0,0 +1,856 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2015 6WIND S.A.
+ * Copyright 2015 Mellanox Technologies, Ltd
+ */
+
+#ifndef RTE_PMD_MLX5_RXTX_H_
+#define RTE_PMD_MLX5_RXTX_H_
+
+#include <stddef.h>
+#include <stdint.h>
+#include <sys/queue.h>
+
+/* Verbs header. */
+/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <infiniband/verbs.h>
+#include <infiniband/mlx5dv.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+#include <rte_mbuf.h>
+#include <rte_mempool.h>
+#include <rte_common.h>
+#include <rte_hexdump.h>
+#include <rte_atomic.h>
+#include <rte_spinlock.h>
+#include <rte_io.h>
+
+#include "mlx5_utils.h"
+#include "mlx5.h"
+#include "mlx5_mr.h"
+#include "mlx5_autoconf.h"
+#include "mlx5_defs.h"
+#include "mlx5_prm.h"
+
+/* Support tunnel matching. */
+#define MLX5_FLOW_TUNNEL 5
+
+struct mlx5_rxq_stats {
+	unsigned int idx; /**< Mapping index. */
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	uint64_t ipackets; /**< Total of successfully received packets. */
+	uint64_t ibytes; /**< Total of successfully received bytes. */
+#endif
+	uint64_t idropped; /**< Total of packets dropped when RX ring full. */
+	uint64_t rx_nombuf; /**< Total of RX mbuf allocation failures. */
+};
+
+struct mlx5_txq_stats {
+	unsigned int idx; /**< Mapping index. */
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	uint64_t opackets; /**< Total of successfully sent packets. */
+	uint64_t obytes; /**< Total of successfully sent bytes. */
+#endif
+	uint64_t oerrors; /**< Total number of failed transmitted packets. */
+};
+
+struct priv;
+
+/* Compressed CQE context. */
+struct rxq_zip {
+	uint16_t ai; /* Array index. */
+	uint16_t ca; /* Current array index. */
+	uint16_t na; /* Next array index. */
+	uint16_t cq_ci; /* The next CQE. */
+	uint32_t cqe_cnt; /* Number of CQEs. */
+};
+
+/* Multi-Packet RQ buffer header. */
+struct mlx5_mprq_buf {
+	struct rte_mempool *mp;
+	rte_atomic16_t refcnt; /* Atomically accessed refcnt. */
+	uint8_t pad[RTE_PKTMBUF_HEADROOM]; /* Headroom for the first packet. */
+} __rte_cache_aligned;
+
+/* Get pointer to the first stride. */
+#define mlx5_mprq_buf_addr(ptr) ((ptr) + 1)
+
+/* RX queue descriptor. */
+struct mlx5_rxq_data {
+	unsigned int csum:1; /* Enable checksum offloading. */
+	unsigned int hw_timestamp:1; /* Enable HW timestamp. */
+	unsigned int vlan_strip:1; /* Enable VLAN stripping. */
+	unsigned int crc_present:1; /* CRC must be subtracted. */
+	unsigned int sges_n:2; /* Log 2 of SGEs (max buffers per packet). */
+	unsigned int cqe_n:4; /* Log 2 of CQ elements. */
+	unsigned int elts_n:4; /* Log 2 of Mbufs. */
+	unsigned int rss_hash:1; /* RSS hash result is enabled. */
+	unsigned int mark:1; /* Marked flow available on the queue. */
+	unsigned int strd_num_n:5; /* Log 2 of the number of stride. */
+	unsigned int strd_sz_n:4; /* Log 2 of stride size. */
+	unsigned int strd_shift_en:1; /* Enable 2bytes shift on a stride. */
+	unsigned int :6; /* Remaining bits. */
+	volatile uint32_t *rq_db;
+	volatile uint32_t *cq_db;
+	uint16_t port_id;
+	uint16_t rq_ci;
+	uint16_t consumed_strd; /* Number of consumed strides in WQE. */
+	uint16_t rq_pi;
+	uint16_t cq_ci;
+	struct mlx5_mr_ctrl mr_ctrl; /* MR control descriptor. */
+	uint16_t mprq_max_memcpy_len; /* Maximum size of packet to memcpy. */
+	volatile void *wqes;
+	volatile struct mlx5_cqe(*cqes)[];
+	struct rxq_zip zip; /* Compressed context. */
+	RTE_STD_C11
+	union  {
+		struct rte_mbuf *(*elts)[];
+		struct mlx5_mprq_buf *(*mprq_bufs)[];
+	};
+	struct rte_mempool *mp;
+	struct rte_mempool *mprq_mp; /* Mempool for Multi-Packet RQ. */
+	struct mlx5_mprq_buf *mprq_repl; /* Stashed mbuf for replenish. */
+	struct mlx5_rxq_stats stats;
+	uint64_t mbuf_initializer; /* Default rearm_data for vectorized Rx. */
+	struct rte_mbuf fake_mbuf; /* elts padding for vectorized Rx. */
+	void *cq_uar; /* CQ user access region. */
+	uint32_t cqn; /* CQ number. */
+	uint8_t cq_arm_sn; /* CQ arm seq number. */
+#ifndef RTE_ARCH_64
+	rte_spinlock_t *uar_lock_cq;
+	/* CQ (UAR) access lock required for 32bit implementations */
+#endif
+	uint32_t tunnel; /* Tunnel information. */
+} __rte_cache_aligned;
+
+/* Verbs Rx queue elements. */
+struct mlx5_rxq_ibv {
+	LIST_ENTRY(mlx5_rxq_ibv) next; /* Pointer to the next element. */
+	rte_atomic32_t refcnt; /* Reference counter. */
+	struct mlx5_rxq_ctrl *rxq_ctrl; /* Back pointer to parent. */
+	struct ibv_cq *cq; /* Completion Queue. */
+	struct ibv_wq *wq; /* Work Queue. */
+	struct ibv_comp_channel *channel;
+};
+
+/* RX queue control descriptor. */
+struct mlx5_rxq_ctrl {
+	LIST_ENTRY(mlx5_rxq_ctrl) next; /* Pointer to the next element. */
+	rte_atomic32_t refcnt; /* Reference counter. */
+	struct mlx5_rxq_ibv *ibv; /* Verbs elements. */
+	struct priv *priv; /* Back pointer to private data. */
+	struct mlx5_rxq_data rxq; /* Data path structure. */
+	unsigned int socket; /* CPU socket ID for allocations. */
+	unsigned int irq:1; /* Whether IRQ is enabled. */
+	uint16_t idx; /* Queue index. */
+	uint32_t flow_mark_n; /* Number of Mark/Flag flows using this Queue. */
+	uint32_t flow_tunnels_n[MLX5_FLOW_TUNNEL]; /* Tunnels counters. */
+};
+
+/* Indirection table. */
+struct mlx5_ind_table_ibv {
+	LIST_ENTRY(mlx5_ind_table_ibv) next; /* Pointer to the next element. */
+	rte_atomic32_t refcnt; /* Reference counter. */
+	struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */
+	uint32_t queues_n; /**< Number of queues in the list. */
+	uint16_t queues[]; /**< Queue list. */
+};
+
+/* Hash Rx queue. */
+struct mlx5_hrxq {
+	LIST_ENTRY(mlx5_hrxq) next; /* Pointer to the next element. */
+	rte_atomic32_t refcnt; /* Reference counter. */
+	struct mlx5_ind_table_ibv *ind_table; /* Indirection table. */
+	struct ibv_qp *qp; /* Verbs queue pair. */
+	uint64_t hash_fields; /* Verbs Hash fields. */
+	uint32_t rss_key_len; /* Hash key length in bytes. */
+	uint8_t rss_key[]; /* Hash key. */
+};
+
+/* TX queue descriptor. */
+__extension__
+struct mlx5_txq_data {
+	uint16_t elts_head; /* Current counter in (*elts)[]. */
+	uint16_t elts_tail; /* Counter of first element awaiting completion. */
+	uint16_t elts_comp; /* Counter since last completion request. */
+	uint16_t mpw_comp; /* WQ index since last completion request. */
+	uint16_t cq_ci; /* Consumer index for completion queue. */
+#ifndef NDEBUG
+	uint16_t cq_pi; /* Producer index for completion queue. */
+#endif
+	uint16_t wqe_ci; /* Consumer index for work queue. */
+	uint16_t wqe_pi; /* Producer index for work queue. */
+	uint16_t elts_n:4; /* (*elts)[] length (in log2). */
+	uint16_t cqe_n:4; /* Number of CQ elements (in log2). */
+	uint16_t wqe_n:4; /* Number of of WQ elements (in log2). */
+	uint16_t tso_en:1; /* When set hardware TSO is enabled. */
+	uint16_t tunnel_en:1;
+	/* When set TX offload for tunneled packets are supported. */
+	uint16_t swp_en:1; /* Whether SW parser is enabled. */
+	uint16_t mpw_hdr_dseg:1; /* Enable DSEGs in the title WQEBB. */
+	uint16_t max_inline; /* Multiple of RTE_CACHE_LINE_SIZE to inline. */
+	uint16_t inline_max_packet_sz; /* Max packet size for inlining. */
+	uint32_t qp_num_8s; /* QP number shifted by 8. */
+	uint64_t offloads; /* Offloads for Tx Queue. */
+	struct mlx5_mr_ctrl mr_ctrl; /* MR control descriptor. */
+	volatile struct mlx5_cqe (*cqes)[]; /* Completion queue. */
+	volatile void *wqes; /* Work queue (use volatile to write into). */
+	volatile uint32_t *qp_db; /* Work queue doorbell. */
+	volatile uint32_t *cq_db; /* Completion queue doorbell. */
+	volatile void *bf_reg; /* Blueflame register remapped. */
+	struct rte_mbuf *(*elts)[]; /* TX elements. */
+	struct mlx5_txq_stats stats; /* TX queue counters. */
+#ifndef RTE_ARCH_64
+	rte_spinlock_t *uar_lock;
+	/* UAR access lock required for 32bit implementations */
+#endif
+} __rte_cache_aligned;
+
+/* Verbs Rx queue elements. */
+struct mlx5_txq_ibv {
+	LIST_ENTRY(mlx5_txq_ibv) next; /* Pointer to the next element. */
+	rte_atomic32_t refcnt; /* Reference counter. */
+	struct mlx5_txq_ctrl *txq_ctrl; /* Pointer to the control queue. */
+	struct ibv_cq *cq; /* Completion Queue. */
+	struct ibv_qp *qp; /* Queue Pair. */
+};
+
+/* TX queue control descriptor. */
+struct mlx5_txq_ctrl {
+	LIST_ENTRY(mlx5_txq_ctrl) next; /* Pointer to the next element. */
+	rte_atomic32_t refcnt; /* Reference counter. */
+	unsigned int socket; /* CPU socket ID for allocations. */
+	unsigned int max_inline_data; /* Max inline data. */
+	unsigned int max_tso_header; /* Max TSO header size. */
+	struct mlx5_txq_ibv *ibv; /* Verbs queue object. */
+	struct priv *priv; /* Back pointer to private data. */
+	struct mlx5_txq_data txq; /* Data path structure. */
+	off_t uar_mmap_offset; /* UAR mmap offset for non-primary process. */
+	volatile void *bf_reg_orig; /* Blueflame register from verbs. */
+	uint16_t idx; /* Queue index. */
+};
+
+/* mlx5_rxq.c */
+
+extern uint8_t rss_hash_default_key[];
+
+int mlx5_check_mprq_support(struct rte_eth_dev *dev);
+int mlx5_rxq_mprq_enabled(struct mlx5_rxq_data *rxq);
+int mlx5_mprq_enabled(struct rte_eth_dev *dev);
+int mlx5_mprq_free_mp(struct rte_eth_dev *dev);
+int mlx5_mprq_alloc_mp(struct rte_eth_dev *dev);
+void mlx5_rxq_cleanup(struct mlx5_rxq_ctrl *rxq_ctrl);
+int mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
+			unsigned int socket, const struct rte_eth_rxconf *conf,
+			struct rte_mempool *mp);
+void mlx5_rx_queue_release(void *dpdk_rxq);
+int mlx5_rx_intr_vec_enable(struct rte_eth_dev *dev);
+void mlx5_rx_intr_vec_disable(struct rte_eth_dev *dev);
+int mlx5_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id);
+int mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id);
+struct mlx5_rxq_ibv *mlx5_rxq_ibv_new(struct rte_eth_dev *dev, uint16_t idx);
+struct mlx5_rxq_ibv *mlx5_rxq_ibv_get(struct rte_eth_dev *dev, uint16_t idx);
+int mlx5_rxq_ibv_release(struct mlx5_rxq_ibv *rxq_ibv);
+int mlx5_rxq_ibv_releasable(struct mlx5_rxq_ibv *rxq_ibv);
+struct mlx5_rxq_ibv *mlx5_rxq_ibv_drop_new(struct rte_eth_dev *dev);
+void mlx5_rxq_ibv_drop_release(struct rte_eth_dev *dev);
+int mlx5_rxq_ibv_verify(struct rte_eth_dev *dev);
+struct mlx5_rxq_ctrl *mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx,
+				   uint16_t desc, unsigned int socket,
+				   const struct rte_eth_rxconf *conf,
+				   struct rte_mempool *mp);
+struct mlx5_rxq_ctrl *mlx5_rxq_get(struct rte_eth_dev *dev, uint16_t idx);
+int mlx5_rxq_release(struct rte_eth_dev *dev, uint16_t idx);
+int mlx5_rxq_releasable(struct rte_eth_dev *dev, uint16_t idx);
+int mlx5_rxq_verify(struct rte_eth_dev *dev);
+int rxq_alloc_elts(struct mlx5_rxq_ctrl *rxq_ctrl);
+int rxq_alloc_mprq_buf(struct mlx5_rxq_ctrl *rxq_ctrl);
+struct mlx5_ind_table_ibv *mlx5_ind_table_ibv_new(struct rte_eth_dev *dev,
+						  const uint16_t *queues,
+						  uint32_t queues_n);
+struct mlx5_ind_table_ibv *mlx5_ind_table_ibv_get(struct rte_eth_dev *dev,
+						  const uint16_t *queues,
+						  uint32_t queues_n);
+int mlx5_ind_table_ibv_release(struct rte_eth_dev *dev,
+			       struct mlx5_ind_table_ibv *ind_tbl);
+int mlx5_ind_table_ibv_verify(struct rte_eth_dev *dev);
+struct mlx5_ind_table_ibv *mlx5_ind_table_ibv_drop_new(struct rte_eth_dev *dev);
+void mlx5_ind_table_ibv_drop_release(struct rte_eth_dev *dev);
+struct mlx5_hrxq *mlx5_hrxq_new(struct rte_eth_dev *dev,
+				const uint8_t *rss_key, uint32_t rss_key_len,
+				uint64_t hash_fields,
+				const uint16_t *queues, uint32_t queues_n,
+				int tunnel __rte_unused);
+struct mlx5_hrxq *mlx5_hrxq_get(struct rte_eth_dev *dev,
+				const uint8_t *rss_key, uint32_t rss_key_len,
+				uint64_t hash_fields,
+				const uint16_t *queues, uint32_t queues_n);
+int mlx5_hrxq_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hxrq);
+int mlx5_hrxq_ibv_verify(struct rte_eth_dev *dev);
+struct mlx5_hrxq *mlx5_hrxq_drop_new(struct rte_eth_dev *dev);
+void mlx5_hrxq_drop_release(struct rte_eth_dev *dev);
+uint64_t mlx5_get_rx_port_offloads(void);
+uint64_t mlx5_get_rx_queue_offloads(struct rte_eth_dev *dev);
+
+/* mlx5_txq.c */
+
+int mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
+			unsigned int socket, const struct rte_eth_txconf *conf);
+void mlx5_tx_queue_release(void *dpdk_txq);
+int mlx5_tx_uar_remap(struct rte_eth_dev *dev, int fd);
+struct mlx5_txq_ibv *mlx5_txq_ibv_new(struct rte_eth_dev *dev, uint16_t idx);
+struct mlx5_txq_ibv *mlx5_txq_ibv_get(struct rte_eth_dev *dev, uint16_t idx);
+int mlx5_txq_ibv_release(struct mlx5_txq_ibv *txq_ibv);
+int mlx5_txq_ibv_releasable(struct mlx5_txq_ibv *txq_ibv);
+int mlx5_txq_ibv_verify(struct rte_eth_dev *dev);
+struct mlx5_txq_ctrl *mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx,
+				   uint16_t desc, unsigned int socket,
+				   const struct rte_eth_txconf *conf);
+struct mlx5_txq_ctrl *mlx5_txq_get(struct rte_eth_dev *dev, uint16_t idx);
+int mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx);
+int mlx5_txq_releasable(struct rte_eth_dev *dev, uint16_t idx);
+int mlx5_txq_verify(struct rte_eth_dev *dev);
+void txq_alloc_elts(struct mlx5_txq_ctrl *txq_ctrl);
+uint64_t mlx5_get_tx_port_offloads(struct rte_eth_dev *dev);
+
+/* mlx5_rxtx.c */
+
+extern uint32_t mlx5_ptype_table[];
+extern uint8_t mlx5_cksum_table[];
+extern uint8_t mlx5_swp_types_table[];
+
+void mlx5_set_ptype_table(void);
+void mlx5_set_cksum_table(void);
+void mlx5_set_swp_types_table(void);
+uint16_t mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts,
+		       uint16_t pkts_n);
+uint16_t mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts,
+			   uint16_t pkts_n);
+uint16_t mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
+				  uint16_t pkts_n);
+uint16_t mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts,
+			    uint16_t pkts_n);
+uint16_t mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n);
+void mlx5_mprq_buf_free_cb(void *addr, void *opaque);
+void mlx5_mprq_buf_free(struct mlx5_mprq_buf *buf);
+uint16_t mlx5_rx_burst_mprq(void *dpdk_rxq, struct rte_mbuf **pkts,
+			    uint16_t pkts_n);
+uint16_t removed_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts,
+			  uint16_t pkts_n);
+uint16_t removed_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts,
+			  uint16_t pkts_n);
+int mlx5_rx_descriptor_status(void *rx_queue, uint16_t offset);
+int mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset);
+
+/* Vectorized version of mlx5_rxtx.c */
+int mlx5_check_raw_vec_tx_support(struct rte_eth_dev *dev);
+int mlx5_check_vec_tx_support(struct rte_eth_dev *dev);
+int mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq_data);
+int mlx5_check_vec_rx_support(struct rte_eth_dev *dev);
+uint16_t mlx5_tx_burst_raw_vec(void *dpdk_txq, struct rte_mbuf **pkts,
+			       uint16_t pkts_n);
+uint16_t mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts,
+			   uint16_t pkts_n);
+uint16_t mlx5_rx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts,
+			   uint16_t pkts_n);
+
+/* mlx5_mr.c */
+
+void mlx5_mr_flush_local_cache(struct mlx5_mr_ctrl *mr_ctrl);
+uint32_t mlx5_rx_addr2mr_bh(struct mlx5_rxq_data *rxq, uintptr_t addr);
+uint32_t mlx5_tx_addr2mr_bh(struct mlx5_txq_data *txq, uintptr_t addr);
+
+/**
+ * Provide safe 64bit store operation to mlx5 UAR region for both 32bit and
+ * 64bit architectures.
+ *
+ * @param val
+ *   value to write in CPU endian format.
+ * @param addr
+ *   Address to write to.
+ * @param lock
+ *   Address of the lock to use for that UAR access.
+ */
+static __rte_always_inline void
+__mlx5_uar_write64_relaxed(uint64_t val, volatile void *addr,
+			   rte_spinlock_t *lock __rte_unused)
+{
+#ifdef RTE_ARCH_64
+	rte_write64_relaxed(val, addr);
+#else /* !RTE_ARCH_64 */
+	rte_spinlock_lock(lock);
+	rte_write32_relaxed(val, addr);
+	rte_io_wmb();
+	rte_write32_relaxed(val >> 32,
+			    (volatile void *)((volatile char *)addr + 4));
+	rte_spinlock_unlock(lock);
+#endif
+}
+
+/**
+ * Provide safe 64bit store operation to mlx5 UAR region for both 32bit and
+ * 64bit architectures while guaranteeing the order of execution with the
+ * code being executed.
+ *
+ * @param val
+ *   value to write in CPU endian format.
+ * @param addr
+ *   Address to write to.
+ * @param lock
+ *   Address of the lock to use for that UAR access.
+ */
+static __rte_always_inline void
+__mlx5_uar_write64(uint64_t val, volatile void *addr, rte_spinlock_t *lock)
+{
+	rte_io_wmb();
+	__mlx5_uar_write64_relaxed(val, addr, lock);
+}
+
+/* Assist macros, used instead of directly calling the functions they wrap. */
+#ifdef RTE_ARCH_64
+#define mlx5_uar_write64_relaxed(val, dst, lock) \
+		__mlx5_uar_write64_relaxed(val, dst, NULL)
+#define mlx5_uar_write64(val, dst, lock) __mlx5_uar_write64(val, dst, NULL)
+#else
+#define mlx5_uar_write64_relaxed(val, dst, lock) \
+		__mlx5_uar_write64_relaxed(val, dst, lock)
+#define mlx5_uar_write64(val, dst, lock) __mlx5_uar_write64(val, dst, lock)
+#endif
+
+#ifndef NDEBUG
+/**
+ * Verify or set magic value in CQE.
+ *
+ * @param cqe
+ *   Pointer to CQE.
+ *
+ * @return
+ *   0 the first time.
+ */
+static inline int
+check_cqe_seen(volatile struct mlx5_cqe *cqe)
+{
+	static const uint8_t magic[] = "seen";
+	volatile uint8_t (*buf)[sizeof(cqe->rsvd1)] = &cqe->rsvd1;
+	int ret = 1;
+	unsigned int i;
+
+	for (i = 0; i < sizeof(magic) && i < sizeof(*buf); ++i)
+		if (!ret || (*buf)[i] != magic[i]) {
+			ret = 0;
+			(*buf)[i] = magic[i];
+		}
+	return ret;
+}
+#endif /* NDEBUG */
+
+/**
+ * Check whether CQE is valid.
+ *
+ * @param cqe
+ *   Pointer to CQE.
+ * @param cqes_n
+ *   Size of completion queue.
+ * @param ci
+ *   Consumer index.
+ *
+ * @return
+ *   0 on success, 1 on failure.
+ */
+static __rte_always_inline int
+check_cqe(volatile struct mlx5_cqe *cqe,
+	  unsigned int cqes_n, const uint16_t ci)
+{
+	uint16_t idx = ci & cqes_n;
+	uint8_t op_own = cqe->op_own;
+	uint8_t op_owner = MLX5_CQE_OWNER(op_own);
+	uint8_t op_code = MLX5_CQE_OPCODE(op_own);
+
+	if (unlikely((op_owner != (!!(idx))) || (op_code == MLX5_CQE_INVALID)))
+		return 1; /* No CQE. */
+#ifndef NDEBUG
+	if ((op_code == MLX5_CQE_RESP_ERR) ||
+	    (op_code == MLX5_CQE_REQ_ERR)) {
+		volatile struct mlx5_err_cqe *err_cqe = (volatile void *)cqe;
+		uint8_t syndrome = err_cqe->syndrome;
+
+		if ((syndrome == MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR) ||
+		    (syndrome == MLX5_CQE_SYNDROME_REMOTE_ABORTED_ERR))
+			return 0;
+		if (!check_cqe_seen(cqe)) {
+			DRV_LOG(ERR,
+				"unexpected CQE error %u (0x%02x) syndrome"
+				" 0x%02x",
+				op_code, op_code, syndrome);
+			rte_hexdump(stderr, "MLX5 Error CQE:",
+				    (const void *)((uintptr_t)err_cqe),
+				    sizeof(*err_cqe));
+		}
+		return 1;
+	} else if ((op_code != MLX5_CQE_RESP_SEND) &&
+		   (op_code != MLX5_CQE_REQ)) {
+		if (!check_cqe_seen(cqe)) {
+			DRV_LOG(ERR, "unexpected CQE opcode %u (0x%02x)",
+				op_code, op_code);
+			rte_hexdump(stderr, "MLX5 CQE:",
+				    (const void *)((uintptr_t)cqe),
+				    sizeof(*cqe));
+		}
+		return 1;
+	}
+#endif /* NDEBUG */
+	return 0;
+}
+
+/**
+ * Return the address of the WQE.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param  wqe_ci
+ *   WQE consumer index.
+ *
+ * @return
+ *   WQE address.
+ */
+static inline uintptr_t *
+tx_mlx5_wqe(struct mlx5_txq_data *txq, uint16_t ci)
+{
+	ci &= ((1 << txq->wqe_n) - 1);
+	return (uintptr_t *)((uintptr_t)txq->wqes + ci * MLX5_WQE_SIZE);
+}
+
+/**
+ * Manage TX completions.
+ *
+ * When sending a burst, mlx5_tx_burst() posts several WRs.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ */
+static __rte_always_inline void
+mlx5_tx_complete(struct mlx5_txq_data *txq)
+{
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
+	const unsigned int cqe_n = 1 << txq->cqe_n;
+	const unsigned int cqe_cnt = cqe_n - 1;
+	uint16_t elts_free = txq->elts_tail;
+	uint16_t elts_tail;
+	uint16_t cq_ci = txq->cq_ci;
+	volatile struct mlx5_cqe *cqe = NULL;
+	volatile struct mlx5_wqe_ctrl *ctrl;
+	struct rte_mbuf *m, *free[elts_n];
+	struct rte_mempool *pool = NULL;
+	unsigned int blk_n = 0;
+
+	cqe = &(*txq->cqes)[cq_ci & cqe_cnt];
+	if (unlikely(check_cqe(cqe, cqe_n, cq_ci)))
+		return;
+#ifndef NDEBUG
+	if ((MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_RESP_ERR) ||
+	    (MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_REQ_ERR)) {
+		if (!check_cqe_seen(cqe)) {
+			DRV_LOG(ERR, "unexpected error CQE, Tx stopped");
+			rte_hexdump(stderr, "MLX5 TXQ:",
+				    (const void *)((uintptr_t)txq->wqes),
+				    ((1 << txq->wqe_n) *
+				     MLX5_WQE_SIZE));
+		}
+		return;
+	}
+#endif /* NDEBUG */
+	++cq_ci;
+	txq->wqe_pi = rte_be_to_cpu_16(cqe->wqe_counter);
+	ctrl = (volatile struct mlx5_wqe_ctrl *)
+		tx_mlx5_wqe(txq, txq->wqe_pi);
+	elts_tail = ctrl->ctrl3;
+	assert((elts_tail & elts_m) < (1 << txq->wqe_n));
+	/* Free buffers. */
+	while (elts_free != elts_tail) {
+		m = rte_pktmbuf_prefree_seg((*txq->elts)[elts_free++ & elts_m]);
+		if (likely(m != NULL)) {
+			if (likely(m->pool == pool)) {
+				free[blk_n++] = m;
+			} else {
+				if (likely(pool != NULL))
+					rte_mempool_put_bulk(pool,
+							     (void *)free,
+							     blk_n);
+				free[0] = m;
+				pool = m->pool;
+				blk_n = 1;
+			}
+		}
+	}
+	if (blk_n)
+		rte_mempool_put_bulk(pool, (void *)free, blk_n);
+#ifndef NDEBUG
+	elts_free = txq->elts_tail;
+	/* Poisoning. */
+	while (elts_free != elts_tail) {
+		memset(&(*txq->elts)[elts_free & elts_m],
+		       0x66,
+		       sizeof((*txq->elts)[elts_free & elts_m]));
+		++elts_free;
+	}
+#endif
+	txq->cq_ci = cq_ci;
+	txq->elts_tail = elts_tail;
+	/* Update the consumer index. */
+	rte_compiler_barrier();
+	*txq->cq_db = rte_cpu_to_be_32(cq_ci);
+}
+
+/**
+ * Query LKey from a packet buffer for Rx. No need to flush local caches for Rx
+ * as mempool is pre-configured and static.
+ *
+ * @param rxq
+ *   Pointer to Rx queue structure.
+ * @param addr
+ *   Address to search.
+ *
+ * @return
+ *   Searched LKey on success, UINT32_MAX on no match.
+ */
+static __rte_always_inline uint32_t
+mlx5_rx_addr2mr(struct mlx5_rxq_data *rxq, uintptr_t addr)
+{
+	struct mlx5_mr_ctrl *mr_ctrl = &rxq->mr_ctrl;
+	uint32_t lkey;
+
+	/* Linear search on MR cache array. */
+	lkey = mlx5_mr_lookup_cache(mr_ctrl->cache, &mr_ctrl->mru,
+				    MLX5_MR_CACHE_N, addr);
+	if (likely(lkey != UINT32_MAX))
+		return lkey;
+	/* Take slower bottom-half (Binary Search) on miss. */
+	return mlx5_rx_addr2mr_bh(rxq, addr);
+}
+
+#define mlx5_rx_mb2mr(rxq, mb) mlx5_rx_addr2mr(rxq, (uintptr_t)((mb)->buf_addr))
+
+/**
+ * Query LKey from a packet buffer for Tx. If not found, add the mempool.
+ *
+ * @param txq
+ *   Pointer to Tx queue structure.
+ * @param addr
+ *   Address to search.
+ *
+ * @return
+ *   Searched LKey on success, UINT32_MAX on no match.
+ */
+static __rte_always_inline uint32_t
+mlx5_tx_addr2mr(struct mlx5_txq_data *txq, uintptr_t addr)
+{
+	struct mlx5_mr_ctrl *mr_ctrl = &txq->mr_ctrl;
+	uint32_t lkey;
+
+	/* Check generation bit to see if there's any change on existing MRs. */
+	if (unlikely(*mr_ctrl->dev_gen_ptr != mr_ctrl->cur_gen))
+		mlx5_mr_flush_local_cache(mr_ctrl);
+	/* Linear search on MR cache array. */
+	lkey = mlx5_mr_lookup_cache(mr_ctrl->cache, &mr_ctrl->mru,
+				    MLX5_MR_CACHE_N, addr);
+	if (likely(lkey != UINT32_MAX))
+		return lkey;
+	/* Take slower bottom-half (binary search) on miss. */
+	return mlx5_tx_addr2mr_bh(txq, addr);
+}
+
+#define mlx5_tx_mb2mr(rxq, mb) mlx5_tx_addr2mr(rxq, (uintptr_t)((mb)->buf_addr))
+
+/**
+ * Ring TX queue doorbell and flush the update if requested.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param wqe
+ *   Pointer to the last WQE posted in the NIC.
+ * @param cond
+ *   Request for write memory barrier after BlueFlame update.
+ */
+static __rte_always_inline void
+mlx5_tx_dbrec_cond_wmb(struct mlx5_txq_data *txq, volatile struct mlx5_wqe *wqe,
+		       int cond)
+{
+	uint64_t *dst = (uint64_t *)((uintptr_t)txq->bf_reg);
+	volatile uint64_t *src = ((volatile uint64_t *)wqe);
+
+	rte_cio_wmb();
+	*txq->qp_db = rte_cpu_to_be_32(txq->wqe_ci);
+	/* Ensure ordering between DB record and BF copy. */
+	rte_wmb();
+	mlx5_uar_write64_relaxed(*src, dst, txq->uar_lock);
+	if (cond)
+		rte_wmb();
+}
+
+/**
+ * Ring TX queue doorbell and flush the update by write memory barrier.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param wqe
+ *   Pointer to the last WQE posted in the NIC.
+ */
+static __rte_always_inline void
+mlx5_tx_dbrec(struct mlx5_txq_data *txq, volatile struct mlx5_wqe *wqe)
+{
+	mlx5_tx_dbrec_cond_wmb(txq, wqe, 1);
+}
+
+/**
+ * Convert mbuf to Verb SWP.
+ *
+ * @param txq_data
+ *   Pointer to the Tx queue.
+ * @param buf
+ *   Pointer to the mbuf.
+ * @param tso
+ *   TSO offloads enabled.
+ * @param vlan
+ *   VLAN offloads enabled
+ * @param offsets
+ *   Pointer to the SWP header offsets.
+ * @param swp_types
+ *   Pointer to the SWP header types.
+ */
+static __rte_always_inline void
+txq_mbuf_to_swp(struct mlx5_txq_data *txq, struct rte_mbuf *buf,
+		uint8_t *offsets, uint8_t *swp_types)
+{
+	const uint64_t vlan = buf->ol_flags & PKT_TX_VLAN_PKT;
+	const uint64_t tunnel = buf->ol_flags & PKT_TX_TUNNEL_MASK;
+	const uint64_t tso = buf->ol_flags & PKT_TX_TCP_SEG;
+	const uint64_t csum_flags = buf->ol_flags & PKT_TX_L4_MASK;
+	const uint64_t inner_ip =
+		buf->ol_flags & (PKT_TX_IPV4 | PKT_TX_IPV6);
+	const uint64_t ol_flags_mask = PKT_TX_L4_MASK | PKT_TX_IPV6 |
+				       PKT_TX_OUTER_IPV6;
+	uint16_t idx;
+	uint16_t off;
+
+	if (likely(!txq->swp_en || (tunnel != PKT_TX_TUNNEL_UDP &&
+				    tunnel != PKT_TX_TUNNEL_IP)))
+		return;
+	/*
+	 * The index should have:
+	 * bit[0:1] = PKT_TX_L4_MASK
+	 * bit[4] = PKT_TX_IPV6
+	 * bit[8] = PKT_TX_OUTER_IPV6
+	 * bit[9] = PKT_TX_OUTER_UDP
+	 */
+	idx = (buf->ol_flags & ol_flags_mask) >> 52;
+	if (tunnel == PKT_TX_TUNNEL_UDP)
+		idx |= 1 << 9;
+	*swp_types = mlx5_swp_types_table[idx];
+	/*
+	 * Set offsets for SW parser. Since ConnectX-5, SW parser just
+	 * complements HW parser. SW parser starts to engage only if HW parser
+	 * can't reach a header. For the older devices, HW parser will not kick
+	 * in if any of SWP offsets is set. Therefore, all of the L3 offsets
+	 * should be set regardless of HW offload.
+	 */
+	off = buf->outer_l2_len + (vlan ? sizeof(struct vlan_hdr) : 0);
+	offsets[1] = off >> 1; /* Outer L3 offset. */
+	off += buf->outer_l3_len;
+	if (tunnel == PKT_TX_TUNNEL_UDP)
+		offsets[0] = off >> 1; /* Outer L4 offset. */
+	if (inner_ip) {
+		off += buf->l2_len;
+		offsets[3] = off >> 1; /* Inner L3 offset. */
+		if (csum_flags == PKT_TX_TCP_CKSUM || tso ||
+		    csum_flags == PKT_TX_UDP_CKSUM) {
+			off += buf->l3_len;
+			offsets[2] = off >> 1; /* Inner L4 offset. */
+		}
+	}
+}
+
+/**
+ * Convert the Checksum offloads to Verbs.
+ *
+ * @param buf
+ *   Pointer to the mbuf.
+ *
+ * @return
+ *   Converted checksum flags.
+ */
+static __rte_always_inline uint8_t
+txq_ol_cksum_to_cs(struct rte_mbuf *buf)
+{
+	uint32_t idx;
+	uint8_t is_tunnel = !!(buf->ol_flags & PKT_TX_TUNNEL_MASK);
+	const uint64_t ol_flags_mask = PKT_TX_TCP_SEG | PKT_TX_L4_MASK |
+				       PKT_TX_IP_CKSUM | PKT_TX_OUTER_IP_CKSUM;
+
+	/*
+	 * The index should have:
+	 * bit[0] = PKT_TX_TCP_SEG
+	 * bit[2:3] = PKT_TX_UDP_CKSUM, PKT_TX_TCP_CKSUM
+	 * bit[4] = PKT_TX_IP_CKSUM
+	 * bit[8] = PKT_TX_OUTER_IP_CKSUM
+	 * bit[9] = tunnel
+	 */
+	idx = ((buf->ol_flags & ol_flags_mask) >> 50) | (!!is_tunnel << 9);
+	return mlx5_cksum_table[idx];
+}
+
+/**
+ * Count the number of contiguous single segment packets.
+ *
+ * @param pkts
+ *   Pointer to array of packets.
+ * @param pkts_n
+ *   Number of packets.
+ *
+ * @return
+ *   Number of contiguous single segment packets.
+ */
+static __rte_always_inline unsigned int
+txq_count_contig_single_seg(struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	unsigned int pos;
+
+	if (!pkts_n)
+		return 0;
+	/* Count the number of contiguous single segment packets. */
+	for (pos = 0; pos < pkts_n; ++pos)
+		if (NB_SEGS(pkts[pos]) > 1)
+			break;
+	return pos;
+}
+
+/**
+ * Count the number of contiguous multi-segment packets.
+ *
+ * @param pkts
+ *   Pointer to array of packets.
+ * @param pkts_n
+ *   Number of packets.
+ *
+ * @return
+ *   Number of contiguous multi-segment packets.
+ */
+static __rte_always_inline unsigned int
+txq_count_contig_multi_seg(struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	unsigned int pos;
+
+	if (!pkts_n)
+		return 0;
+	/* Count the number of contiguous multi-segment packets. */
+	for (pos = 0; pos < pkts_n; ++pos)
+		if (NB_SEGS(pkts[pos]) == 1)
+			break;
+	return pos;
+}
+
+#endif /* RTE_PMD_MLX5_RXTX_H_ */
diff --git a/src/spdk/dpdk/drivers/net/mlx5/mlx5_rxtx_vec.c b/src/spdk/dpdk/drivers/net/mlx5/mlx5_rxtx_vec.c
new file mode 100644
index 00000000..0a4aed8f
--- /dev/null
+++ b/src/spdk/dpdk/drivers/net/mlx5/mlx5_rxtx_vec.c
@@ -0,0 +1,316 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2017 6WIND S.A.
+ * Copyright 2017 Mellanox Technologies, Ltd
+ */
+
+#include <assert.h>
+#include <stdint.h>
+#include <string.h>
+#include <stdlib.h>
+
+/* Verbs header. */
+/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <infiniband/verbs.h>
+#include <infiniband/mlx5dv.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+#include <rte_mbuf.h>
+#include <rte_mempool.h>
+#include <rte_prefetch.h>
+
+#include "mlx5.h"
+#include "mlx5_utils.h"
+#include "mlx5_rxtx.h"
+#include "mlx5_rxtx_vec.h"
+#include "mlx5_autoconf.h"
+#include "mlx5_defs.h"
+#include "mlx5_prm.h"
+
+#if defined RTE_ARCH_X86_64
+#include "mlx5_rxtx_vec_sse.h"
+#elif defined RTE_ARCH_ARM64
+#include "mlx5_rxtx_vec_neon.h"
+#else
+#error "This should not be compiled if SIMD instructions are not supported."
+#endif
+
+/**
+ * Count the number of packets having same ol_flags and calculate cs_flags.
+ *
+ * @param pkts
+ *   Pointer to array of packets.
+ * @param pkts_n
+ *   Number of packets.
+ * @param cs_flags
+ *   Pointer of flags to be returned.
+ *
+ * @return
+ *   Number of packets having same ol_flags.
+ */
+static inline unsigned int
+txq_calc_offload(struct rte_mbuf **pkts, uint16_t pkts_n, uint8_t *cs_flags)
+{
+	unsigned int pos;
+	const uint64_t ol_mask =
+		PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM |
+		PKT_TX_UDP_CKSUM | PKT_TX_TUNNEL_GRE |
+		PKT_TX_TUNNEL_VXLAN | PKT_TX_OUTER_IP_CKSUM;
+
+	if (!pkts_n)
+		return 0;
+	/* Count the number of packets having same ol_flags. */
+	for (pos = 1; pos < pkts_n; ++pos)
+		if ((pkts[pos]->ol_flags ^ pkts[0]->ol_flags) & ol_mask)
+			break;
+	*cs_flags = txq_ol_cksum_to_cs(pkts[0]);
+	return pos;
+}
+
+/**
+ * DPDK callback for vectorized TX.
+ *
+ * @param dpdk_txq
+ *   Generic pointer to TX queue structure.
+ * @param[in] pkts
+ *   Packets to transmit.
+ * @param pkts_n
+ *   Number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully transmitted (<= pkts_n).
+ */
+uint16_t
+mlx5_tx_burst_raw_vec(void *dpdk_txq, struct rte_mbuf **pkts,
+		      uint16_t pkts_n)
+{
+	struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq;
+	uint16_t nb_tx = 0;
+
+	while (pkts_n > nb_tx) {
+		uint16_t n;
+		uint16_t ret;
+
+		n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST);
+		ret = txq_burst_v(txq, &pkts[nb_tx], n, 0);
+		nb_tx += ret;
+		if (!ret)
+			break;
+	}
+	return nb_tx;
+}
+
+/**
+ * DPDK callback for vectorized TX with multi-seg packets and offload.
+ *
+ * @param dpdk_txq
+ *   Generic pointer to TX queue structure.
+ * @param[in] pkts
+ *   Packets to transmit.
+ * @param pkts_n
+ *   Number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully transmitted (<= pkts_n).
+ */
+uint16_t
+mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq;
+	uint16_t nb_tx = 0;
+
+	while (pkts_n > nb_tx) {
+		uint8_t cs_flags = 0;
+		uint16_t n;
+		uint16_t ret;
+
+		/* Transmit multi-seg packets in the head of pkts list. */
+		if ((txq->offloads & DEV_TX_OFFLOAD_MULTI_SEGS) &&
+		    NB_SEGS(pkts[nb_tx]) > 1)
+			nb_tx += txq_scatter_v(txq,
+					       &pkts[nb_tx],
+					       pkts_n - nb_tx);
+		n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST);
+		if (txq->offloads & DEV_TX_OFFLOAD_MULTI_SEGS)
+			n = txq_count_contig_single_seg(&pkts[nb_tx], n);
+		if (txq->offloads & MLX5_VEC_TX_CKSUM_OFFLOAD_CAP)
+			n = txq_calc_offload(&pkts[nb_tx], n, &cs_flags);
+		ret = txq_burst_v(txq, &pkts[nb_tx], n, cs_flags);
+		nb_tx += ret;
+		if (!ret)
+			break;
+	}
+	return nb_tx;
+}
+
+/**
+ * Skip error packets.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ * @param[out] pkts
+ *   Array to store received packets.
+ * @param pkts_n
+ *   Maximum number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully received (<= pkts_n).
+ */
+static uint16_t
+rxq_handle_pending_error(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts,
+			 uint16_t pkts_n)
+{
+	uint16_t n = 0;
+	unsigned int i;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	uint32_t err_bytes = 0;
+#endif
+
+	for (i = 0; i < pkts_n; ++i) {
+		struct rte_mbuf *pkt = pkts[i];
+
+		if (pkt->packet_type == RTE_PTYPE_ALL_MASK) {
+#ifdef MLX5_PMD_SOFT_COUNTERS
+			err_bytes += PKT_LEN(pkt);
+#endif
+			rte_pktmbuf_free_seg(pkt);
+		} else {
+			pkts[n++] = pkt;
+		}
+	}
+	rxq->stats.idropped += (pkts_n - n);
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	/* Correct counters of errored completions. */
+	rxq->stats.ipackets -= (pkts_n - n);
+	rxq->stats.ibytes -= err_bytes;
+#endif
+	return n;
+}
+
+/**
+ * DPDK callback for vectorized RX.
+ *
+ * @param dpdk_rxq
+ *   Generic pointer to RX queue structure.
+ * @param[out] pkts
+ *   Array to store received packets.
+ * @param pkts_n
+ *   Maximum number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully received (<= pkts_n).
+ */
+uint16_t
+mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	struct mlx5_rxq_data *rxq = dpdk_rxq;
+	uint16_t nb_rx;
+	uint64_t err = 0;
+
+	nb_rx = rxq_burst_v(rxq, pkts, pkts_n, &err);
+	if (unlikely(err))
+		nb_rx = rxq_handle_pending_error(rxq, pkts, nb_rx);
+	return nb_rx;
+}
+
+/**
+ * Check Tx queue flags are set for raw vectorized Tx.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   1 if supported, negative errno value if not.
+ */
+int __attribute__((cold))
+mlx5_check_raw_vec_tx_support(struct rte_eth_dev *dev)
+{
+	uint64_t offloads = dev->data->dev_conf.txmode.offloads;
+
+	/* Doesn't support any offload. */
+	if (offloads)
+		return -ENOTSUP;
+	return 1;
+}
+
+/**
+ * Check a device can support vectorized TX.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   1 if supported, negative errno value if not.
+ */
+int __attribute__((cold))
+mlx5_check_vec_tx_support(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	uint64_t offloads = dev->data->dev_conf.txmode.offloads;
+
+	if (!priv->config.tx_vec_en ||
+	    priv->txqs_n > MLX5_VPMD_MIN_TXQS ||
+	    priv->config.mps != MLX5_MPW_ENHANCED ||
+	    offloads & ~MLX5_VEC_TX_OFFLOAD_CAP)
+		return -ENOTSUP;
+	return 1;
+}
+
+/**
+ * Check a RX queue can support vectorized RX.
+ *
+ * @param rxq
+ *   Pointer to RX queue.
+ *
+ * @return
+ *   1 if supported, negative errno value if not.
+ */
+int __attribute__((cold))
+mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq)
+{
+	struct mlx5_rxq_ctrl *ctrl =
+		container_of(rxq, struct mlx5_rxq_ctrl, rxq);
+
+	if (mlx5_mprq_enabled(ETH_DEV(ctrl->priv)))
+		return -ENOTSUP;
+	if (!ctrl->priv->config.rx_vec_en || rxq->sges_n != 0)
+		return -ENOTSUP;
+	return 1;
+}
+
+/**
+ * Check a device can support vectorized RX.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   1 if supported, negative errno value if not.
+ */
+int __attribute__((cold))
+mlx5_check_vec_rx_support(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	uint16_t i;
+
+	if (!priv->config.rx_vec_en)
+		return -ENOTSUP;
+	if (mlx5_mprq_enabled(dev))
+		return -ENOTSUP;
+	/* All the configured queues should support. */
+	for (i = 0; i < priv->rxqs_n; ++i) {
+		struct mlx5_rxq_data *rxq = (*priv->rxqs)[i];
+
+		if (!rxq)
+			continue;
+		if (mlx5_rxq_check_vec_support(rxq) < 0)
+			break;
+	}
+	if (i != priv->rxqs_n)
+		return -ENOTSUP;
+	return 1;
+}
diff --git a/src/spdk/dpdk/drivers/net/mlx5/mlx5_rxtx_vec.h b/src/spdk/dpdk/drivers/net/mlx5/mlx5_rxtx_vec.h
new file mode 100644
index 00000000..fb884f92
--- /dev/null
+++ b/src/spdk/dpdk/drivers/net/mlx5/mlx5_rxtx_vec.h
@@ -0,0 +1,119 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2017 6WIND S.A.
+ * Copyright 2017 Mellanox Technologies, Ltd
+ */
+
+#ifndef RTE_PMD_MLX5_RXTX_VEC_H_
+#define RTE_PMD_MLX5_RXTX_VEC_H_
+
+#include <rte_common.h>
+#include <rte_mbuf.h>
+
+#include "mlx5_autoconf.h"
+#include "mlx5_prm.h"
+
+/* HW checksum offload capabilities of vectorized Tx. */
+#define MLX5_VEC_TX_CKSUM_OFFLOAD_CAP \
+	(DEV_TX_OFFLOAD_IPV4_CKSUM | \
+	 DEV_TX_OFFLOAD_UDP_CKSUM | \
+	 DEV_TX_OFFLOAD_TCP_CKSUM | \
+	 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM)
+
+/* HW offload capabilities of vectorized Tx. */
+#define MLX5_VEC_TX_OFFLOAD_CAP \
+	(MLX5_VEC_TX_CKSUM_OFFLOAD_CAP | \
+	 DEV_TX_OFFLOAD_MULTI_SEGS)
+
+/*
+ * Compile time sanity check for vectorized functions.
+ */
+
+#define S_ASSERT_RTE_MBUF(s) \
+	static_assert(s, "A field of struct rte_mbuf is changed")
+#define S_ASSERT_MLX5_CQE(s) \
+	static_assert(s, "A field of struct mlx5_cqe is changed")
+
+/* rxq_cq_decompress_v() */
+S_ASSERT_RTE_MBUF(offsetof(struct rte_mbuf, pkt_len) ==
+		  offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4);
+S_ASSERT_RTE_MBUF(offsetof(struct rte_mbuf, data_len) ==
+		  offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8);
+S_ASSERT_RTE_MBUF(offsetof(struct rte_mbuf, hash) ==
+		  offsetof(struct rte_mbuf, rx_descriptor_fields1) + 12);
+
+/* rxq_cq_to_ptype_oflags_v() */
+S_ASSERT_RTE_MBUF(offsetof(struct rte_mbuf, ol_flags) ==
+		  offsetof(struct rte_mbuf, rearm_data) + 8);
+S_ASSERT_RTE_MBUF(offsetof(struct rte_mbuf, rearm_data) ==
+		  RTE_ALIGN(offsetof(struct rte_mbuf, rearm_data), 16));
+
+/* rxq_burst_v() */
+S_ASSERT_RTE_MBUF(offsetof(struct rte_mbuf, pkt_len) ==
+		  offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4);
+S_ASSERT_RTE_MBUF(offsetof(struct rte_mbuf, data_len) ==
+		  offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8);
+#if (RTE_CACHE_LINE_SIZE == 128)
+S_ASSERT_MLX5_CQE(offsetof(struct mlx5_cqe, pkt_info) == 64);
+#else
+S_ASSERT_MLX5_CQE(offsetof(struct mlx5_cqe, pkt_info) == 0);
+#endif
+S_ASSERT_MLX5_CQE(offsetof(struct mlx5_cqe, rx_hash_res) ==
+		  offsetof(struct mlx5_cqe, pkt_info) + 12);
+S_ASSERT_MLX5_CQE(offsetof(struct mlx5_cqe, rsvd1) +
+		  sizeof(((struct mlx5_cqe *)0)->rsvd1) ==
+		  offsetof(struct mlx5_cqe, hdr_type_etc));
+S_ASSERT_MLX5_CQE(offsetof(struct mlx5_cqe, vlan_info) ==
+		  offsetof(struct mlx5_cqe, hdr_type_etc) + 2);
+S_ASSERT_MLX5_CQE(offsetof(struct mlx5_cqe, rsvd2) +
+		  sizeof(((struct mlx5_cqe *)0)->rsvd2) ==
+		  offsetof(struct mlx5_cqe, byte_cnt));
+S_ASSERT_MLX5_CQE(offsetof(struct mlx5_cqe, sop_drop_qpn) ==
+		  RTE_ALIGN(offsetof(struct mlx5_cqe, sop_drop_qpn), 8));
+S_ASSERT_MLX5_CQE(offsetof(struct mlx5_cqe, op_own) ==
+		  offsetof(struct mlx5_cqe, sop_drop_qpn) + 7);
+
+/**
+ * Replenish buffers for RX in bulk.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ * @param n
+ *   Number of buffers to be replenished.
+ */
+static inline void
+mlx5_rx_replenish_bulk_mbuf(struct mlx5_rxq_data *rxq, uint16_t n)
+{
+	const uint16_t q_n = 1 << rxq->elts_n;
+	const uint16_t q_mask = q_n - 1;
+	uint16_t elts_idx = rxq->rq_ci & q_mask;
+	struct rte_mbuf **elts = &(*rxq->elts)[elts_idx];
+	volatile struct mlx5_wqe_data_seg *wq =
+		&((volatile struct mlx5_wqe_data_seg *)rxq->wqes)[elts_idx];
+	unsigned int i;
+
+	assert(n >= MLX5_VPMD_RXQ_RPLNSH_THRESH(q_n));
+	assert(n <= (uint16_t)(q_n - (rxq->rq_ci - rxq->rq_pi)));
+	assert(MLX5_VPMD_RXQ_RPLNSH_THRESH(q_n) > MLX5_VPMD_DESCS_PER_LOOP);
+	/* Not to cross queue end. */
+	n = RTE_MIN(n - MLX5_VPMD_DESCS_PER_LOOP, q_n - elts_idx);
+	if (rte_mempool_get_bulk(rxq->mp, (void *)elts, n) < 0) {
+		rxq->stats.rx_nombuf += n;
+		return;
+	}
+	for (i = 0; i < n; ++i) {
+		wq[i].addr = rte_cpu_to_be_64((uintptr_t)elts[i]->buf_addr +
+					      RTE_PKTMBUF_HEADROOM);
+		/* If there's only one MR, no need to replace LKey in WQE. */
+		if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) > 1))
+			wq[i].lkey = mlx5_rx_mb2mr(rxq, elts[i]);
+	}
+	rxq->rq_ci += n;
+	/* Prevent overflowing into consumed mbufs. */
+	elts_idx = rxq->rq_ci & q_mask;
+	for (i = 0; i < MLX5_VPMD_DESCS_PER_LOOP; ++i)
+		(*rxq->elts)[elts_idx + i] = &rxq->fake_mbuf;
+	rte_cio_wmb();
+	*rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci);
+}
+
+#endif /* RTE_PMD_MLX5_RXTX_VEC_H_ */
diff --git a/src/spdk/dpdk/drivers/net/mlx5/mlx5_rxtx_vec_neon.h b/src/spdk/dpdk/drivers/net/mlx5/mlx5_rxtx_vec_neon.h
new file mode 100644
index 00000000..b37b7381
--- /dev/null
+++ b/src/spdk/dpdk/drivers/net/mlx5/mlx5_rxtx_vec_neon.h
@@ -0,0 +1,1017 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2017 6WIND S.A.
+ * Copyright 2017 Mellanox Technologies, Ltd
+ */
+
+#ifndef RTE_PMD_MLX5_RXTX_VEC_NEON_H_
+#define RTE_PMD_MLX5_RXTX_VEC_NEON_H_
+
+#include <assert.h>
+#include <stdint.h>
+#include <string.h>
+#include <stdlib.h>
+#include <arm_neon.h>
+
+#include <rte_mbuf.h>
+#include <rte_mempool.h>
+#include <rte_prefetch.h>
+
+#include "mlx5.h"
+#include "mlx5_utils.h"
+#include "mlx5_rxtx.h"
+#include "mlx5_rxtx_vec.h"
+#include "mlx5_autoconf.h"
+#include "mlx5_defs.h"
+#include "mlx5_prm.h"
+
+#pragma GCC diagnostic ignored "-Wcast-qual"
+
+/**
+ * Fill in buffer descriptors in a multi-packet send descriptor.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param dseg
+ *   Pointer to buffer descriptor to be written.
+ * @param pkts
+ *   Pointer to array of packets to be sent.
+ * @param n
+ *   Number of packets to be filled.
+ */
+static inline void
+txq_wr_dseg_v(struct mlx5_txq_data *txq, uint8_t *dseg,
+	      struct rte_mbuf **pkts, unsigned int n)
+{
+	unsigned int pos;
+	uintptr_t addr;
+	const uint8x16_t dseg_shuf_m = {
+		 3,  2,  1,  0, /* length, bswap32 */
+		 4,  5,  6,  7, /* lkey */
+		15, 14, 13, 12, /* addr, bswap64 */
+		11, 10,  9,  8
+	};
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	uint32_t tx_byte = 0;
+#endif
+
+	for (pos = 0; pos < n; ++pos, dseg += MLX5_WQE_DWORD_SIZE) {
+		uint8x16_t desc;
+		struct rte_mbuf *pkt = pkts[pos];
+
+		addr = rte_pktmbuf_mtod(pkt, uintptr_t);
+		desc = vreinterpretq_u8_u32((uint32x4_t) {
+				DATA_LEN(pkt),
+				mlx5_tx_mb2mr(txq, pkt),
+				addr,
+				addr >> 32 });
+		desc = vqtbl1q_u8(desc, dseg_shuf_m);
+		vst1q_u8(dseg, desc);
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		tx_byte += DATA_LEN(pkt);
+#endif
+	}
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	txq->stats.obytes += tx_byte;
+#endif
+}
+
+/**
+ * Send multi-segmented packets until it encounters a single segment packet in
+ * the pkts list.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param pkts
+ *   Pointer to array of packets to be sent.
+ * @param pkts_n
+ *   Number of packets to be sent.
+ *
+ * @return
+ *   Number of packets successfully transmitted (<= pkts_n).
+ */
+static uint16_t
+txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
+	      uint16_t pkts_n)
+{
+	uint16_t elts_head = txq->elts_head;
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
+	const uint16_t wq_n = 1 << txq->wqe_n;
+	const uint16_t wq_mask = wq_n - 1;
+	const unsigned int nb_dword_per_wqebb =
+		MLX5_WQE_SIZE / MLX5_WQE_DWORD_SIZE;
+	const unsigned int nb_dword_in_hdr =
+		sizeof(struct mlx5_wqe) / MLX5_WQE_DWORD_SIZE;
+	unsigned int n;
+	volatile struct mlx5_wqe *wqe = NULL;
+
+	assert(elts_n > pkts_n);
+	mlx5_tx_complete(txq);
+	if (unlikely(!pkts_n))
+		return 0;
+	for (n = 0; n < pkts_n; ++n) {
+		struct rte_mbuf *buf = pkts[n];
+		unsigned int segs_n = buf->nb_segs;
+		unsigned int ds = nb_dword_in_hdr;
+		unsigned int len = PKT_LEN(buf);
+		uint16_t wqe_ci = txq->wqe_ci;
+		const uint8x16_t ctrl_shuf_m = {
+			3,  2,  1,  0, /* bswap32 */
+			7,  6,  5,  4, /* bswap32 */
+			11, 10,  9,  8, /* bswap32 */
+			12, 13, 14, 15
+		};
+		uint8_t cs_flags;
+		uint16_t max_elts;
+		uint16_t max_wqe;
+		uint8x16_t *t_wqe;
+		uint8_t *dseg;
+		uint8x16_t ctrl;
+
+		assert(segs_n);
+		max_elts = elts_n - (elts_head - txq->elts_tail);
+		max_wqe = wq_n - (txq->wqe_ci - txq->wqe_pi);
+		/*
+		 * A MPW session consumes 2 WQEs at most to
+		 * include MLX5_MPW_DSEG_MAX pointers.
+		 */
+		if (segs_n == 1 ||
+		    max_elts < segs_n || max_wqe < 2)
+			break;
+		wqe = &((volatile struct mlx5_wqe64 *)
+			 txq->wqes)[wqe_ci & wq_mask].hdr;
+		cs_flags = txq_ol_cksum_to_cs(buf);
+		/* Title WQEBB pointer. */
+		t_wqe = (uint8x16_t *)wqe;
+		dseg = (uint8_t *)(wqe + 1);
+		do {
+			if (!(ds++ % nb_dword_per_wqebb)) {
+				dseg = (uint8_t *)
+					&((volatile struct mlx5_wqe64 *)
+					   txq->wqes)[++wqe_ci & wq_mask];
+			}
+			txq_wr_dseg_v(txq, dseg, &buf, 1);
+			dseg += MLX5_WQE_DWORD_SIZE;
+			(*txq->elts)[elts_head++ & elts_m] = buf;
+			buf = buf->next;
+		} while (--segs_n);
+		++wqe_ci;
+		/* Fill CTRL in the header. */
+		ctrl = vreinterpretq_u8_u32((uint32x4_t) {
+				MLX5_OPC_MOD_MPW << 24 |
+				txq->wqe_ci << 8 | MLX5_OPCODE_TSO,
+				txq->qp_num_8s | ds, 0, 0});
+		ctrl = vqtbl1q_u8(ctrl, ctrl_shuf_m);
+		vst1q_u8((void *)t_wqe, ctrl);
+		/* Fill ESEG in the header. */
+		vst1q_u16((void *)(t_wqe + 1),
+			  ((uint16x8_t) { 0, 0, cs_flags, rte_cpu_to_be_16(len),
+					  0, 0, 0, 0 }));
+		txq->wqe_ci = wqe_ci;
+	}
+	if (!n)
+		return 0;
+	txq->elts_comp += (uint16_t)(elts_head - txq->elts_head);
+	txq->elts_head = elts_head;
+	if (txq->elts_comp >= MLX5_TX_COMP_THRESH) {
+		/* A CQE slot must always be available. */
+		assert((1u << txq->cqe_n) - (txq->cq_pi++ - txq->cq_ci));
+		wqe->ctrl[2] = rte_cpu_to_be_32(8);
+		wqe->ctrl[3] = txq->elts_head;
+		txq->elts_comp = 0;
+	}
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	txq->stats.opackets += n;
+#endif
+	mlx5_tx_dbrec(txq, wqe);
+	return n;
+}
+
+/**
+ * Send burst of packets with Enhanced MPW. If it encounters a multi-seg packet,
+ * it returns to make it processed by txq_scatter_v(). All the packets in
+ * the pkts list should be single segment packets having same offload flags.
+ * This must be checked by txq_count_contig_single_seg() and txq_calc_offload().
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param pkts
+ *   Pointer to array of packets to be sent.
+ * @param pkts_n
+ *   Number of packets to be sent (<= MLX5_VPMD_TX_MAX_BURST).
+ * @param cs_flags
+ *   Checksum offload flags to be written in the descriptor.
+ *
+ * @return
+ *   Number of packets successfully transmitted (<= pkts_n).
+ */
+static inline uint16_t
+txq_burst_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, uint16_t pkts_n,
+	    uint8_t cs_flags)
+{
+	struct rte_mbuf **elts;
+	uint16_t elts_head = txq->elts_head;
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
+	const unsigned int nb_dword_per_wqebb =
+		MLX5_WQE_SIZE / MLX5_WQE_DWORD_SIZE;
+	const unsigned int nb_dword_in_hdr =
+		sizeof(struct mlx5_wqe) / MLX5_WQE_DWORD_SIZE;
+	unsigned int n = 0;
+	unsigned int pos;
+	uint16_t max_elts;
+	uint16_t max_wqe;
+	uint32_t comp_req = 0;
+	const uint16_t wq_n = 1 << txq->wqe_n;
+	const uint16_t wq_mask = wq_n - 1;
+	uint16_t wq_idx = txq->wqe_ci & wq_mask;
+	volatile struct mlx5_wqe64 *wq =
+		&((volatile struct mlx5_wqe64 *)txq->wqes)[wq_idx];
+	volatile struct mlx5_wqe *wqe = (volatile struct mlx5_wqe *)wq;
+	const uint8x16_t ctrl_shuf_m = {
+		 3,  2,  1,  0, /* bswap32 */
+		 7,  6,  5,  4, /* bswap32 */
+		11, 10,  9,  8, /* bswap32 */
+		12, 13, 14, 15
+	};
+	uint8x16_t *t_wqe;
+	uint8_t *dseg;
+	uint8x16_t ctrl;
+
+	/* Make sure all packets can fit into a single WQE. */
+	assert(elts_n > pkts_n);
+	mlx5_tx_complete(txq);
+	max_elts = (elts_n - (elts_head - txq->elts_tail));
+	max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
+	pkts_n = RTE_MIN((unsigned int)RTE_MIN(pkts_n, max_wqe), max_elts);
+	if (unlikely(!pkts_n))
+		return 0;
+	elts = &(*txq->elts)[elts_head & elts_m];
+	/* Loop for available tailroom first. */
+	n = RTE_MIN(elts_n - (elts_head & elts_m), pkts_n);
+	for (pos = 0; pos < (n & -2); pos += 2)
+		vst1q_u64((void *)&elts[pos], vld1q_u64((void *)&pkts[pos]));
+	if (n & 1)
+		elts[pos] = pkts[pos];
+	/* Check if it crosses the end of the queue. */
+	if (unlikely(n < pkts_n)) {
+		elts = &(*txq->elts)[0];
+		for (pos = 0; pos < pkts_n - n; ++pos)
+			elts[pos] = pkts[n + pos];
+	}
+	txq->elts_head += pkts_n;
+	/* Save title WQEBB pointer. */
+	t_wqe = (uint8x16_t *)wqe;
+	dseg = (uint8_t *)(wqe + 1);
+	/* Calculate the number of entries to the end. */
+	n = RTE_MIN(
+		(wq_n - wq_idx) * nb_dword_per_wqebb - nb_dword_in_hdr,
+		pkts_n);
+	/* Fill DSEGs. */
+	txq_wr_dseg_v(txq, dseg, pkts, n);
+	/* Check if it crosses the end of the queue. */
+	if (n < pkts_n) {
+		dseg = (uint8_t *)txq->wqes;
+		txq_wr_dseg_v(txq, dseg, &pkts[n], pkts_n - n);
+	}
+	if (txq->elts_comp + pkts_n < MLX5_TX_COMP_THRESH) {
+		txq->elts_comp += pkts_n;
+	} else {
+		/* A CQE slot must always be available. */
+		assert((1u << txq->cqe_n) - (txq->cq_pi++ - txq->cq_ci));
+		/* Request a completion. */
+		txq->elts_comp = 0;
+		comp_req = 8;
+	}
+	/* Fill CTRL in the header. */
+	ctrl = vreinterpretq_u8_u32((uint32x4_t) {
+			MLX5_OPC_MOD_ENHANCED_MPSW << 24 |
+			txq->wqe_ci << 8 | MLX5_OPCODE_ENHANCED_MPSW,
+			txq->qp_num_8s | (pkts_n + 2),
+			comp_req,
+			txq->elts_head });
+	ctrl = vqtbl1q_u8(ctrl, ctrl_shuf_m);
+	vst1q_u8((void *)t_wqe, ctrl);
+	/* Fill ESEG in the header. */
+	vst1q_u8((void *)(t_wqe + 1),
+		 ((uint8x16_t) { 0, 0, 0, 0,
+				 cs_flags, 0, 0, 0,
+				 0, 0, 0, 0,
+				 0, 0, 0, 0 }));
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	txq->stats.opackets += pkts_n;
+#endif
+	txq->wqe_ci += (nb_dword_in_hdr + pkts_n + (nb_dword_per_wqebb - 1)) /
+		       nb_dword_per_wqebb;
+	/* Ring QP doorbell. */
+	mlx5_tx_dbrec_cond_wmb(txq, wqe, pkts_n < MLX5_VPMD_TX_MAX_BURST);
+	return pkts_n;
+}
+
+/**
+ * Store free buffers to RX SW ring.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ * @param pkts
+ *   Pointer to array of packets to be stored.
+ * @param pkts_n
+ *   Number of packets to be stored.
+ */
+static inline void
+rxq_copy_mbuf_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t n)
+{
+	const uint16_t q_mask = (1 << rxq->elts_n) - 1;
+	struct rte_mbuf **elts = &(*rxq->elts)[rxq->rq_pi & q_mask];
+	unsigned int pos;
+	uint16_t p = n & -2;
+
+	for (pos = 0; pos < p; pos += 2) {
+		uint64x2_t mbp;
+
+		mbp = vld1q_u64((void *)&elts[pos]);
+		vst1q_u64((void *)&pkts[pos], mbp);
+	}
+	if (n & 1)
+		pkts[pos] = elts[pos];
+}
+
+/**
+ * Decompress a compressed completion and fill in mbufs in RX SW ring with data
+ * extracted from the title completion descriptor.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ * @param cq
+ *   Pointer to completion array having a compressed completion at first.
+ * @param elts
+ *   Pointer to SW ring to be filled. The first mbuf has to be pre-built from
+ *   the title completion descriptor to be copied to the rest of mbufs.
+ */
+static inline void
+rxq_cq_decompress_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq,
+		    struct rte_mbuf **elts)
+{
+	volatile struct mlx5_mini_cqe8 *mcq = (void *)&(cq + 1)->pkt_info;
+	struct rte_mbuf *t_pkt = elts[0]; /* Title packet is pre-built. */
+	unsigned int pos;
+	unsigned int i;
+	unsigned int inv = 0;
+	/* Mask to shuffle from extracted mini CQE to mbuf. */
+	const uint8x16_t mcqe_shuf_m1 = {
+		-1, -1, -1, -1, /* skip packet_type */
+		 7,  6, -1, -1, /* pkt_len, bswap16 */
+		 7,  6,         /* data_len, bswap16 */
+		-1, -1,         /* skip vlan_tci */
+		 3,  2,  1,  0  /* hash.rss, bswap32 */
+	};
+	const uint8x16_t mcqe_shuf_m2 = {
+		-1, -1, -1, -1, /* skip packet_type */
+		15, 14, -1, -1, /* pkt_len, bswap16 */
+		15, 14,         /* data_len, bswap16 */
+		-1, -1,         /* skip vlan_tci */
+		11, 10,  9,  8  /* hash.rss, bswap32 */
+	};
+	/* Restore the compressed count. Must be 16 bits. */
+	const uint16_t mcqe_n = t_pkt->data_len +
+				(rxq->crc_present * ETHER_CRC_LEN);
+	const uint64x2_t rearm =
+		vld1q_u64((void *)&t_pkt->rearm_data);
+	const uint32x4_t rxdf_mask = {
+		0xffffffff, /* packet_type */
+		0,          /* skip pkt_len */
+		0xffff0000, /* vlan_tci, skip data_len */
+		0,          /* skip hash.rss */
+	};
+	const uint8x16_t rxdf =
+		vandq_u8(vld1q_u8((void *)&t_pkt->rx_descriptor_fields1),
+			 vreinterpretq_u8_u32(rxdf_mask));
+	const uint16x8_t crc_adj = {
+		0, 0,
+		rxq->crc_present * ETHER_CRC_LEN, 0,
+		rxq->crc_present * ETHER_CRC_LEN, 0,
+		0, 0
+	};
+	const uint32_t flow_tag = t_pkt->hash.fdir.hi;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	uint32_t rcvd_byte = 0;
+#endif
+	/* Mask to shuffle byte_cnt to add up stats. Do bswap16 for all. */
+	const uint8x8_t len_shuf_m = {
+		 7,  6,         /* 1st mCQE */
+		15, 14,         /* 2nd mCQE */
+		23, 22,         /* 3rd mCQE */
+		31, 30          /* 4th mCQE */
+	};
+
+	/*
+	 * A. load mCQEs into a 128bit register.
+	 * B. store rearm data to mbuf.
+	 * C. combine data from mCQEs with rx_descriptor_fields1.
+	 * D. store rx_descriptor_fields1.
+	 * E. store flow tag (rte_flow mark).
+	 */
+	for (pos = 0; pos < mcqe_n; ) {
+		uint8_t *p = (void *)&mcq[pos % 8];
+		uint8_t *e0 = (void *)&elts[pos]->rearm_data;
+		uint8_t *e1 = (void *)&elts[pos + 1]->rearm_data;
+		uint8_t *e2 = (void *)&elts[pos + 2]->rearm_data;
+		uint8_t *e3 = (void *)&elts[pos + 3]->rearm_data;
+		uint16x4_t byte_cnt;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		uint16x4_t invalid_mask =
+			vcreate_u16(mcqe_n - pos < MLX5_VPMD_DESCS_PER_LOOP ?
+				    -1UL << ((mcqe_n - pos) *
+					     sizeof(uint16_t) * 8) : 0);
+#endif
+
+		if (!(pos & 0x7) && pos + 8 < mcqe_n)
+			rte_prefetch0((void *)(cq + pos + 8));
+		__asm__ volatile (
+		/* A.1 load mCQEs into a 128bit register. */
+		"ld1 {v16.16b - v17.16b}, [%[mcq]] \n\t"
+		/* B.1 store rearm data to mbuf. */
+		"st1 {%[rearm].2d}, [%[e0]] \n\t"
+		"add %[e0], %[e0], #16 \n\t"
+		"st1 {%[rearm].2d}, [%[e1]] \n\t"
+		"add %[e1], %[e1], #16 \n\t"
+		/* C.1 combine data from mCQEs with rx_descriptor_fields1. */
+		"tbl v18.16b, {v16.16b}, %[mcqe_shuf_m1].16b \n\t"
+		"tbl v19.16b, {v16.16b}, %[mcqe_shuf_m2].16b \n\t"
+		"sub v18.8h, v18.8h, %[crc_adj].8h \n\t"
+		"sub v19.8h, v19.8h, %[crc_adj].8h \n\t"
+		"orr v18.16b, v18.16b, %[rxdf].16b \n\t"
+		"orr v19.16b, v19.16b, %[rxdf].16b \n\t"
+		/* D.1 store rx_descriptor_fields1. */
+		"st1 {v18.2d}, [%[e0]] \n\t"
+		"st1 {v19.2d}, [%[e1]] \n\t"
+		/* B.1 store rearm data to mbuf. */
+		"st1 {%[rearm].2d}, [%[e2]] \n\t"
+		"add %[e2], %[e2], #16 \n\t"
+		"st1 {%[rearm].2d}, [%[e3]] \n\t"
+		"add %[e3], %[e3], #16 \n\t"
+		/* C.1 combine data from mCQEs with rx_descriptor_fields1. */
+		"tbl v18.16b, {v17.16b}, %[mcqe_shuf_m1].16b \n\t"
+		"tbl v19.16b, {v17.16b}, %[mcqe_shuf_m2].16b \n\t"
+		"sub v18.8h, v18.8h, %[crc_adj].8h \n\t"
+		"sub v19.8h, v19.8h, %[crc_adj].8h \n\t"
+		"orr v18.16b, v18.16b, %[rxdf].16b \n\t"
+		"orr v19.16b, v19.16b, %[rxdf].16b \n\t"
+		/* D.1 store rx_descriptor_fields1. */
+		"st1 {v18.2d}, [%[e2]] \n\t"
+		"st1 {v19.2d}, [%[e3]] \n\t"
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		"tbl %[byte_cnt].8b, {v16.16b - v17.16b}, %[len_shuf_m].8b \n\t"
+#endif
+		:[byte_cnt]"=&w"(byte_cnt)
+		:[mcq]"r"(p),
+		 [rxdf]"w"(rxdf),
+		 [rearm]"w"(rearm),
+		 [e3]"r"(e3), [e2]"r"(e2), [e1]"r"(e1), [e0]"r"(e0),
+		 [mcqe_shuf_m1]"w"(mcqe_shuf_m1),
+		 [mcqe_shuf_m2]"w"(mcqe_shuf_m2),
+		 [crc_adj]"w"(crc_adj),
+		 [len_shuf_m]"w"(len_shuf_m)
+		:"memory", "v16", "v17", "v18", "v19");
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		byte_cnt = vbic_u16(byte_cnt, invalid_mask);
+		rcvd_byte += vget_lane_u64(vpaddl_u32(vpaddl_u16(byte_cnt)), 0);
+#endif
+		if (rxq->mark) {
+			/* E.1 store flow tag (rte_flow mark). */
+			elts[pos]->hash.fdir.hi = flow_tag;
+			elts[pos + 1]->hash.fdir.hi = flow_tag;
+			elts[pos + 2]->hash.fdir.hi = flow_tag;
+			elts[pos + 3]->hash.fdir.hi = flow_tag;
+		}
+		pos += MLX5_VPMD_DESCS_PER_LOOP;
+		/* Move to next CQE and invalidate consumed CQEs. */
+		if (!(pos & 0x7) && pos < mcqe_n) {
+			mcq = (void *)&(cq + pos)->pkt_info;
+			for (i = 0; i < 8; ++i)
+				cq[inv++].op_own = MLX5_CQE_INVALIDATE;
+		}
+	}
+	/* Invalidate the rest of CQEs. */
+	for (; inv < mcqe_n; ++inv)
+		cq[inv].op_own = MLX5_CQE_INVALIDATE;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	rxq->stats.ipackets += mcqe_n;
+	rxq->stats.ibytes += rcvd_byte;
+#endif
+	rxq->cq_ci += mcqe_n;
+}
+
+/**
+ * Calculate packet type and offload flag for mbuf and store it.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ * @param ptype_info
+ *   Array of four 4bytes packet type info extracted from the original
+ *   completion descriptor.
+ * @param flow_tag
+ *   Array of four 4bytes flow ID extracted from the original completion
+ *   descriptor.
+ * @param op_err
+ *   Opcode vector having responder error status. Each field is 4B.
+ * @param pkts
+ *   Pointer to array of packets to be filled.
+ */
+static inline void
+rxq_cq_to_ptype_oflags_v(struct mlx5_rxq_data *rxq,
+			 uint32x4_t ptype_info, uint32x4_t flow_tag,
+			 uint16x4_t op_err, struct rte_mbuf **pkts)
+{
+	uint16x4_t ptype;
+	uint32x4_t pinfo, cv_flags;
+	uint32x4_t ol_flags =
+		vdupq_n_u32(rxq->rss_hash * PKT_RX_RSS_HASH |
+			    rxq->hw_timestamp * PKT_RX_TIMESTAMP);
+	const uint32x4_t ptype_ol_mask = { 0x106, 0x106, 0x106, 0x106 };
+	const uint8x16_t cv_flag_sel = {
+		0,
+		(uint8_t)(PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED),
+		(uint8_t)(PKT_RX_IP_CKSUM_GOOD >> 1),
+		0,
+		(uint8_t)(PKT_RX_L4_CKSUM_GOOD >> 1),
+		0,
+		(uint8_t)((PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD) >> 1),
+		0, 0, 0, 0, 0, 0, 0, 0, 0
+	};
+	const uint32x4_t cv_mask =
+		vdupq_n_u32(PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD |
+			    PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED);
+	const uint64x1_t mbuf_init = vld1_u64(&rxq->mbuf_initializer);
+	const uint64x1_t r32_mask = vcreate_u64(0xffffffff);
+	uint64x2_t rearm0, rearm1, rearm2, rearm3;
+	uint8_t pt_idx0, pt_idx1, pt_idx2, pt_idx3;
+
+	if (rxq->mark) {
+		const uint32x4_t ft_def = vdupq_n_u32(MLX5_FLOW_MARK_DEFAULT);
+		const uint32x4_t fdir_flags = vdupq_n_u32(PKT_RX_FDIR);
+		uint32x4_t fdir_id_flags = vdupq_n_u32(PKT_RX_FDIR_ID);
+		uint32x4_t invalid_mask;
+
+		/* Check if flow tag is non-zero then set PKT_RX_FDIR. */
+		invalid_mask = vceqzq_u32(flow_tag);
+		ol_flags = vorrq_u32(ol_flags,
+				     vbicq_u32(fdir_flags, invalid_mask));
+		/* Mask out invalid entries. */
+		fdir_id_flags = vbicq_u32(fdir_id_flags, invalid_mask);
+		/* Check if flow tag MLX5_FLOW_MARK_DEFAULT. */
+		ol_flags = vorrq_u32(ol_flags,
+				     vbicq_u32(fdir_id_flags,
+					       vceqq_u32(flow_tag, ft_def)));
+	}
+	/*
+	 * ptype_info has the following:
+	 * bit[1]     = l3_ok
+	 * bit[2]     = l4_ok
+	 * bit[8]     = cv
+	 * bit[11:10] = l3_hdr_type
+	 * bit[14:12] = l4_hdr_type
+	 * bit[15]    = ip_frag
+	 * bit[16]    = tunneled
+	 * bit[17]    = outer_l3_type
+	 */
+	ptype = vshrn_n_u32(ptype_info, 10);
+	/* Errored packets will have RTE_PTYPE_ALL_MASK. */
+	ptype = vorr_u16(ptype, op_err);
+	pt_idx0 = vget_lane_u8(vreinterpret_u8_u16(ptype), 6);
+	pt_idx1 = vget_lane_u8(vreinterpret_u8_u16(ptype), 4);
+	pt_idx2 = vget_lane_u8(vreinterpret_u8_u16(ptype), 2);
+	pt_idx3 = vget_lane_u8(vreinterpret_u8_u16(ptype), 0);
+	pkts[0]->packet_type = mlx5_ptype_table[pt_idx0] |
+			       !!(pt_idx0 & (1 << 6)) * rxq->tunnel;
+	pkts[1]->packet_type = mlx5_ptype_table[pt_idx1] |
+			       !!(pt_idx1 & (1 << 6)) * rxq->tunnel;
+	pkts[2]->packet_type = mlx5_ptype_table[pt_idx2] |
+			       !!(pt_idx2 & (1 << 6)) * rxq->tunnel;
+	pkts[3]->packet_type = mlx5_ptype_table[pt_idx3] |
+			       !!(pt_idx3 & (1 << 6)) * rxq->tunnel;
+	/* Fill flags for checksum and VLAN. */
+	pinfo = vandq_u32(ptype_info, ptype_ol_mask);
+	pinfo = vreinterpretq_u32_u8(
+		vqtbl1q_u8(cv_flag_sel, vreinterpretq_u8_u32(pinfo)));
+	/* Locate checksum flags at byte[2:1] and merge with VLAN flags. */
+	cv_flags = vshlq_n_u32(pinfo, 9);
+	cv_flags = vorrq_u32(pinfo, cv_flags);
+	/* Move back flags to start from byte[0]. */
+	cv_flags = vshrq_n_u32(cv_flags, 8);
+	/* Mask out garbage bits. */
+	cv_flags = vandq_u32(cv_flags, cv_mask);
+	/* Merge to ol_flags. */
+	ol_flags = vorrq_u32(ol_flags, cv_flags);
+	/* Merge mbuf_init and ol_flags, and store. */
+	rearm0 = vcombine_u64(mbuf_init,
+			      vshr_n_u64(vget_high_u64(vreinterpretq_u64_u32(
+						       ol_flags)), 32));
+	rearm1 = vcombine_u64(mbuf_init,
+			      vand_u64(vget_high_u64(vreinterpretq_u64_u32(
+						     ol_flags)), r32_mask));
+	rearm2 = vcombine_u64(mbuf_init,
+			      vshr_n_u64(vget_low_u64(vreinterpretq_u64_u32(
+						      ol_flags)), 32));
+	rearm3 = vcombine_u64(mbuf_init,
+			      vand_u64(vget_low_u64(vreinterpretq_u64_u32(
+						    ol_flags)), r32_mask));
+	vst1q_u64((void *)&pkts[0]->rearm_data, rearm0);
+	vst1q_u64((void *)&pkts[1]->rearm_data, rearm1);
+	vst1q_u64((void *)&pkts[2]->rearm_data, rearm2);
+	vst1q_u64((void *)&pkts[3]->rearm_data, rearm3);
+}
+
+/**
+ * Receive burst of packets. An errored completion also consumes a mbuf, but the
+ * packet_type is set to be RTE_PTYPE_ALL_MASK. Marked mbufs should be freed
+ * before returning to application.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ * @param[out] pkts
+ *   Array to store received packets.
+ * @param pkts_n
+ *   Maximum number of packets in array.
+ * @param[out] err
+ *   Pointer to a flag. Set non-zero value if pkts array has at least one error
+ *   packet to handle.
+ *
+ * @return
+ *   Number of packets received including errors (<= pkts_n).
+ */
+static inline uint16_t
+rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t pkts_n,
+	    uint64_t *err)
+{
+	const uint16_t q_n = 1 << rxq->cqe_n;
+	const uint16_t q_mask = q_n - 1;
+	volatile struct mlx5_cqe *cq;
+	struct rte_mbuf **elts;
+	unsigned int pos;
+	uint64_t n;
+	uint16_t repl_n;
+	uint64_t comp_idx = MLX5_VPMD_DESCS_PER_LOOP;
+	uint16_t nocmp_n = 0;
+	uint16_t rcvd_pkt = 0;
+	unsigned int cq_idx = rxq->cq_ci & q_mask;
+	unsigned int elts_idx;
+	const uint16x4_t ownership = vdup_n_u16(!(rxq->cq_ci & (q_mask + 1)));
+	const uint16x4_t owner_check = vcreate_u16(0x0001000100010001);
+	const uint16x4_t opcode_check = vcreate_u16(0x00f000f000f000f0);
+	const uint16x4_t format_check = vcreate_u16(0x000c000c000c000c);
+	const uint16x4_t resp_err_check = vcreate_u16(0x00e000e000e000e0);
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	uint32_t rcvd_byte = 0;
+#endif
+	/* Mask to generate 16B length vector. */
+	const uint8x8_t len_shuf_m = {
+		52, 53,         /* 4th CQE */
+		36, 37,         /* 3rd CQE */
+		20, 21,         /* 2nd CQE */
+		 4,  5          /* 1st CQE */
+	};
+	/* Mask to extract 16B data from a 64B CQE. */
+	const uint8x16_t cqe_shuf_m = {
+		28, 29,         /* hdr_type_etc */
+		 0,             /* pkt_info */
+		-1,             /* null */
+		47, 46,         /* byte_cnt, bswap16 */
+		31, 30,         /* vlan_info, bswap16 */
+		15, 14, 13, 12, /* rx_hash_res, bswap32 */
+		57, 58, 59,     /* flow_tag */
+		63              /* op_own */
+	};
+	/* Mask to generate 16B data for mbuf. */
+	const uint8x16_t mb_shuf_m = {
+		 4,  5, -1, -1, /* pkt_len */
+		 4,  5,         /* data_len */
+		 6,  7,         /* vlan_tci */
+		 8,  9, 10, 11, /* hash.rss */
+		12, 13, 14, -1  /* hash.fdir.hi */
+	};
+	/* Mask to generate 16B owner vector. */
+	const uint8x8_t owner_shuf_m = {
+		63, -1,         /* 4th CQE */
+		47, -1,         /* 3rd CQE */
+		31, -1,         /* 2nd CQE */
+		15, -1          /* 1st CQE */
+	};
+	/* Mask to generate a vector having packet_type/ol_flags. */
+	const uint8x16_t ptype_shuf_m = {
+		48, 49, 50, -1, /* 4th CQE */
+		32, 33, 34, -1, /* 3rd CQE */
+		16, 17, 18, -1, /* 2nd CQE */
+		 0,  1,  2, -1  /* 1st CQE */
+	};
+	/* Mask to generate a vector having flow tags. */
+	const uint8x16_t ftag_shuf_m = {
+		60, 61, 62, -1, /* 4th CQE */
+		44, 45, 46, -1, /* 3rd CQE */
+		28, 29, 30, -1, /* 2nd CQE */
+		12, 13, 14, -1  /* 1st CQE */
+	};
+	const uint16x8_t crc_adj = {
+		0, 0, rxq->crc_present * ETHER_CRC_LEN, 0, 0, 0, 0, 0
+	};
+	const uint32x4_t flow_mark_adj = { 0, 0, 0, rxq->mark * (-1) };
+
+	assert(rxq->sges_n == 0);
+	assert(rxq->cqe_n == rxq->elts_n);
+	cq = &(*rxq->cqes)[cq_idx];
+	rte_prefetch_non_temporal(cq);
+	rte_prefetch_non_temporal(cq + 1);
+	rte_prefetch_non_temporal(cq + 2);
+	rte_prefetch_non_temporal(cq + 3);
+	pkts_n = RTE_MIN(pkts_n, MLX5_VPMD_RX_MAX_BURST);
+	/*
+	 * Order of indexes:
+	 *   rq_ci >= cq_ci >= rq_pi
+	 * Definition of indexes:
+	 *   rq_ci - cq_ci := # of buffers owned by HW (posted).
+	 *   cq_ci - rq_pi := # of buffers not returned to app (decompressed).
+	 *   N - (rq_ci - rq_pi) := # of buffers consumed (to be replenished).
+	 */
+	repl_n = q_n - (rxq->rq_ci - rxq->rq_pi);
+	if (repl_n >= MLX5_VPMD_RXQ_RPLNSH_THRESH(q_n))
+		mlx5_rx_replenish_bulk_mbuf(rxq, repl_n);
+	/* See if there're unreturned mbufs from compressed CQE. */
+	rcvd_pkt = rxq->cq_ci - rxq->rq_pi;
+	if (rcvd_pkt > 0) {
+		rcvd_pkt = RTE_MIN(rcvd_pkt, pkts_n);
+		rxq_copy_mbuf_v(rxq, pkts, rcvd_pkt);
+		rxq->rq_pi += rcvd_pkt;
+		pkts += rcvd_pkt;
+	}
+	elts_idx = rxq->rq_pi & q_mask;
+	elts = &(*rxq->elts)[elts_idx];
+	/* Not to overflow pkts array. */
+	pkts_n = RTE_ALIGN_FLOOR(pkts_n - rcvd_pkt, MLX5_VPMD_DESCS_PER_LOOP);
+	/* Not to cross queue end. */
+	pkts_n = RTE_MIN(pkts_n, q_n - elts_idx);
+	if (!pkts_n)
+		return rcvd_pkt;
+	/* At this point, there shouldn't be any remained packets. */
+	assert(rxq->rq_pi == rxq->cq_ci);
+	/*
+	 * Note that vectors have reverse order - {v3, v2, v1, v0}, because
+	 * there's no instruction to count trailing zeros. __builtin_clzl() is
+	 * used instead.
+	 *
+	 * A. copy 4 mbuf pointers from elts ring to returing pkts.
+	 * B. load 64B CQE and extract necessary fields
+	 *    Final 16bytes cqes[] extracted from original 64bytes CQE has the
+	 *    following structure:
+	 *        struct {
+	 *          uint16_t hdr_type_etc;
+	 *          uint8_t  pkt_info;
+	 *          uint8_t  rsvd;
+	 *          uint16_t byte_cnt;
+	 *          uint16_t vlan_info;
+	 *          uint32_t rx_has_res;
+	 *          uint8_t  flow_tag[3];
+	 *          uint8_t  op_own;
+	 *        } c;
+	 * C. fill in mbuf.
+	 * D. get valid CQEs.
+	 * E. find compressed CQE.
+	 */
+	for (pos = 0;
+	     pos < pkts_n;
+	     pos += MLX5_VPMD_DESCS_PER_LOOP) {
+		uint16x4_t op_own;
+		uint16x4_t opcode, owner_mask, invalid_mask;
+		uint16x4_t comp_mask;
+		uint16x4_t mask;
+		uint16x4_t byte_cnt;
+		uint32x4_t ptype_info, flow_tag;
+		register uint64x2_t c0, c1, c2, c3;
+		uint8_t *p0, *p1, *p2, *p3;
+		uint8_t *e0 = (void *)&elts[pos]->pkt_len;
+		uint8_t *e1 = (void *)&elts[pos + 1]->pkt_len;
+		uint8_t *e2 = (void *)&elts[pos + 2]->pkt_len;
+		uint8_t *e3 = (void *)&elts[pos + 3]->pkt_len;
+		void *elts_p = (void *)&elts[pos];
+		void *pkts_p = (void *)&pkts[pos];
+
+		/* A.0 do not cross the end of CQ. */
+		mask = vcreate_u16(pkts_n - pos < MLX5_VPMD_DESCS_PER_LOOP ?
+				   -1UL >> ((pkts_n - pos) *
+					    sizeof(uint16_t) * 8) : 0);
+		p0 = (void *)&cq[pos].pkt_info;
+		p1 = p0 + (pkts_n - pos > 1) * sizeof(struct mlx5_cqe);
+		p2 = p1 + (pkts_n - pos > 2) * sizeof(struct mlx5_cqe);
+		p3 = p2 + (pkts_n - pos > 3) * sizeof(struct mlx5_cqe);
+		/* B.0 (CQE 3) load a block having op_own. */
+		c3 = vld1q_u64((uint64_t *)(p3 + 48));
+		/* B.0 (CQE 2) load a block having op_own. */
+		c2 = vld1q_u64((uint64_t *)(p2 + 48));
+		/* B.0 (CQE 1) load a block having op_own. */
+		c1 = vld1q_u64((uint64_t *)(p1 + 48));
+		/* B.0 (CQE 0) load a block having op_own. */
+		c0 = vld1q_u64((uint64_t *)(p0 + 48));
+		/* Synchronize for loading the rest of blocks. */
+		rte_cio_rmb();
+		/* Prefetch next 4 CQEs. */
+		if (pkts_n - pos >= 2 * MLX5_VPMD_DESCS_PER_LOOP) {
+			unsigned int next = pos + MLX5_VPMD_DESCS_PER_LOOP;
+			rte_prefetch_non_temporal(&cq[next]);
+			rte_prefetch_non_temporal(&cq[next + 1]);
+			rte_prefetch_non_temporal(&cq[next + 2]);
+			rte_prefetch_non_temporal(&cq[next + 3]);
+		}
+		__asm__ volatile (
+		/* B.1 (CQE 3) load the rest of blocks. */
+		"ld1 {v16.16b - v18.16b}, [%[p3]] \n\t"
+		/* B.2 (CQE 3) move the block having op_own. */
+		"mov v19.16b, %[c3].16b \n\t"
+		/* B.3 (CQE 3) extract 16B fields. */
+		"tbl v23.16b, {v16.16b - v19.16b}, %[cqe_shuf_m].16b \n\t"
+		/* B.1 (CQE 2) load the rest of blocks. */
+		"ld1 {v16.16b - v18.16b}, [%[p2]] \n\t"
+		/* B.4 (CQE 3) adjust CRC length. */
+		"sub v23.8h, v23.8h, %[crc_adj].8h \n\t"
+		/* C.1 (CQE 3) generate final structure for mbuf. */
+		"tbl v15.16b, {v23.16b}, %[mb_shuf_m].16b \n\t"
+		/* B.2 (CQE 2) move the block having op_own. */
+		"mov v19.16b, %[c2].16b \n\t"
+		/* B.3 (CQE 2) extract 16B fields. */
+		"tbl v22.16b, {v16.16b - v19.16b}, %[cqe_shuf_m].16b \n\t"
+		/* B.1 (CQE 1) load the rest of blocks. */
+		"ld1 {v16.16b - v18.16b}, [%[p1]] \n\t"
+		/* B.4 (CQE 2) adjust CRC length. */
+		"sub v22.8h, v22.8h, %[crc_adj].8h \n\t"
+		/* C.1 (CQE 2) generate final structure for mbuf. */
+		"tbl v14.16b, {v22.16b}, %[mb_shuf_m].16b \n\t"
+		/* B.2 (CQE 1) move the block having op_own. */
+		"mov v19.16b, %[c1].16b \n\t"
+		/* B.3 (CQE 1) extract 16B fields. */
+		"tbl v21.16b, {v16.16b - v19.16b}, %[cqe_shuf_m].16b \n\t"
+		/* B.1 (CQE 0) load the rest of blocks. */
+		"ld1 {v16.16b - v18.16b}, [%[p0]] \n\t"
+		/* B.4 (CQE 1) adjust CRC length. */
+		"sub v21.8h, v21.8h, %[crc_adj].8h \n\t"
+		/* C.1 (CQE 1) generate final structure for mbuf. */
+		"tbl v13.16b, {v21.16b}, %[mb_shuf_m].16b \n\t"
+		/* B.2 (CQE 0) move the block having op_own. */
+		"mov v19.16b, %[c0].16b \n\t"
+		/* A.1 load mbuf pointers. */
+		"ld1 {v24.2d - v25.2d}, [%[elts_p]] \n\t"
+		/* B.3 (CQE 0) extract 16B fields. */
+		"tbl v20.16b, {v16.16b - v19.16b}, %[cqe_shuf_m].16b \n\t"
+		/* B.4 (CQE 0) adjust CRC length. */
+		"sub v20.8h, v20.8h, %[crc_adj].8h \n\t"
+		/* D.1 extract op_own byte. */
+		"tbl %[op_own].8b, {v20.16b - v23.16b}, %[owner_shuf_m].8b \n\t"
+		/* C.2 (CQE 3) adjust flow mark. */
+		"add v15.4s, v15.4s, %[flow_mark_adj].4s \n\t"
+		/* C.3 (CQE 3) fill in mbuf - rx_descriptor_fields1. */
+		"st1 {v15.2d}, [%[e3]] \n\t"
+		/* C.2 (CQE 2) adjust flow mark. */
+		"add v14.4s, v14.4s, %[flow_mark_adj].4s \n\t"
+		/* C.3 (CQE 2) fill in mbuf - rx_descriptor_fields1. */
+		"st1 {v14.2d}, [%[e2]] \n\t"
+		/* C.1 (CQE 0) generate final structure for mbuf. */
+		"tbl v12.16b, {v20.16b}, %[mb_shuf_m].16b \n\t"
+		/* C.2 (CQE 1) adjust flow mark. */
+		"add v13.4s, v13.4s, %[flow_mark_adj].4s \n\t"
+		/* C.3 (CQE 1) fill in mbuf - rx_descriptor_fields1. */
+		"st1 {v13.2d}, [%[e1]] \n\t"
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		/* Extract byte_cnt. */
+		"tbl %[byte_cnt].8b, {v20.16b - v23.16b}, %[len_shuf_m].8b \n\t"
+#endif
+		/* Extract ptype_info. */
+		"tbl %[ptype_info].16b, {v20.16b - v23.16b}, %[ptype_shuf_m].16b \n\t"
+		/* Extract flow_tag. */
+		"tbl %[flow_tag].16b, {v20.16b - v23.16b}, %[ftag_shuf_m].16b \n\t"
+		/* A.2 copy mbuf pointers. */
+		"st1 {v24.2d - v25.2d}, [%[pkts_p]] \n\t"
+		/* C.2 (CQE 0) adjust flow mark. */
+		"add v12.4s, v12.4s, %[flow_mark_adj].4s \n\t"
+		/* C.3 (CQE 1) fill in mbuf - rx_descriptor_fields1. */
+		"st1 {v12.2d}, [%[e0]] \n\t"
+		:[op_own]"=&w"(op_own),
+		 [byte_cnt]"=&w"(byte_cnt),
+		 [ptype_info]"=&w"(ptype_info),
+		 [flow_tag]"=&w"(flow_tag)
+		:[p3]"r"(p3), [p2]"r"(p2), [p1]"r"(p1), [p0]"r"(p0),
+		 [e3]"r"(e3), [e2]"r"(e2), [e1]"r"(e1), [e0]"r"(e0),
+		 [c3]"w"(c3), [c2]"w"(c2), [c1]"w"(c1), [c0]"w"(c0),
+		 [elts_p]"r"(elts_p),
+		 [pkts_p]"r"(pkts_p),
+		 [cqe_shuf_m]"w"(cqe_shuf_m),
+		 [mb_shuf_m]"w"(mb_shuf_m),
+		 [owner_shuf_m]"w"(owner_shuf_m),
+		 [len_shuf_m]"w"(len_shuf_m),
+		 [ptype_shuf_m]"w"(ptype_shuf_m),
+		 [ftag_shuf_m]"w"(ftag_shuf_m),
+		 [crc_adj]"w"(crc_adj),
+		 [flow_mark_adj]"w"(flow_mark_adj)
+		:"memory",
+		 "v12", "v13", "v14", "v15",
+		 "v16", "v17", "v18", "v19",
+		 "v20", "v21", "v22", "v23",
+		 "v24", "v25");
+		/* D.2 flip owner bit to mark CQEs from last round. */
+		owner_mask = vand_u16(op_own, owner_check);
+		owner_mask = vceq_u16(owner_mask, ownership);
+		/* D.3 get mask for invalidated CQEs. */
+		opcode = vand_u16(op_own, opcode_check);
+		invalid_mask = vceq_u16(opcode_check, opcode);
+		/* E.1 find compressed CQE format. */
+		comp_mask = vand_u16(op_own, format_check);
+		comp_mask = vceq_u16(comp_mask, format_check);
+		/* D.4 mask out beyond boundary. */
+		invalid_mask = vorr_u16(invalid_mask, mask);
+		/* D.5 merge invalid_mask with invalid owner. */
+		invalid_mask = vorr_u16(invalid_mask, owner_mask);
+		/* E.2 mask out invalid entries. */
+		comp_mask = vbic_u16(comp_mask, invalid_mask);
+		/* E.3 get the first compressed CQE. */
+		comp_idx = __builtin_clzl(vget_lane_u64(vreinterpret_u64_u16(
+					  comp_mask), 0)) /
+					  (sizeof(uint16_t) * 8);
+		/* D.6 mask out entries after the compressed CQE. */
+		mask = vcreate_u16(comp_idx < MLX5_VPMD_DESCS_PER_LOOP ?
+				   -1UL >> (comp_idx * sizeof(uint16_t) * 8) :
+				   0);
+		invalid_mask = vorr_u16(invalid_mask, mask);
+		/* D.7 count non-compressed valid CQEs. */
+		n = __builtin_clzl(vget_lane_u64(vreinterpret_u64_u16(
+				   invalid_mask), 0)) / (sizeof(uint16_t) * 8);
+		nocmp_n += n;
+		/* D.2 get the final invalid mask. */
+		mask = vcreate_u16(n < MLX5_VPMD_DESCS_PER_LOOP ?
+				   -1UL >> (n * sizeof(uint16_t) * 8) : 0);
+		invalid_mask = vorr_u16(invalid_mask, mask);
+		/* D.3 check error in opcode. */
+		opcode = vceq_u16(resp_err_check, opcode);
+		opcode = vbic_u16(opcode, invalid_mask);
+		/* D.4 mark if any error is set */
+		*err |= vget_lane_u64(vreinterpret_u64_u16(opcode), 0);
+		/* C.4 fill in mbuf - rearm_data and packet_type. */
+		rxq_cq_to_ptype_oflags_v(rxq, ptype_info, flow_tag,
+					 opcode, &elts[pos]);
+		if (rxq->hw_timestamp) {
+			elts[pos]->timestamp =
+				rte_be_to_cpu_64(
+					container_of(p0, struct mlx5_cqe,
+						     pkt_info)->timestamp);
+			elts[pos + 1]->timestamp =
+				rte_be_to_cpu_64(
+					container_of(p1, struct mlx5_cqe,
+						     pkt_info)->timestamp);
+			elts[pos + 2]->timestamp =
+				rte_be_to_cpu_64(
+					container_of(p2, struct mlx5_cqe,
+						     pkt_info)->timestamp);
+			elts[pos + 3]->timestamp =
+				rte_be_to_cpu_64(
+					container_of(p3, struct mlx5_cqe,
+						     pkt_info)->timestamp);
+		}
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		/* Add up received bytes count. */
+		byte_cnt = vbic_u16(byte_cnt, invalid_mask);
+		rcvd_byte += vget_lane_u64(vpaddl_u32(vpaddl_u16(byte_cnt)), 0);
+#endif
+		/*
+		 * Break the loop unless more valid CQE is expected, or if
+		 * there's a compressed CQE.
+		 */
+		if (n != MLX5_VPMD_DESCS_PER_LOOP)
+			break;
+	}
+	/* If no new CQE seen, return without updating cq_db. */
+	if (unlikely(!nocmp_n && comp_idx == MLX5_VPMD_DESCS_PER_LOOP))
+		return rcvd_pkt;
+	/* Update the consumer indexes for non-compressed CQEs. */
+	assert(nocmp_n <= pkts_n);
+	rxq->cq_ci += nocmp_n;
+	rxq->rq_pi += nocmp_n;
+	rcvd_pkt += nocmp_n;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	rxq->stats.ipackets += nocmp_n;
+	rxq->stats.ibytes += rcvd_byte;
+#endif
+	/* Decompress the last CQE if compressed. */
+	if (comp_idx < MLX5_VPMD_DESCS_PER_LOOP && comp_idx == n) {
+		assert(comp_idx == (nocmp_n % MLX5_VPMD_DESCS_PER_LOOP));
+		rxq_cq_decompress_v(rxq, &cq[nocmp_n], &elts[nocmp_n]);
+		/* Return more packets if needed. */
+		if (nocmp_n < pkts_n) {
+			uint16_t n = rxq->cq_ci - rxq->rq_pi;
+
+			n = RTE_MIN(n, pkts_n - nocmp_n);
+			rxq_copy_mbuf_v(rxq, &pkts[nocmp_n], n);
+			rxq->rq_pi += n;
+			rcvd_pkt += n;
+		}
+	}
+	rte_compiler_barrier();
+	*rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci);
+	return rcvd_pkt;
+}
+
+#endif /* RTE_PMD_MLX5_RXTX_VEC_NEON_H_ */
diff --git a/src/spdk/dpdk/drivers/net/mlx5/mlx5_rxtx_vec_sse.h b/src/spdk/dpdk/drivers/net/mlx5/mlx5_rxtx_vec_sse.h
new file mode 100644
index 00000000..54b3783c
--- /dev/null
+++ b/src/spdk/dpdk/drivers/net/mlx5/mlx5_rxtx_vec_sse.h
@@ -0,0 +1,969 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2017 6WIND S.A.
+ * Copyright 2017 Mellanox Technologies, Ltd
+ */
+
+#ifndef RTE_PMD_MLX5_RXTX_VEC_SSE_H_
+#define RTE_PMD_MLX5_RXTX_VEC_SSE_H_
+
+#include <assert.h>
+#include <stdint.h>
+#include <string.h>
+#include <stdlib.h>
+#include <smmintrin.h>
+
+#include <rte_mbuf.h>
+#include <rte_mempool.h>
+#include <rte_prefetch.h>
+
+#include "mlx5.h"
+#include "mlx5_utils.h"
+#include "mlx5_rxtx.h"
+#include "mlx5_rxtx_vec.h"
+#include "mlx5_autoconf.h"
+#include "mlx5_defs.h"
+#include "mlx5_prm.h"
+
+#ifndef __INTEL_COMPILER
+#pragma GCC diagnostic ignored "-Wcast-qual"
+#endif
+
+/**
+ * Fill in buffer descriptors in a multi-packet send descriptor.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param dseg
+ *   Pointer to buffer descriptor to be written.
+ * @param pkts
+ *   Pointer to array of packets to be sent.
+ * @param n
+ *   Number of packets to be filled.
+ */
+static inline void
+txq_wr_dseg_v(struct mlx5_txq_data *txq, __m128i *dseg,
+	      struct rte_mbuf **pkts, unsigned int n)
+{
+	unsigned int pos;
+	uintptr_t addr;
+	const __m128i shuf_mask_dseg =
+		_mm_set_epi8(8,  9, 10, 11, /* addr, bswap64 */
+			    12, 13, 14, 15,
+			     7,  6,  5,  4, /* lkey */
+			     0,  1,  2,  3  /* length, bswap32 */);
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	uint32_t tx_byte = 0;
+#endif
+
+	for (pos = 0; pos < n; ++pos, ++dseg) {
+		__m128i desc;
+		struct rte_mbuf *pkt = pkts[pos];
+
+		addr = rte_pktmbuf_mtod(pkt, uintptr_t);
+		desc = _mm_set_epi32(addr >> 32,
+				     addr,
+				     mlx5_tx_mb2mr(txq, pkt),
+				     DATA_LEN(pkt));
+		desc = _mm_shuffle_epi8(desc, shuf_mask_dseg);
+		_mm_store_si128(dseg, desc);
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		tx_byte += DATA_LEN(pkt);
+#endif
+	}
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	txq->stats.obytes += tx_byte;
+#endif
+}
+
+/**
+ * Send multi-segmented packets until it encounters a single segment packet in
+ * the pkts list.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param pkts
+ *   Pointer to array of packets to be sent.
+ * @param pkts_n
+ *   Number of packets to be sent.
+ *
+ * @return
+ *   Number of packets successfully transmitted (<= pkts_n).
+ */
+static uint16_t
+txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
+	      uint16_t pkts_n)
+{
+	uint16_t elts_head = txq->elts_head;
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
+	const uint16_t wq_n = 1 << txq->wqe_n;
+	const uint16_t wq_mask = wq_n - 1;
+	const unsigned int nb_dword_per_wqebb =
+		MLX5_WQE_SIZE / MLX5_WQE_DWORD_SIZE;
+	const unsigned int nb_dword_in_hdr =
+		sizeof(struct mlx5_wqe) / MLX5_WQE_DWORD_SIZE;
+	unsigned int n;
+	volatile struct mlx5_wqe *wqe = NULL;
+
+	assert(elts_n > pkts_n);
+	mlx5_tx_complete(txq);
+	if (unlikely(!pkts_n))
+		return 0;
+	for (n = 0; n < pkts_n; ++n) {
+		struct rte_mbuf *buf = pkts[n];
+		unsigned int segs_n = buf->nb_segs;
+		unsigned int ds = nb_dword_in_hdr;
+		unsigned int len = PKT_LEN(buf);
+		uint16_t wqe_ci = txq->wqe_ci;
+		const __m128i shuf_mask_ctrl =
+			_mm_set_epi8(15, 14, 13, 12,
+				      8,  9, 10, 11, /* bswap32 */
+				      4,  5,  6,  7, /* bswap32 */
+				      0,  1,  2,  3  /* bswap32 */);
+		uint8_t cs_flags;
+		uint16_t max_elts;
+		uint16_t max_wqe;
+		__m128i *t_wqe, *dseg;
+		__m128i ctrl;
+
+		assert(segs_n);
+		max_elts = elts_n - (elts_head - txq->elts_tail);
+		max_wqe = wq_n - (txq->wqe_ci - txq->wqe_pi);
+		/*
+		 * A MPW session consumes 2 WQEs at most to
+		 * include MLX5_MPW_DSEG_MAX pointers.
+		 */
+		if (segs_n == 1 ||
+		    max_elts < segs_n || max_wqe < 2)
+			break;
+		if (segs_n > MLX5_MPW_DSEG_MAX) {
+			txq->stats.oerrors++;
+			break;
+		}
+		wqe = &((volatile struct mlx5_wqe64 *)
+			 txq->wqes)[wqe_ci & wq_mask].hdr;
+		cs_flags = txq_ol_cksum_to_cs(buf);
+		/* Title WQEBB pointer. */
+		t_wqe = (__m128i *)wqe;
+		dseg = (__m128i *)(wqe + 1);
+		do {
+			if (!(ds++ % nb_dword_per_wqebb)) {
+				dseg = (__m128i *)
+					&((volatile struct mlx5_wqe64 *)
+					   txq->wqes)[++wqe_ci & wq_mask];
+			}
+			txq_wr_dseg_v(txq, dseg++, &buf, 1);
+			(*txq->elts)[elts_head++ & elts_m] = buf;
+			buf = buf->next;
+		} while (--segs_n);
+		++wqe_ci;
+		/* Fill CTRL in the header. */
+		ctrl = _mm_set_epi32(0, 0, txq->qp_num_8s | ds,
+				     MLX5_OPC_MOD_MPW << 24 |
+				     txq->wqe_ci << 8 | MLX5_OPCODE_TSO);
+		ctrl = _mm_shuffle_epi8(ctrl, shuf_mask_ctrl);
+		_mm_store_si128(t_wqe, ctrl);
+		/* Fill ESEG in the header. */
+		_mm_store_si128(t_wqe + 1,
+				_mm_set_epi16(0, 0, 0, 0,
+					      rte_cpu_to_be_16(len), cs_flags,
+					      0, 0));
+		txq->wqe_ci = wqe_ci;
+	}
+	if (!n)
+		return 0;
+	txq->elts_comp += (uint16_t)(elts_head - txq->elts_head);
+	txq->elts_head = elts_head;
+	if (txq->elts_comp >= MLX5_TX_COMP_THRESH) {
+		/* A CQE slot must always be available. */
+		assert((1u << txq->cqe_n) - (txq->cq_pi++ - txq->cq_ci));
+		wqe->ctrl[2] = rte_cpu_to_be_32(8);
+		wqe->ctrl[3] = txq->elts_head;
+		txq->elts_comp = 0;
+	}
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	txq->stats.opackets += n;
+#endif
+	mlx5_tx_dbrec(txq, wqe);
+	return n;
+}
+
+/**
+ * Send burst of packets with Enhanced MPW. If it encounters a multi-seg packet,
+ * it returns to make it processed by txq_scatter_v(). All the packets in
+ * the pkts list should be single segment packets having same offload flags.
+ * This must be checked by txq_count_contig_single_seg() and txq_calc_offload().
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param pkts
+ *   Pointer to array of packets to be sent.
+ * @param pkts_n
+ *   Number of packets to be sent (<= MLX5_VPMD_TX_MAX_BURST).
+ * @param cs_flags
+ *   Checksum offload flags to be written in the descriptor.
+ *
+ * @return
+ *   Number of packets successfully transmitted (<= pkts_n).
+ */
+static inline uint16_t
+txq_burst_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, uint16_t pkts_n,
+	    uint8_t cs_flags)
+{
+	struct rte_mbuf **elts;
+	uint16_t elts_head = txq->elts_head;
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
+	const unsigned int nb_dword_per_wqebb =
+		MLX5_WQE_SIZE / MLX5_WQE_DWORD_SIZE;
+	const unsigned int nb_dword_in_hdr =
+		sizeof(struct mlx5_wqe) / MLX5_WQE_DWORD_SIZE;
+	unsigned int n = 0;
+	unsigned int pos;
+	uint16_t max_elts;
+	uint16_t max_wqe;
+	uint32_t comp_req = 0;
+	const uint16_t wq_n = 1 << txq->wqe_n;
+	const uint16_t wq_mask = wq_n - 1;
+	uint16_t wq_idx = txq->wqe_ci & wq_mask;
+	volatile struct mlx5_wqe64 *wq =
+		&((volatile struct mlx5_wqe64 *)txq->wqes)[wq_idx];
+	volatile struct mlx5_wqe *wqe = (volatile struct mlx5_wqe *)wq;
+	const __m128i shuf_mask_ctrl =
+		_mm_set_epi8(15, 14, 13, 12,
+			      8,  9, 10, 11, /* bswap32 */
+			      4,  5,  6,  7, /* bswap32 */
+			      0,  1,  2,  3  /* bswap32 */);
+	__m128i *t_wqe, *dseg;
+	__m128i ctrl;
+
+	/* Make sure all packets can fit into a single WQE. */
+	assert(elts_n > pkts_n);
+	mlx5_tx_complete(txq);
+	max_elts = (elts_n - (elts_head - txq->elts_tail));
+	max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
+	pkts_n = RTE_MIN((unsigned int)RTE_MIN(pkts_n, max_wqe), max_elts);
+	assert(pkts_n <= MLX5_DSEG_MAX - nb_dword_in_hdr);
+	if (unlikely(!pkts_n))
+		return 0;
+	elts = &(*txq->elts)[elts_head & elts_m];
+	/* Loop for available tailroom first. */
+	n = RTE_MIN(elts_n - (elts_head & elts_m), pkts_n);
+	for (pos = 0; pos < (n & -2); pos += 2)
+		_mm_storeu_si128((__m128i *)&elts[pos],
+				 _mm_loadu_si128((__m128i *)&pkts[pos]));
+	if (n & 1)
+		elts[pos] = pkts[pos];
+	/* Check if it crosses the end of the queue. */
+	if (unlikely(n < pkts_n)) {
+		elts = &(*txq->elts)[0];
+		for (pos = 0; pos < pkts_n - n; ++pos)
+			elts[pos] = pkts[n + pos];
+	}
+	txq->elts_head += pkts_n;
+	/* Save title WQEBB pointer. */
+	t_wqe = (__m128i *)wqe;
+	dseg = (__m128i *)(wqe + 1);
+	/* Calculate the number of entries to the end. */
+	n = RTE_MIN(
+		(wq_n - wq_idx) * nb_dword_per_wqebb - nb_dword_in_hdr,
+		pkts_n);
+	/* Fill DSEGs. */
+	txq_wr_dseg_v(txq, dseg, pkts, n);
+	/* Check if it crosses the end of the queue. */
+	if (n < pkts_n) {
+		dseg = (__m128i *)txq->wqes;
+		txq_wr_dseg_v(txq, dseg, &pkts[n], pkts_n - n);
+	}
+	if (txq->elts_comp + pkts_n < MLX5_TX_COMP_THRESH) {
+		txq->elts_comp += pkts_n;
+	} else {
+		/* A CQE slot must always be available. */
+		assert((1u << txq->cqe_n) - (txq->cq_pi++ - txq->cq_ci));
+		/* Request a completion. */
+		txq->elts_comp = 0;
+		comp_req = 8;
+	}
+	/* Fill CTRL in the header. */
+	ctrl = _mm_set_epi32(txq->elts_head, comp_req,
+			     txq->qp_num_8s | (pkts_n + 2),
+			     MLX5_OPC_MOD_ENHANCED_MPSW << 24 |
+				txq->wqe_ci << 8 | MLX5_OPCODE_ENHANCED_MPSW);
+	ctrl = _mm_shuffle_epi8(ctrl, shuf_mask_ctrl);
+	_mm_store_si128(t_wqe, ctrl);
+	/* Fill ESEG in the header. */
+	_mm_store_si128(t_wqe + 1,
+			_mm_set_epi8(0, 0, 0, 0,
+				     0, 0, 0, 0,
+				     0, 0, 0, cs_flags,
+				     0, 0, 0, 0));
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	txq->stats.opackets += pkts_n;
+#endif
+	txq->wqe_ci += (nb_dword_in_hdr + pkts_n + (nb_dword_per_wqebb - 1)) /
+		       nb_dword_per_wqebb;
+	/* Ring QP doorbell. */
+	mlx5_tx_dbrec_cond_wmb(txq, wqe, pkts_n < MLX5_VPMD_TX_MAX_BURST);
+	return pkts_n;
+}
+
+/**
+ * Store free buffers to RX SW ring.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ * @param pkts
+ *   Pointer to array of packets to be stored.
+ * @param pkts_n
+ *   Number of packets to be stored.
+ */
+static inline void
+rxq_copy_mbuf_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t n)
+{
+	const uint16_t q_mask = (1 << rxq->elts_n) - 1;
+	struct rte_mbuf **elts = &(*rxq->elts)[rxq->rq_pi & q_mask];
+	unsigned int pos;
+	uint16_t p = n & -2;
+
+	for (pos = 0; pos < p; pos += 2) {
+		__m128i mbp;
+
+		mbp = _mm_loadu_si128((__m128i *)&elts[pos]);
+		_mm_storeu_si128((__m128i *)&pkts[pos], mbp);
+	}
+	if (n & 1)
+		pkts[pos] = elts[pos];
+}
+
+/**
+ * Decompress a compressed completion and fill in mbufs in RX SW ring with data
+ * extracted from the title completion descriptor.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ * @param cq
+ *   Pointer to completion array having a compressed completion at first.
+ * @param elts
+ *   Pointer to SW ring to be filled. The first mbuf has to be pre-built from
+ *   the title completion descriptor to be copied to the rest of mbufs.
+ */
+static inline void
+rxq_cq_decompress_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq,
+		    struct rte_mbuf **elts)
+{
+	volatile struct mlx5_mini_cqe8 *mcq = (void *)(cq + 1);
+	struct rte_mbuf *t_pkt = elts[0]; /* Title packet is pre-built. */
+	unsigned int pos;
+	unsigned int i;
+	unsigned int inv = 0;
+	/* Mask to shuffle from extracted mini CQE to mbuf. */
+	const __m128i shuf_mask1 =
+		_mm_set_epi8(0,  1,  2,  3, /* rss, bswap32 */
+			    -1, -1,         /* skip vlan_tci */
+			     6,  7,         /* data_len, bswap16 */
+			    -1, -1,  6,  7, /* pkt_len, bswap16 */
+			    -1, -1, -1, -1  /* skip packet_type */);
+	const __m128i shuf_mask2 =
+		_mm_set_epi8(8,  9, 10, 11, /* rss, bswap32 */
+			    -1, -1,         /* skip vlan_tci */
+			    14, 15,         /* data_len, bswap16 */
+			    -1, -1, 14, 15, /* pkt_len, bswap16 */
+			    -1, -1, -1, -1  /* skip packet_type */);
+	/* Restore the compressed count. Must be 16 bits. */
+	const uint16_t mcqe_n = t_pkt->data_len +
+				(rxq->crc_present * ETHER_CRC_LEN);
+	const __m128i rearm =
+		_mm_loadu_si128((__m128i *)&t_pkt->rearm_data);
+	const __m128i rxdf =
+		_mm_loadu_si128((__m128i *)&t_pkt->rx_descriptor_fields1);
+	const __m128i crc_adj =
+		_mm_set_epi16(0, 0, 0,
+			      rxq->crc_present * ETHER_CRC_LEN,
+			      0,
+			      rxq->crc_present * ETHER_CRC_LEN,
+			      0, 0);
+	const uint32_t flow_tag = t_pkt->hash.fdir.hi;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	const __m128i zero = _mm_setzero_si128();
+	const __m128i ones = _mm_cmpeq_epi32(zero, zero);
+	uint32_t rcvd_byte = 0;
+	/* Mask to shuffle byte_cnt to add up stats. Do bswap16 for all. */
+	const __m128i len_shuf_mask =
+		_mm_set_epi8(-1, -1, -1, -1,
+			     -1, -1, -1, -1,
+			     14, 15,  6,  7,
+			     10, 11,  2,  3);
+#endif
+
+	/*
+	 * A. load mCQEs into a 128bit register.
+	 * B. store rearm data to mbuf.
+	 * C. combine data from mCQEs with rx_descriptor_fields1.
+	 * D. store rx_descriptor_fields1.
+	 * E. store flow tag (rte_flow mark).
+	 */
+	for (pos = 0; pos < mcqe_n; ) {
+		__m128i mcqe1, mcqe2;
+		__m128i rxdf1, rxdf2;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		__m128i byte_cnt, invalid_mask;
+#endif
+
+		if (!(pos & 0x7) && pos + 8 < mcqe_n)
+			rte_prefetch0((void *)(cq + pos + 8));
+		/* A.1 load mCQEs into a 128bit register. */
+		mcqe1 = _mm_loadu_si128((__m128i *)&mcq[pos % 8]);
+		mcqe2 = _mm_loadu_si128((__m128i *)&mcq[pos % 8 + 2]);
+		/* B.1 store rearm data to mbuf. */
+		_mm_storeu_si128((__m128i *)&elts[pos]->rearm_data, rearm);
+		_mm_storeu_si128((__m128i *)&elts[pos + 1]->rearm_data, rearm);
+		/* C.1 combine data from mCQEs with rx_descriptor_fields1. */
+		rxdf1 = _mm_shuffle_epi8(mcqe1, shuf_mask1);
+		rxdf2 = _mm_shuffle_epi8(mcqe1, shuf_mask2);
+		rxdf1 = _mm_sub_epi16(rxdf1, crc_adj);
+		rxdf2 = _mm_sub_epi16(rxdf2, crc_adj);
+		rxdf1 = _mm_blend_epi16(rxdf1, rxdf, 0x23);
+		rxdf2 = _mm_blend_epi16(rxdf2, rxdf, 0x23);
+		/* D.1 store rx_descriptor_fields1. */
+		_mm_storeu_si128((__m128i *)
+				  &elts[pos]->rx_descriptor_fields1,
+				 rxdf1);
+		_mm_storeu_si128((__m128i *)
+				  &elts[pos + 1]->rx_descriptor_fields1,
+				 rxdf2);
+		/* B.1 store rearm data to mbuf. */
+		_mm_storeu_si128((__m128i *)&elts[pos + 2]->rearm_data, rearm);
+		_mm_storeu_si128((__m128i *)&elts[pos + 3]->rearm_data, rearm);
+		/* C.1 combine data from mCQEs with rx_descriptor_fields1. */
+		rxdf1 = _mm_shuffle_epi8(mcqe2, shuf_mask1);
+		rxdf2 = _mm_shuffle_epi8(mcqe2, shuf_mask2);
+		rxdf1 = _mm_sub_epi16(rxdf1, crc_adj);
+		rxdf2 = _mm_sub_epi16(rxdf2, crc_adj);
+		rxdf1 = _mm_blend_epi16(rxdf1, rxdf, 0x23);
+		rxdf2 = _mm_blend_epi16(rxdf2, rxdf, 0x23);
+		/* D.1 store rx_descriptor_fields1. */
+		_mm_storeu_si128((__m128i *)
+				  &elts[pos + 2]->rx_descriptor_fields1,
+				 rxdf1);
+		_mm_storeu_si128((__m128i *)
+				  &elts[pos + 3]->rx_descriptor_fields1,
+				 rxdf2);
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		invalid_mask = _mm_set_epi64x(0,
+					      (mcqe_n - pos) *
+					      sizeof(uint16_t) * 8);
+		invalid_mask = _mm_sll_epi64(ones, invalid_mask);
+		mcqe1 = _mm_srli_si128(mcqe1, 4);
+		byte_cnt = _mm_blend_epi16(mcqe1, mcqe2, 0xcc);
+		byte_cnt = _mm_shuffle_epi8(byte_cnt, len_shuf_mask);
+		byte_cnt = _mm_andnot_si128(invalid_mask, byte_cnt);
+		byte_cnt = _mm_hadd_epi16(byte_cnt, zero);
+		rcvd_byte += _mm_cvtsi128_si64(_mm_hadd_epi16(byte_cnt, zero));
+#endif
+		if (rxq->mark) {
+			/* E.1 store flow tag (rte_flow mark). */
+			elts[pos]->hash.fdir.hi = flow_tag;
+			elts[pos + 1]->hash.fdir.hi = flow_tag;
+			elts[pos + 2]->hash.fdir.hi = flow_tag;
+			elts[pos + 3]->hash.fdir.hi = flow_tag;
+		}
+		pos += MLX5_VPMD_DESCS_PER_LOOP;
+		/* Move to next CQE and invalidate consumed CQEs. */
+		if (!(pos & 0x7) && pos < mcqe_n) {
+			mcq = (void *)(cq + pos);
+			for (i = 0; i < 8; ++i)
+				cq[inv++].op_own = MLX5_CQE_INVALIDATE;
+		}
+	}
+	/* Invalidate the rest of CQEs. */
+	for (; inv < mcqe_n; ++inv)
+		cq[inv].op_own = MLX5_CQE_INVALIDATE;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	rxq->stats.ipackets += mcqe_n;
+	rxq->stats.ibytes += rcvd_byte;
+#endif
+	rxq->cq_ci += mcqe_n;
+}
+
+/**
+ * Calculate packet type and offload flag for mbuf and store it.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ * @param cqes[4]
+ *   Array of four 16bytes completions extracted from the original completion
+ *   descriptor.
+ * @param op_err
+ *   Opcode vector having responder error status. Each field is 4B.
+ * @param pkts
+ *   Pointer to array of packets to be filled.
+ */
+static inline void
+rxq_cq_to_ptype_oflags_v(struct mlx5_rxq_data *rxq, __m128i cqes[4],
+			 __m128i op_err, struct rte_mbuf **pkts)
+{
+	__m128i pinfo0, pinfo1;
+	__m128i pinfo, ptype;
+	__m128i ol_flags = _mm_set1_epi32(rxq->rss_hash * PKT_RX_RSS_HASH |
+					  rxq->hw_timestamp * PKT_RX_TIMESTAMP);
+	__m128i cv_flags;
+	const __m128i zero = _mm_setzero_si128();
+	const __m128i ptype_mask =
+		_mm_set_epi32(0xfd06, 0xfd06, 0xfd06, 0xfd06);
+	const __m128i ptype_ol_mask =
+		_mm_set_epi32(0x106, 0x106, 0x106, 0x106);
+	const __m128i pinfo_mask =
+		_mm_set_epi32(0x3, 0x3, 0x3, 0x3);
+	const __m128i cv_flag_sel =
+		_mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0,
+			     (uint8_t)((PKT_RX_IP_CKSUM_GOOD |
+					PKT_RX_L4_CKSUM_GOOD) >> 1),
+			     0,
+			     (uint8_t)(PKT_RX_L4_CKSUM_GOOD >> 1),
+			     0,
+			     (uint8_t)(PKT_RX_IP_CKSUM_GOOD >> 1),
+			     (uint8_t)(PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED),
+			     0);
+	const __m128i cv_mask =
+		_mm_set_epi32(PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD |
+			      PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED,
+			      PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD |
+			      PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED,
+			      PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD |
+			      PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED,
+			      PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD |
+			      PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED);
+	const __m128i mbuf_init =
+		_mm_loadl_epi64((__m128i *)&rxq->mbuf_initializer);
+	__m128i rearm0, rearm1, rearm2, rearm3;
+	uint8_t pt_idx0, pt_idx1, pt_idx2, pt_idx3;
+
+	/* Extract pkt_info field. */
+	pinfo0 = _mm_unpacklo_epi32(cqes[0], cqes[1]);
+	pinfo1 = _mm_unpacklo_epi32(cqes[2], cqes[3]);
+	pinfo = _mm_unpacklo_epi64(pinfo0, pinfo1);
+	/* Extract hdr_type_etc field. */
+	pinfo0 = _mm_unpackhi_epi32(cqes[0], cqes[1]);
+	pinfo1 = _mm_unpackhi_epi32(cqes[2], cqes[3]);
+	ptype = _mm_unpacklo_epi64(pinfo0, pinfo1);
+	if (rxq->mark) {
+		const __m128i pinfo_ft_mask =
+			_mm_set_epi32(0xffffff00, 0xffffff00,
+				      0xffffff00, 0xffffff00);
+		const __m128i fdir_flags = _mm_set1_epi32(PKT_RX_FDIR);
+		__m128i fdir_id_flags = _mm_set1_epi32(PKT_RX_FDIR_ID);
+		__m128i flow_tag, invalid_mask;
+
+		flow_tag = _mm_and_si128(pinfo, pinfo_ft_mask);
+		/* Check if flow tag is non-zero then set PKT_RX_FDIR. */
+		invalid_mask = _mm_cmpeq_epi32(flow_tag, zero);
+		ol_flags = _mm_or_si128(ol_flags,
+					_mm_andnot_si128(invalid_mask,
+							 fdir_flags));
+		/* Mask out invalid entries. */
+		fdir_id_flags = _mm_andnot_si128(invalid_mask, fdir_id_flags);
+		/* Check if flow tag MLX5_FLOW_MARK_DEFAULT. */
+		ol_flags = _mm_or_si128(ol_flags,
+					_mm_andnot_si128(
+						_mm_cmpeq_epi32(flow_tag,
+								pinfo_ft_mask),
+						fdir_id_flags));
+	}
+	/*
+	 * Merge the two fields to generate the following:
+	 * bit[1]     = l3_ok
+	 * bit[2]     = l4_ok
+	 * bit[8]     = cv
+	 * bit[11:10] = l3_hdr_type
+	 * bit[14:12] = l4_hdr_type
+	 * bit[15]    = ip_frag
+	 * bit[16]    = tunneled
+	 * bit[17]    = outer_l3_type
+	 */
+	ptype = _mm_and_si128(ptype, ptype_mask);
+	pinfo = _mm_and_si128(pinfo, pinfo_mask);
+	pinfo = _mm_slli_epi32(pinfo, 16);
+	/* Make pinfo has merged fields for ol_flags calculation. */
+	pinfo = _mm_or_si128(ptype, pinfo);
+	ptype = _mm_srli_epi32(pinfo, 10);
+	ptype = _mm_packs_epi32(ptype, zero);
+	/* Errored packets will have RTE_PTYPE_ALL_MASK. */
+	op_err = _mm_srli_epi16(op_err, 8);
+	ptype = _mm_or_si128(ptype, op_err);
+	pt_idx0 = _mm_extract_epi8(ptype, 0);
+	pt_idx1 = _mm_extract_epi8(ptype, 2);
+	pt_idx2 = _mm_extract_epi8(ptype, 4);
+	pt_idx3 = _mm_extract_epi8(ptype, 6);
+	pkts[0]->packet_type = mlx5_ptype_table[pt_idx0] |
+			       !!(pt_idx0 & (1 << 6)) * rxq->tunnel;
+	pkts[1]->packet_type = mlx5_ptype_table[pt_idx1] |
+			       !!(pt_idx1 & (1 << 6)) * rxq->tunnel;
+	pkts[2]->packet_type = mlx5_ptype_table[pt_idx2] |
+			       !!(pt_idx2 & (1 << 6)) * rxq->tunnel;
+	pkts[3]->packet_type = mlx5_ptype_table[pt_idx3] |
+			       !!(pt_idx3 & (1 << 6)) * rxq->tunnel;
+	/* Fill flags for checksum and VLAN. */
+	pinfo = _mm_and_si128(pinfo, ptype_ol_mask);
+	pinfo = _mm_shuffle_epi8(cv_flag_sel, pinfo);
+	/* Locate checksum flags at byte[2:1] and merge with VLAN flags. */
+	cv_flags = _mm_slli_epi32(pinfo, 9);
+	cv_flags = _mm_or_si128(pinfo, cv_flags);
+	/* Move back flags to start from byte[0]. */
+	cv_flags = _mm_srli_epi32(cv_flags, 8);
+	/* Mask out garbage bits. */
+	cv_flags = _mm_and_si128(cv_flags, cv_mask);
+	/* Merge to ol_flags. */
+	ol_flags = _mm_or_si128(ol_flags, cv_flags);
+	/* Merge mbuf_init and ol_flags. */
+	rearm0 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(ol_flags, 8), 0x30);
+	rearm1 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(ol_flags, 4), 0x30);
+	rearm2 = _mm_blend_epi16(mbuf_init, ol_flags, 0x30);
+	rearm3 = _mm_blend_epi16(mbuf_init, _mm_srli_si128(ol_flags, 4), 0x30);
+	/* Write 8B rearm_data and 8B ol_flags. */
+	_mm_store_si128((__m128i *)&pkts[0]->rearm_data, rearm0);
+	_mm_store_si128((__m128i *)&pkts[1]->rearm_data, rearm1);
+	_mm_store_si128((__m128i *)&pkts[2]->rearm_data, rearm2);
+	_mm_store_si128((__m128i *)&pkts[3]->rearm_data, rearm3);
+}
+
+/**
+ * Receive burst of packets. An errored completion also consumes a mbuf, but the
+ * packet_type is set to be RTE_PTYPE_ALL_MASK. Marked mbufs should be freed
+ * before returning to application.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ * @param[out] pkts
+ *   Array to store received packets.
+ * @param pkts_n
+ *   Maximum number of packets in array.
+ * @param[out] err
+ *   Pointer to a flag. Set non-zero value if pkts array has at least one error
+ *   packet to handle.
+ *
+ * @return
+ *   Number of packets received including errors (<= pkts_n).
+ */
+static inline uint16_t
+rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t pkts_n,
+	    uint64_t *err)
+{
+	const uint16_t q_n = 1 << rxq->cqe_n;
+	const uint16_t q_mask = q_n - 1;
+	volatile struct mlx5_cqe *cq;
+	struct rte_mbuf **elts;
+	unsigned int pos;
+	uint64_t n;
+	uint16_t repl_n;
+	uint64_t comp_idx = MLX5_VPMD_DESCS_PER_LOOP;
+	uint16_t nocmp_n = 0;
+	uint16_t rcvd_pkt = 0;
+	unsigned int cq_idx = rxq->cq_ci & q_mask;
+	unsigned int elts_idx;
+	unsigned int ownership = !!(rxq->cq_ci & (q_mask + 1));
+	const __m128i owner_check =
+		_mm_set_epi64x(0x0100000001000000LL, 0x0100000001000000LL);
+	const __m128i opcode_check =
+		_mm_set_epi64x(0xf0000000f0000000LL, 0xf0000000f0000000LL);
+	const __m128i format_check =
+		_mm_set_epi64x(0x0c0000000c000000LL, 0x0c0000000c000000LL);
+	const __m128i resp_err_check =
+		_mm_set_epi64x(0xe0000000e0000000LL, 0xe0000000e0000000LL);
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	uint32_t rcvd_byte = 0;
+	/* Mask to shuffle byte_cnt to add up stats. Do bswap16 for all. */
+	const __m128i len_shuf_mask =
+		_mm_set_epi8(-1, -1, -1, -1,
+			     -1, -1, -1, -1,
+			     12, 13,  8,  9,
+			      4,  5,  0,  1);
+#endif
+	/* Mask to shuffle from extracted CQE to mbuf. */
+	const __m128i shuf_mask =
+		_mm_set_epi8(-1,  3,  2,  1, /* fdir.hi */
+			     12, 13, 14, 15, /* rss, bswap32 */
+			     10, 11,         /* vlan_tci, bswap16 */
+			      4,  5,         /* data_len, bswap16 */
+			     -1, -1,         /* zero out 2nd half of pkt_len */
+			      4,  5          /* pkt_len, bswap16 */);
+	/* Mask to blend from the last Qword to the first DQword. */
+	const __m128i blend_mask =
+		_mm_set_epi8(-1, -1, -1, -1,
+			     -1, -1, -1, -1,
+			      0,  0,  0,  0,
+			      0,  0,  0, -1);
+	const __m128i zero = _mm_setzero_si128();
+	const __m128i ones = _mm_cmpeq_epi32(zero, zero);
+	const __m128i crc_adj =
+		_mm_set_epi16(0, 0, 0, 0, 0,
+			      rxq->crc_present * ETHER_CRC_LEN,
+			      0,
+			      rxq->crc_present * ETHER_CRC_LEN);
+	const __m128i flow_mark_adj = _mm_set_epi32(rxq->mark * (-1), 0, 0, 0);
+
+	assert(rxq->sges_n == 0);
+	assert(rxq->cqe_n == rxq->elts_n);
+	cq = &(*rxq->cqes)[cq_idx];
+	rte_prefetch0(cq);
+	rte_prefetch0(cq + 1);
+	rte_prefetch0(cq + 2);
+	rte_prefetch0(cq + 3);
+	pkts_n = RTE_MIN(pkts_n, MLX5_VPMD_RX_MAX_BURST);
+	/*
+	 * Order of indexes:
+	 *   rq_ci >= cq_ci >= rq_pi
+	 * Definition of indexes:
+	 *   rq_ci - cq_ci := # of buffers owned by HW (posted).
+	 *   cq_ci - rq_pi := # of buffers not returned to app (decompressed).
+	 *   N - (rq_ci - rq_pi) := # of buffers consumed (to be replenished).
+	 */
+	repl_n = q_n - (rxq->rq_ci - rxq->rq_pi);
+	if (repl_n >= MLX5_VPMD_RXQ_RPLNSH_THRESH(q_n))
+		mlx5_rx_replenish_bulk_mbuf(rxq, repl_n);
+	/* See if there're unreturned mbufs from compressed CQE. */
+	rcvd_pkt = rxq->cq_ci - rxq->rq_pi;
+	if (rcvd_pkt > 0) {
+		rcvd_pkt = RTE_MIN(rcvd_pkt, pkts_n);
+		rxq_copy_mbuf_v(rxq, pkts, rcvd_pkt);
+		rxq->rq_pi += rcvd_pkt;
+		pkts += rcvd_pkt;
+	}
+	elts_idx = rxq->rq_pi & q_mask;
+	elts = &(*rxq->elts)[elts_idx];
+	/* Not to overflow pkts array. */
+	pkts_n = RTE_ALIGN_FLOOR(pkts_n - rcvd_pkt, MLX5_VPMD_DESCS_PER_LOOP);
+	/* Not to cross queue end. */
+	pkts_n = RTE_MIN(pkts_n, q_n - elts_idx);
+	if (!pkts_n)
+		return rcvd_pkt;
+	/* At this point, there shouldn't be any remained packets. */
+	assert(rxq->rq_pi == rxq->cq_ci);
+	/*
+	 * A. load first Qword (8bytes) in one loop.
+	 * B. copy 4 mbuf pointers from elts ring to returing pkts.
+	 * C. load remained CQE data and extract necessary fields.
+	 *    Final 16bytes cqes[] extracted from original 64bytes CQE has the
+	 *    following structure:
+	 *        struct {
+	 *          uint8_t  pkt_info;
+	 *          uint8_t  flow_tag[3];
+	 *          uint16_t byte_cnt;
+	 *          uint8_t  rsvd4;
+	 *          uint8_t  op_own;
+	 *          uint16_t hdr_type_etc;
+	 *          uint16_t vlan_info;
+	 *          uint32_t rx_has_res;
+	 *        } c;
+	 * D. fill in mbuf.
+	 * E. get valid CQEs.
+	 * F. find compressed CQE.
+	 */
+	for (pos = 0;
+	     pos < pkts_n;
+	     pos += MLX5_VPMD_DESCS_PER_LOOP) {
+		__m128i cqes[MLX5_VPMD_DESCS_PER_LOOP];
+		__m128i cqe_tmp1, cqe_tmp2;
+		__m128i pkt_mb0, pkt_mb1, pkt_mb2, pkt_mb3;
+		__m128i op_own, op_own_tmp1, op_own_tmp2;
+		__m128i opcode, owner_mask, invalid_mask;
+		__m128i comp_mask;
+		__m128i mask;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		__m128i byte_cnt;
+#endif
+		__m128i mbp1, mbp2;
+		__m128i p = _mm_set_epi16(0, 0, 0, 0, 3, 2, 1, 0);
+		unsigned int p1, p2, p3;
+
+		/* Prefetch next 4 CQEs. */
+		if (pkts_n - pos >= 2 * MLX5_VPMD_DESCS_PER_LOOP) {
+			rte_prefetch0(&cq[pos + MLX5_VPMD_DESCS_PER_LOOP]);
+			rte_prefetch0(&cq[pos + MLX5_VPMD_DESCS_PER_LOOP + 1]);
+			rte_prefetch0(&cq[pos + MLX5_VPMD_DESCS_PER_LOOP + 2]);
+			rte_prefetch0(&cq[pos + MLX5_VPMD_DESCS_PER_LOOP + 3]);
+		}
+		/* A.0 do not cross the end of CQ. */
+		mask = _mm_set_epi64x(0, (pkts_n - pos) * sizeof(uint16_t) * 8);
+		mask = _mm_sll_epi64(ones, mask);
+		p = _mm_andnot_si128(mask, p);
+		/* A.1 load cqes. */
+		p3 = _mm_extract_epi16(p, 3);
+		cqes[3] = _mm_loadl_epi64((__m128i *)
+					   &cq[pos + p3].sop_drop_qpn);
+		rte_compiler_barrier();
+		p2 = _mm_extract_epi16(p, 2);
+		cqes[2] = _mm_loadl_epi64((__m128i *)
+					   &cq[pos + p2].sop_drop_qpn);
+		rte_compiler_barrier();
+		/* B.1 load mbuf pointers. */
+		mbp1 = _mm_loadu_si128((__m128i *)&elts[pos]);
+		mbp2 = _mm_loadu_si128((__m128i *)&elts[pos + 2]);
+		/* A.1 load a block having op_own. */
+		p1 = _mm_extract_epi16(p, 1);
+		cqes[1] = _mm_loadl_epi64((__m128i *)
+					   &cq[pos + p1].sop_drop_qpn);
+		rte_compiler_barrier();
+		cqes[0] = _mm_loadl_epi64((__m128i *)
+					   &cq[pos].sop_drop_qpn);
+		/* B.2 copy mbuf pointers. */
+		_mm_storeu_si128((__m128i *)&pkts[pos], mbp1);
+		_mm_storeu_si128((__m128i *)&pkts[pos + 2], mbp2);
+		rte_cio_rmb();
+		/* C.1 load remained CQE data and extract necessary fields. */
+		cqe_tmp2 = _mm_load_si128((__m128i *)&cq[pos + p3]);
+		cqe_tmp1 = _mm_load_si128((__m128i *)&cq[pos + p2]);
+		cqes[3] = _mm_blendv_epi8(cqes[3], cqe_tmp2, blend_mask);
+		cqes[2] = _mm_blendv_epi8(cqes[2], cqe_tmp1, blend_mask);
+		cqe_tmp2 = _mm_loadu_si128((__m128i *)&cq[pos + p3].rsvd1[3]);
+		cqe_tmp1 = _mm_loadu_si128((__m128i *)&cq[pos + p2].rsvd1[3]);
+		cqes[3] = _mm_blend_epi16(cqes[3], cqe_tmp2, 0x30);
+		cqes[2] = _mm_blend_epi16(cqes[2], cqe_tmp1, 0x30);
+		cqe_tmp2 = _mm_loadl_epi64((__m128i *)&cq[pos + p3].rsvd2[10]);
+		cqe_tmp1 = _mm_loadl_epi64((__m128i *)&cq[pos + p2].rsvd2[10]);
+		cqes[3] = _mm_blend_epi16(cqes[3], cqe_tmp2, 0x04);
+		cqes[2] = _mm_blend_epi16(cqes[2], cqe_tmp1, 0x04);
+		/* C.2 generate final structure for mbuf with swapping bytes. */
+		pkt_mb3 = _mm_shuffle_epi8(cqes[3], shuf_mask);
+		pkt_mb2 = _mm_shuffle_epi8(cqes[2], shuf_mask);
+		/* C.3 adjust CRC length. */
+		pkt_mb3 = _mm_sub_epi16(pkt_mb3, crc_adj);
+		pkt_mb2 = _mm_sub_epi16(pkt_mb2, crc_adj);
+		/* C.4 adjust flow mark. */
+		pkt_mb3 = _mm_add_epi32(pkt_mb3, flow_mark_adj);
+		pkt_mb2 = _mm_add_epi32(pkt_mb2, flow_mark_adj);
+		/* D.1 fill in mbuf - rx_descriptor_fields1. */
+		_mm_storeu_si128((void *)&pkts[pos + 3]->pkt_len, pkt_mb3);
+		_mm_storeu_si128((void *)&pkts[pos + 2]->pkt_len, pkt_mb2);
+		/* E.1 extract op_own field. */
+		op_own_tmp2 = _mm_unpacklo_epi32(cqes[2], cqes[3]);
+		/* C.1 load remained CQE data and extract necessary fields. */
+		cqe_tmp2 = _mm_load_si128((__m128i *)&cq[pos + p1]);
+		cqe_tmp1 = _mm_load_si128((__m128i *)&cq[pos]);
+		cqes[1] = _mm_blendv_epi8(cqes[1], cqe_tmp2, blend_mask);
+		cqes[0] = _mm_blendv_epi8(cqes[0], cqe_tmp1, blend_mask);
+		cqe_tmp2 = _mm_loadu_si128((__m128i *)&cq[pos + p1].rsvd1[3]);
+		cqe_tmp1 = _mm_loadu_si128((__m128i *)&cq[pos].rsvd1[3]);
+		cqes[1] = _mm_blend_epi16(cqes[1], cqe_tmp2, 0x30);
+		cqes[0] = _mm_blend_epi16(cqes[0], cqe_tmp1, 0x30);
+		cqe_tmp2 = _mm_loadl_epi64((__m128i *)&cq[pos + p1].rsvd2[10]);
+		cqe_tmp1 = _mm_loadl_epi64((__m128i *)&cq[pos].rsvd2[10]);
+		cqes[1] = _mm_blend_epi16(cqes[1], cqe_tmp2, 0x04);
+		cqes[0] = _mm_blend_epi16(cqes[0], cqe_tmp1, 0x04);
+		/* C.2 generate final structure for mbuf with swapping bytes. */
+		pkt_mb1 = _mm_shuffle_epi8(cqes[1], shuf_mask);
+		pkt_mb0 = _mm_shuffle_epi8(cqes[0], shuf_mask);
+		/* C.3 adjust CRC length. */
+		pkt_mb1 = _mm_sub_epi16(pkt_mb1, crc_adj);
+		pkt_mb0 = _mm_sub_epi16(pkt_mb0, crc_adj);
+		/* C.4 adjust flow mark. */
+		pkt_mb1 = _mm_add_epi32(pkt_mb1, flow_mark_adj);
+		pkt_mb0 = _mm_add_epi32(pkt_mb0, flow_mark_adj);
+		/* E.1 extract op_own byte. */
+		op_own_tmp1 = _mm_unpacklo_epi32(cqes[0], cqes[1]);
+		op_own = _mm_unpackhi_epi64(op_own_tmp1, op_own_tmp2);
+		/* D.1 fill in mbuf - rx_descriptor_fields1. */
+		_mm_storeu_si128((void *)&pkts[pos + 1]->pkt_len, pkt_mb1);
+		_mm_storeu_si128((void *)&pkts[pos]->pkt_len, pkt_mb0);
+		/* E.2 flip owner bit to mark CQEs from last round. */
+		owner_mask = _mm_and_si128(op_own, owner_check);
+		if (ownership)
+			owner_mask = _mm_xor_si128(owner_mask, owner_check);
+		owner_mask = _mm_cmpeq_epi32(owner_mask, owner_check);
+		owner_mask = _mm_packs_epi32(owner_mask, zero);
+		/* E.3 get mask for invalidated CQEs. */
+		opcode = _mm_and_si128(op_own, opcode_check);
+		invalid_mask = _mm_cmpeq_epi32(opcode_check, opcode);
+		invalid_mask = _mm_packs_epi32(invalid_mask, zero);
+		/* E.4 mask out beyond boundary. */
+		invalid_mask = _mm_or_si128(invalid_mask, mask);
+		/* E.5 merge invalid_mask with invalid owner. */
+		invalid_mask = _mm_or_si128(invalid_mask, owner_mask);
+		/* F.1 find compressed CQE format. */
+		comp_mask = _mm_and_si128(op_own, format_check);
+		comp_mask = _mm_cmpeq_epi32(comp_mask, format_check);
+		comp_mask = _mm_packs_epi32(comp_mask, zero);
+		/* F.2 mask out invalid entries. */
+		comp_mask = _mm_andnot_si128(invalid_mask, comp_mask);
+		comp_idx = _mm_cvtsi128_si64(comp_mask);
+		/* F.3 get the first compressed CQE. */
+		comp_idx = comp_idx ?
+				__builtin_ctzll(comp_idx) /
+					(sizeof(uint16_t) * 8) :
+				MLX5_VPMD_DESCS_PER_LOOP;
+		/* E.6 mask out entries after the compressed CQE. */
+		mask = _mm_set_epi64x(0, comp_idx * sizeof(uint16_t) * 8);
+		mask = _mm_sll_epi64(ones, mask);
+		invalid_mask = _mm_or_si128(invalid_mask, mask);
+		/* E.7 count non-compressed valid CQEs. */
+		n = _mm_cvtsi128_si64(invalid_mask);
+		n = n ? __builtin_ctzll(n) / (sizeof(uint16_t) * 8) :
+			MLX5_VPMD_DESCS_PER_LOOP;
+		nocmp_n += n;
+		/* D.2 get the final invalid mask. */
+		mask = _mm_set_epi64x(0, n * sizeof(uint16_t) * 8);
+		mask = _mm_sll_epi64(ones, mask);
+		invalid_mask = _mm_or_si128(invalid_mask, mask);
+		/* D.3 check error in opcode. */
+		opcode = _mm_cmpeq_epi32(resp_err_check, opcode);
+		opcode = _mm_packs_epi32(opcode, zero);
+		opcode = _mm_andnot_si128(invalid_mask, opcode);
+		/* D.4 mark if any error is set */
+		*err |= _mm_cvtsi128_si64(opcode);
+		/* D.5 fill in mbuf - rearm_data and packet_type. */
+		rxq_cq_to_ptype_oflags_v(rxq, cqes, opcode, &pkts[pos]);
+		if (rxq->hw_timestamp) {
+			pkts[pos]->timestamp =
+				rte_be_to_cpu_64(cq[pos].timestamp);
+			pkts[pos + 1]->timestamp =
+				rte_be_to_cpu_64(cq[pos + p1].timestamp);
+			pkts[pos + 2]->timestamp =
+				rte_be_to_cpu_64(cq[pos + p2].timestamp);
+			pkts[pos + 3]->timestamp =
+				rte_be_to_cpu_64(cq[pos + p3].timestamp);
+		}
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		/* Add up received bytes count. */
+		byte_cnt = _mm_shuffle_epi8(op_own, len_shuf_mask);
+		byte_cnt = _mm_andnot_si128(invalid_mask, byte_cnt);
+		byte_cnt = _mm_hadd_epi16(byte_cnt, zero);
+		rcvd_byte += _mm_cvtsi128_si64(_mm_hadd_epi16(byte_cnt, zero));
+#endif
+		/*
+		 * Break the loop unless more valid CQE is expected, or if
+		 * there's a compressed CQE.
+		 */
+		if (n != MLX5_VPMD_DESCS_PER_LOOP)
+			break;
+	}
+	/* If no new CQE seen, return without updating cq_db. */
+	if (unlikely(!nocmp_n && comp_idx == MLX5_VPMD_DESCS_PER_LOOP))
+		return rcvd_pkt;
+	/* Update the consumer indexes for non-compressed CQEs. */
+	assert(nocmp_n <= pkts_n);
+	rxq->cq_ci += nocmp_n;
+	rxq->rq_pi += nocmp_n;
+	rcvd_pkt += nocmp_n;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	rxq->stats.ipackets += nocmp_n;
+	rxq->stats.ibytes += rcvd_byte;
+#endif
+	/* Decompress the last CQE if compressed. */
+	if (comp_idx < MLX5_VPMD_DESCS_PER_LOOP && comp_idx == n) {
+		assert(comp_idx == (nocmp_n % MLX5_VPMD_DESCS_PER_LOOP));
+		rxq_cq_decompress_v(rxq, &cq[nocmp_n], &elts[nocmp_n]);
+		/* Return more packets if needed. */
+		if (nocmp_n < pkts_n) {
+			uint16_t n = rxq->cq_ci - rxq->rq_pi;
+
+			n = RTE_MIN(n, pkts_n - nocmp_n);
+			rxq_copy_mbuf_v(rxq, &pkts[nocmp_n], n);
+			rxq->rq_pi += n;
+			rcvd_pkt += n;
+		}
+	}
+	rte_compiler_barrier();
+	*rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci);
+	return rcvd_pkt;
+}
+
+#endif /* RTE_PMD_MLX5_RXTX_VEC_SSE_H_ */
diff --git a/src/spdk/dpdk/drivers/net/mlx5/mlx5_socket.c b/src/spdk/dpdk/drivers/net/mlx5/mlx5_socket.c
new file mode 100644
index 00000000..a3a52291
--- /dev/null
+++ b/src/spdk/dpdk/drivers/net/mlx5/mlx5_socket.c
@@ -0,0 +1,308 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2016 6WIND S.A.
+ * Copyright 2016 Mellanox Technologies, Ltd
+ */
+
+#define _GNU_SOURCE
+
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <sys/stat.h>
+
+#include "mlx5.h"
+#include "mlx5_utils.h"
+
+/**
+ * Initialise the socket to communicate with the secondary process
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_socket_init(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct sockaddr_un sun = {
+		.sun_family = AF_UNIX,
+	};
+	int ret;
+	int flags;
+
+	/*
+	 * Close the last socket that was used to communicate
+	 * with the secondary process
+	 */
+	if (priv->primary_socket)
+		mlx5_socket_uninit(dev);
+	/*
+	 * Initialise the socket to communicate with the secondary
+	 * process.
+	 */
+	ret = socket(AF_UNIX, SOCK_STREAM, 0);
+	if (ret < 0) {
+		rte_errno = errno;
+		DRV_LOG(WARNING, "port %u secondary process not supported: %s",
+			dev->data->port_id, strerror(errno));
+		goto error;
+	}
+	priv->primary_socket = ret;
+	flags = fcntl(priv->primary_socket, F_GETFL, 0);
+	if (flags == -1) {
+		rte_errno = errno;
+		goto error;
+	}
+	ret = fcntl(priv->primary_socket, F_SETFL, flags | O_NONBLOCK);
+	if (ret < 0) {
+		rte_errno = errno;
+		goto error;
+	}
+	snprintf(sun.sun_path, sizeof(sun.sun_path), "/var/tmp/%s_%d",
+		 MLX5_DRIVER_NAME, priv->primary_socket);
+	remove(sun.sun_path);
+	ret = bind(priv->primary_socket, (const struct sockaddr *)&sun,
+		   sizeof(sun));
+	if (ret < 0) {
+		rte_errno = errno;
+		DRV_LOG(WARNING,
+			"port %u cannot bind socket, secondary process not"
+			" supported: %s",
+			dev->data->port_id, strerror(errno));
+		goto close;
+	}
+	ret = listen(priv->primary_socket, 0);
+	if (ret < 0) {
+		rte_errno = errno;
+		DRV_LOG(WARNING, "port %u secondary process not supported: %s",
+			dev->data->port_id, strerror(errno));
+		goto close;
+	}
+	return 0;
+close:
+	remove(sun.sun_path);
+error:
+	claim_zero(close(priv->primary_socket));
+	priv->primary_socket = 0;
+	return -rte_errno;
+}
+
+/**
+ * Un-Initialise the socket to communicate with the secondary process
+ *
+ * @param[in] dev
+ */
+void
+mlx5_socket_uninit(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+
+	MKSTR(path, "/var/tmp/%s_%d", MLX5_DRIVER_NAME, priv->primary_socket);
+	claim_zero(close(priv->primary_socket));
+	priv->primary_socket = 0;
+	claim_zero(remove(path));
+}
+
+/**
+ * Handle socket interrupts.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ */
+void
+mlx5_socket_handle(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	int conn_sock;
+	int ret = 0;
+	struct cmsghdr *cmsg = NULL;
+	struct ucred *cred = NULL;
+	char buf[CMSG_SPACE(sizeof(struct ucred))] = { 0 };
+	char vbuf[1024] = { 0 };
+	struct iovec io = {
+		.iov_base = vbuf,
+		.iov_len = sizeof(*vbuf),
+	};
+	struct msghdr msg = {
+		.msg_iov = &io,
+		.msg_iovlen = 1,
+		.msg_control = buf,
+		.msg_controllen = sizeof(buf),
+	};
+	int *fd;
+
+	/* Accept the connection from the client. */
+	conn_sock = accept(priv->primary_socket, NULL, NULL);
+	if (conn_sock < 0) {
+		DRV_LOG(WARNING, "port %u connection failed: %s",
+			dev->data->port_id, strerror(errno));
+		return;
+	}
+	ret = setsockopt(conn_sock, SOL_SOCKET, SO_PASSCRED, &(int){1},
+					 sizeof(int));
+	if (ret < 0) {
+		ret = errno;
+		DRV_LOG(WARNING, "port %u cannot change socket options: %s",
+			dev->data->port_id, strerror(rte_errno));
+		goto error;
+	}
+	ret = recvmsg(conn_sock, &msg, MSG_WAITALL);
+	if (ret < 0) {
+		ret = errno;
+		DRV_LOG(WARNING, "port %u received an empty message: %s",
+			dev->data->port_id, strerror(rte_errno));
+		goto error;
+	}
+	/* Expect to receive credentials only. */
+	cmsg = CMSG_FIRSTHDR(&msg);
+	if (cmsg == NULL) {
+		DRV_LOG(WARNING, "port %u no message", dev->data->port_id);
+		goto error;
+	}
+	if ((cmsg->cmsg_type == SCM_CREDENTIALS) &&
+		(cmsg->cmsg_len >= sizeof(*cred))) {
+		cred = (struct ucred *)CMSG_DATA(cmsg);
+		assert(cred != NULL);
+	}
+	cmsg = CMSG_NXTHDR(&msg, cmsg);
+	if (cmsg != NULL) {
+		DRV_LOG(WARNING, "port %u message wrongly formatted",
+			dev->data->port_id);
+		goto error;
+	}
+	/* Make sure all the ancillary data was received and valid. */
+	if ((cred == NULL) || (cred->uid != getuid()) ||
+	    (cred->gid != getgid())) {
+		DRV_LOG(WARNING, "port %u wrong credentials",
+			dev->data->port_id);
+		goto error;
+	}
+	/* Set-up the ancillary data. */
+	cmsg = CMSG_FIRSTHDR(&msg);
+	assert(cmsg != NULL);
+	cmsg->cmsg_level = SOL_SOCKET;
+	cmsg->cmsg_type = SCM_RIGHTS;
+	cmsg->cmsg_len = CMSG_LEN(sizeof(priv->ctx->cmd_fd));
+	fd = (int *)CMSG_DATA(cmsg);
+	*fd = priv->ctx->cmd_fd;
+	ret = sendmsg(conn_sock, &msg, 0);
+	if (ret < 0)
+		DRV_LOG(WARNING, "port %u cannot send response",
+			dev->data->port_id);
+error:
+	close(conn_sock);
+}
+
+/**
+ * Connect to the primary process.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet structure.
+ *
+ * @return
+ *   fd on success, negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_socket_connect(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct sockaddr_un sun = {
+		.sun_family = AF_UNIX,
+	};
+	int socket_fd = -1;
+	int *fd = NULL;
+	int ret;
+	struct ucred *cred;
+	char buf[CMSG_SPACE(sizeof(*cred))] = { 0 };
+	char vbuf[1024] = { 0 };
+	struct iovec io = {
+		.iov_base = vbuf,
+		.iov_len = sizeof(*vbuf),
+	};
+	struct msghdr msg = {
+		.msg_control = buf,
+		.msg_controllen = sizeof(buf),
+		.msg_iov = &io,
+		.msg_iovlen = 1,
+	};
+	struct cmsghdr *cmsg;
+
+	ret = socket(AF_UNIX, SOCK_STREAM, 0);
+	if (ret < 0) {
+		rte_errno = errno;
+		DRV_LOG(WARNING, "port %u cannot connect to primary",
+			dev->data->port_id);
+		goto error;
+	}
+	socket_fd = ret;
+	snprintf(sun.sun_path, sizeof(sun.sun_path), "/var/tmp/%s_%d",
+		 MLX5_DRIVER_NAME, priv->primary_socket);
+	ret = connect(socket_fd, (const struct sockaddr *)&sun, sizeof(sun));
+	if (ret < 0) {
+		rte_errno = errno;
+		DRV_LOG(WARNING, "port %u cannot connect to primary",
+			dev->data->port_id);
+		goto error;
+	}
+	cmsg = CMSG_FIRSTHDR(&msg);
+	if (cmsg == NULL) {
+		rte_errno = EINVAL;
+		DRV_LOG(DEBUG, "port %u cannot get first message",
+			dev->data->port_id);
+		goto error;
+	}
+	cmsg->cmsg_level = SOL_SOCKET;
+	cmsg->cmsg_type = SCM_CREDENTIALS;
+	cmsg->cmsg_len = CMSG_LEN(sizeof(*cred));
+	cred = (struct ucred *)CMSG_DATA(cmsg);
+	if (cred == NULL) {
+		rte_errno = EINVAL;
+		DRV_LOG(DEBUG, "port %u no credentials received",
+			dev->data->port_id);
+		goto error;
+	}
+	cred->pid = getpid();
+	cred->uid = getuid();
+	cred->gid = getgid();
+	ret = sendmsg(socket_fd, &msg, MSG_DONTWAIT);
+	if (ret < 0) {
+		rte_errno = errno;
+		DRV_LOG(WARNING,
+			"port %u cannot send credentials to primary: %s",
+			dev->data->port_id, strerror(errno));
+		goto error;
+	}
+	ret = recvmsg(socket_fd, &msg, MSG_WAITALL);
+	if (ret <= 0) {
+		rte_errno = errno;
+		DRV_LOG(WARNING, "port %u no message from primary: %s",
+			dev->data->port_id, strerror(errno));
+		goto error;
+	}
+	cmsg = CMSG_FIRSTHDR(&msg);
+	if (cmsg == NULL) {
+		rte_errno = EINVAL;
+		DRV_LOG(WARNING, "port %u no file descriptor received",
+			dev->data->port_id);
+		goto error;
+	}
+	fd = (int *)CMSG_DATA(cmsg);
+	if (*fd < 0) {
+		DRV_LOG(WARNING, "port %u no file descriptor received: %s",
+			dev->data->port_id, strerror(errno));
+		rte_errno = *fd;
+		goto error;
+	}
+	ret = *fd;
+	close(socket_fd);
+	return ret;
+error:
+	if (socket_fd != -1)
+		close(socket_fd);
+	return -rte_errno;
+}
diff --git a/src/spdk/dpdk/drivers/net/mlx5/mlx5_stats.c b/src/spdk/dpdk/drivers/net/mlx5/mlx5_stats.c
new file mode 100644
index 00000000..91f3d474
--- /dev/null
+++ b/src/spdk/dpdk/drivers/net/mlx5/mlx5_stats.c
@@ -0,0 +1,494 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2015 6WIND S.A.
+ * Copyright 2015 Mellanox Technologies, Ltd
+ */
+
+#include <inttypes.h>
+#include <linux/sockios.h>
+#include <linux/ethtool.h>
+#include <stdint.h>
+#include <stdio.h>
+
+#include <rte_ethdev_driver.h>
+#include <rte_common.h>
+#include <rte_malloc.h>
+
+#include "mlx5.h"
+#include "mlx5_rxtx.h"
+#include "mlx5_defs.h"
+
+struct mlx5_counter_ctrl {
+	/* Name of the counter. */
+	char dpdk_name[RTE_ETH_XSTATS_NAME_SIZE];
+	/* Name of the counter on the device table. */
+	char ctr_name[RTE_ETH_XSTATS_NAME_SIZE];
+	uint32_t ib:1; /**< Nonzero for IB counters. */
+};
+
+static const struct mlx5_counter_ctrl mlx5_counters_init[] = {
+	{
+		.dpdk_name = "rx_port_unicast_bytes",
+		.ctr_name = "rx_vport_unicast_bytes",
+	},
+	{
+		.dpdk_name = "rx_port_multicast_bytes",
+		.ctr_name = "rx_vport_multicast_bytes",
+	},
+	{
+		.dpdk_name = "rx_port_broadcast_bytes",
+		.ctr_name = "rx_vport_broadcast_bytes",
+	},
+	{
+		.dpdk_name = "rx_port_unicast_packets",
+		.ctr_name = "rx_vport_unicast_packets",
+	},
+	{
+		.dpdk_name = "rx_port_multicast_packets",
+		.ctr_name = "rx_vport_multicast_packets",
+	},
+	{
+		.dpdk_name = "rx_port_broadcast_packets",
+		.ctr_name = "rx_vport_broadcast_packets",
+	},
+	{
+		.dpdk_name = "tx_port_unicast_bytes",
+		.ctr_name = "tx_vport_unicast_bytes",
+	},
+	{
+		.dpdk_name = "tx_port_multicast_bytes",
+		.ctr_name = "tx_vport_multicast_bytes",
+	},
+	{
+		.dpdk_name = "tx_port_broadcast_bytes",
+		.ctr_name = "tx_vport_broadcast_bytes",
+	},
+	{
+		.dpdk_name = "tx_port_unicast_packets",
+		.ctr_name = "tx_vport_unicast_packets",
+	},
+	{
+		.dpdk_name = "tx_port_multicast_packets",
+		.ctr_name = "tx_vport_multicast_packets",
+	},
+	{
+		.dpdk_name = "tx_port_broadcast_packets",
+		.ctr_name = "tx_vport_broadcast_packets",
+	},
+	{
+		.dpdk_name = "rx_wqe_err",
+		.ctr_name = "rx_wqe_err",
+	},
+	{
+		.dpdk_name = "rx_crc_errors_phy",
+		.ctr_name = "rx_crc_errors_phy",
+	},
+	{
+		.dpdk_name = "rx_in_range_len_errors_phy",
+		.ctr_name = "rx_in_range_len_errors_phy",
+	},
+	{
+		.dpdk_name = "rx_symbol_err_phy",
+		.ctr_name = "rx_symbol_err_phy",
+	},
+	{
+		.dpdk_name = "tx_errors_phy",
+		.ctr_name = "tx_errors_phy",
+	},
+	{
+		.dpdk_name = "rx_out_of_buffer",
+		.ctr_name = "out_of_buffer",
+		.ib = 1,
+	},
+	{
+		.dpdk_name = "tx_packets_phy",
+		.ctr_name = "tx_packets_phy",
+	},
+	{
+		.dpdk_name = "rx_packets_phy",
+		.ctr_name = "rx_packets_phy",
+	},
+	{
+		.dpdk_name = "tx_bytes_phy",
+		.ctr_name = "tx_bytes_phy",
+	},
+	{
+		.dpdk_name = "rx_bytes_phy",
+		.ctr_name = "rx_bytes_phy",
+	},
+};
+
+static const unsigned int xstats_n = RTE_DIM(mlx5_counters_init);
+
+/**
+ * Read device counters table.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param[out] stats
+ *   Counters table output buffer.
+ *
+ * @return
+ *   0 on success and stats is filled, negative errno value otherwise and
+ *   rte_errno is set.
+ */
+static int
+mlx5_read_dev_counters(struct rte_eth_dev *dev, uint64_t *stats)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
+	unsigned int i;
+	struct ifreq ifr;
+	unsigned int stats_sz = xstats_ctrl->stats_n * sizeof(uint64_t);
+	unsigned char et_stat_buf[sizeof(struct ethtool_stats) + stats_sz];
+	struct ethtool_stats *et_stats = (struct ethtool_stats *)et_stat_buf;
+	int ret;
+
+	et_stats->cmd = ETHTOOL_GSTATS;
+	et_stats->n_stats = xstats_ctrl->stats_n;
+	ifr.ifr_data = (caddr_t)et_stats;
+	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr, 1);
+	if (ret) {
+		DRV_LOG(WARNING,
+			"port %u unable to read statistic values from device",
+			dev->data->port_id);
+		return ret;
+	}
+	for (i = 0; i != xstats_n; ++i) {
+		if (mlx5_counters_init[i].ib) {
+			FILE *file;
+			MKSTR(path, "%s/ports/1/hw_counters/%s",
+			      priv->ibdev_path,
+			      mlx5_counters_init[i].ctr_name);
+
+			file = fopen(path, "rb");
+			if (file) {
+				int n = fscanf(file, "%" SCNu64, &stats[i]);
+
+				fclose(file);
+				if (n != 1)
+					stats[i] = 0;
+			}
+		} else {
+			stats[i] = (uint64_t)
+				et_stats->data[xstats_ctrl->dev_table_idx[i]];
+		}
+	}
+	return 0;
+}
+
+/**
+ * Query the number of statistics provided by ETHTOOL.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   Number of statistics on success, negative errno value otherwise and
+ *   rte_errno is set.
+ */
+static int
+mlx5_ethtool_get_stats_n(struct rte_eth_dev *dev) {
+	struct ethtool_drvinfo drvinfo;
+	struct ifreq ifr;
+	int ret;
+
+	drvinfo.cmd = ETHTOOL_GDRVINFO;
+	ifr.ifr_data = (caddr_t)&drvinfo;
+	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr, 1);
+	if (ret) {
+		DRV_LOG(WARNING, "port %u unable to query number of statistics",
+			dev->data->port_id);
+		return ret;
+	}
+	return drvinfo.n_stats;
+}
+
+/**
+ * Init the structures to read device counters.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ */
+void
+mlx5_xstats_init(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
+	unsigned int i;
+	unsigned int j;
+	struct ifreq ifr;
+	struct ethtool_gstrings *strings = NULL;
+	unsigned int dev_stats_n;
+	unsigned int str_sz;
+	int ret;
+
+	ret = mlx5_ethtool_get_stats_n(dev);
+	if (ret < 0) {
+		DRV_LOG(WARNING, "port %u no extended statistics available",
+			dev->data->port_id);
+		return;
+	}
+	dev_stats_n = ret;
+	xstats_ctrl->stats_n = dev_stats_n;
+	/* Allocate memory to grab stat names and values. */
+	str_sz = dev_stats_n * ETH_GSTRING_LEN;
+	strings = (struct ethtool_gstrings *)
+		  rte_malloc("xstats_strings",
+			     str_sz + sizeof(struct ethtool_gstrings), 0);
+	if (!strings) {
+		DRV_LOG(WARNING, "port %u unable to allocate memory for xstats",
+		     dev->data->port_id);
+		return;
+	}
+	strings->cmd = ETHTOOL_GSTRINGS;
+	strings->string_set = ETH_SS_STATS;
+	strings->len = dev_stats_n;
+	ifr.ifr_data = (caddr_t)strings;
+	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr, 1);
+	if (ret) {
+		DRV_LOG(WARNING, "port %u unable to get statistic names",
+			dev->data->port_id);
+		goto free;
+	}
+	for (j = 0; j != xstats_n; ++j)
+		xstats_ctrl->dev_table_idx[j] = dev_stats_n;
+	for (i = 0; i != dev_stats_n; ++i) {
+		const char *curr_string = (const char *)
+			&strings->data[i * ETH_GSTRING_LEN];
+
+		for (j = 0; j != xstats_n; ++j) {
+			if (!strcmp(mlx5_counters_init[j].ctr_name,
+				    curr_string)) {
+				xstats_ctrl->dev_table_idx[j] = i;
+				break;
+			}
+		}
+	}
+	for (j = 0; j != xstats_n; ++j) {
+		if (mlx5_counters_init[j].ib)
+			continue;
+		if (xstats_ctrl->dev_table_idx[j] >= dev_stats_n) {
+			DRV_LOG(WARNING,
+				"port %u counter \"%s\" is not recognized",
+				dev->data->port_id,
+				mlx5_counters_init[j].dpdk_name);
+			goto free;
+		}
+	}
+	/* Copy to base at first time. */
+	assert(xstats_n <= MLX5_MAX_XSTATS);
+	ret = mlx5_read_dev_counters(dev, xstats_ctrl->base);
+	if (ret)
+		DRV_LOG(ERR, "port %u cannot read device counters: %s",
+			dev->data->port_id, strerror(rte_errno));
+free:
+	rte_free(strings);
+}
+
+/**
+ * DPDK callback to get extended device statistics.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param[out] stats
+ *   Pointer to rte extended stats table.
+ * @param n
+ *   The size of the stats table.
+ *
+ * @return
+ *   Number of extended stats on success and stats is filled,
+ *   negative on error and rte_errno is set.
+ */
+int
+mlx5_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *stats,
+		unsigned int n)
+{
+	struct priv *priv = dev->data->dev_private;
+	unsigned int i;
+	uint64_t counters[n];
+
+	if (n >= xstats_n && stats) {
+		struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
+		int stats_n;
+		int ret;
+
+		stats_n = mlx5_ethtool_get_stats_n(dev);
+		if (stats_n < 0)
+			return stats_n;
+		if (xstats_ctrl->stats_n != stats_n)
+			mlx5_xstats_init(dev);
+		ret = mlx5_read_dev_counters(dev, counters);
+		if (ret)
+			return ret;
+		for (i = 0; i != xstats_n; ++i) {
+			stats[i].id = i;
+			stats[i].value = (counters[i] - xstats_ctrl->base[i]);
+		}
+	}
+	return xstats_n;
+}
+
+/**
+ * DPDK callback to get device statistics.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param[out] stats
+ *   Stats structure output buffer.
+ *
+ * @return
+ *   0 on success and stats is filled, negative errno value otherwise and
+ *   rte_errno is set.
+ */
+int
+mlx5_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct rte_eth_stats tmp = {0};
+	unsigned int i;
+	unsigned int idx;
+
+	/* Add software counters. */
+	for (i = 0; (i != priv->rxqs_n); ++i) {
+		struct mlx5_rxq_data *rxq = (*priv->rxqs)[i];
+
+		if (rxq == NULL)
+			continue;
+		idx = rxq->stats.idx;
+		if (idx < RTE_ETHDEV_QUEUE_STAT_CNTRS) {
+#ifdef MLX5_PMD_SOFT_COUNTERS
+			tmp.q_ipackets[idx] += rxq->stats.ipackets;
+			tmp.q_ibytes[idx] += rxq->stats.ibytes;
+#endif
+			tmp.q_errors[idx] += (rxq->stats.idropped +
+					      rxq->stats.rx_nombuf);
+		}
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		tmp.ipackets += rxq->stats.ipackets;
+		tmp.ibytes += rxq->stats.ibytes;
+#endif
+		tmp.ierrors += rxq->stats.idropped;
+		tmp.rx_nombuf += rxq->stats.rx_nombuf;
+	}
+	for (i = 0; (i != priv->txqs_n); ++i) {
+		struct mlx5_txq_data *txq = (*priv->txqs)[i];
+
+		if (txq == NULL)
+			continue;
+		idx = txq->stats.idx;
+		if (idx < RTE_ETHDEV_QUEUE_STAT_CNTRS) {
+#ifdef MLX5_PMD_SOFT_COUNTERS
+			tmp.q_opackets[idx] += txq->stats.opackets;
+			tmp.q_obytes[idx] += txq->stats.obytes;
+#endif
+			tmp.q_errors[idx] += txq->stats.oerrors;
+		}
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		tmp.opackets += txq->stats.opackets;
+		tmp.obytes += txq->stats.obytes;
+#endif
+		tmp.oerrors += txq->stats.oerrors;
+	}
+#ifndef MLX5_PMD_SOFT_COUNTERS
+	/* FIXME: retrieve and add hardware counters. */
+#endif
+	*stats = tmp;
+	return 0;
+}
+
+/**
+ * DPDK callback to clear device statistics.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ */
+void
+mlx5_stats_reset(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	unsigned int i;
+	unsigned int idx;
+
+	for (i = 0; (i != priv->rxqs_n); ++i) {
+		if ((*priv->rxqs)[i] == NULL)
+			continue;
+		idx = (*priv->rxqs)[i]->stats.idx;
+		(*priv->rxqs)[i]->stats =
+			(struct mlx5_rxq_stats){ .idx = idx };
+	}
+	for (i = 0; (i != priv->txqs_n); ++i) {
+		if ((*priv->txqs)[i] == NULL)
+			continue;
+		idx = (*priv->txqs)[i]->stats.idx;
+		(*priv->txqs)[i]->stats =
+			(struct mlx5_txq_stats){ .idx = idx };
+	}
+#ifndef MLX5_PMD_SOFT_COUNTERS
+	/* FIXME: reset hardware counters. */
+#endif
+}
+
+/**
+ * DPDK callback to clear device extended statistics.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ */
+void
+mlx5_xstats_reset(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
+	int stats_n;
+	unsigned int i;
+	unsigned int n = xstats_n;
+	uint64_t counters[n];
+	int ret;
+
+	stats_n = mlx5_ethtool_get_stats_n(dev);
+	if (stats_n < 0) {
+		DRV_LOG(ERR, "port %u cannot get stats: %s", dev->data->port_id,
+			strerror(-stats_n));
+		return;
+	}
+	if (xstats_ctrl->stats_n != stats_n)
+		mlx5_xstats_init(dev);
+	ret = mlx5_read_dev_counters(dev, counters);
+	if (ret) {
+		DRV_LOG(ERR, "port %u cannot read device counters: %s",
+			dev->data->port_id, strerror(rte_errno));
+		return;
+	}
+	for (i = 0; i != n; ++i)
+		xstats_ctrl->base[i] = counters[i];
+}
+
+/**
+ * DPDK callback to retrieve names of extended device statistics
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param[out] xstats_names
+ *   Buffer to insert names into.
+ * @param n
+ *   Number of names.
+ *
+ * @return
+ *   Number of xstats names.
+ */
+int
+mlx5_xstats_get_names(struct rte_eth_dev *dev __rte_unused,
+		      struct rte_eth_xstat_name *xstats_names, unsigned int n)
+{
+	unsigned int i;
+
+	if (n >= xstats_n && xstats_names) {
+		for (i = 0; i != xstats_n; ++i) {
+			strncpy(xstats_names[i].name,
+				mlx5_counters_init[i].dpdk_name,
+				RTE_ETH_XSTATS_NAME_SIZE);
+			xstats_names[i].name[RTE_ETH_XSTATS_NAME_SIZE - 1] = 0;
+		}
+	}
+	return xstats_n;
+}
diff --git a/src/spdk/dpdk/drivers/net/mlx5/mlx5_trigger.c b/src/spdk/dpdk/drivers/net/mlx5/mlx5_trigger.c
new file mode 100644
index 00000000..e2a9bb70
--- /dev/null
+++ b/src/spdk/dpdk/drivers/net/mlx5/mlx5_trigger.c
@@ -0,0 +1,404 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2015 6WIND S.A.
+ * Copyright 2015 Mellanox Technologies, Ltd
+ */
+
+#include <unistd.h>
+
+#include <rte_ether.h>
+#include <rte_ethdev_driver.h>
+#include <rte_interrupts.h>
+#include <rte_alarm.h>
+
+#include "mlx5.h"
+#include "mlx5_rxtx.h"
+#include "mlx5_utils.h"
+
+/**
+ * Stop traffic on Tx queues.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ */
+static void
+mlx5_txq_stop(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	unsigned int i;
+
+	for (i = 0; i != priv->txqs_n; ++i)
+		mlx5_txq_release(dev, i);
+}
+
+/**
+ * Start traffic on Tx queues.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_txq_start(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	unsigned int i;
+	int ret;
+
+	for (i = 0; i != priv->txqs_n; ++i) {
+		struct mlx5_txq_ctrl *txq_ctrl = mlx5_txq_get(dev, i);
+
+		if (!txq_ctrl)
+			continue;
+		txq_alloc_elts(txq_ctrl);
+		txq_ctrl->ibv = mlx5_txq_ibv_new(dev, i);
+		if (!txq_ctrl->ibv) {
+			rte_errno = ENOMEM;
+			goto error;
+		}
+	}
+	ret = mlx5_tx_uar_remap(dev, priv->ctx->cmd_fd);
+	if (ret) {
+		/* Adjust index for rollback. */
+		i = priv->txqs_n - 1;
+		goto error;
+	}
+	return 0;
+error:
+	ret = rte_errno; /* Save rte_errno before cleanup. */
+	do {
+		mlx5_txq_release(dev, i);
+	} while (i-- != 0);
+	rte_errno = ret; /* Restore rte_errno. */
+	return -rte_errno;
+}
+
+/**
+ * Stop traffic on Rx queues.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ */
+static void
+mlx5_rxq_stop(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	unsigned int i;
+
+	for (i = 0; i != priv->rxqs_n; ++i)
+		mlx5_rxq_release(dev, i);
+}
+
+/**
+ * Start traffic on Rx queues.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_rxq_start(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	unsigned int i;
+	int ret = 0;
+
+	/* Allocate/reuse/resize mempool for Multi-Packet RQ. */
+	if (mlx5_mprq_alloc_mp(dev)) {
+		/* Should not release Rx queues but return immediately. */
+		return -rte_errno;
+	}
+	for (i = 0; i != priv->rxqs_n; ++i) {
+		struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_get(dev, i);
+		struct rte_mempool *mp;
+
+		if (!rxq_ctrl)
+			continue;
+		/* Pre-register Rx mempool. */
+		mp = mlx5_rxq_mprq_enabled(&rxq_ctrl->rxq) ?
+		     rxq_ctrl->rxq.mprq_mp : rxq_ctrl->rxq.mp;
+		DRV_LOG(DEBUG,
+			"port %u Rx queue %u registering"
+			" mp %s having %u chunks",
+			dev->data->port_id, rxq_ctrl->idx,
+			mp->name, mp->nb_mem_chunks);
+		mlx5_mr_update_mp(dev, &rxq_ctrl->rxq.mr_ctrl, mp);
+		ret = rxq_alloc_elts(rxq_ctrl);
+		if (ret)
+			goto error;
+		rxq_ctrl->ibv = mlx5_rxq_ibv_new(dev, i);
+		if (!rxq_ctrl->ibv)
+			goto error;
+	}
+	return 0;
+error:
+	ret = rte_errno; /* Save rte_errno before cleanup. */
+	do {
+		mlx5_rxq_release(dev, i);
+	} while (i-- != 0);
+	rte_errno = ret; /* Restore rte_errno. */
+	return -rte_errno;
+}
+
+/**
+ * DPDK callback to start the device.
+ *
+ * Simulate device start by attaching all configured flows.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_dev_start(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	int ret;
+
+	DRV_LOG(DEBUG, "port %u starting device", dev->data->port_id);
+	ret = mlx5_txq_start(dev);
+	if (ret) {
+		DRV_LOG(ERR, "port %u Tx queue allocation failed: %s",
+			dev->data->port_id, strerror(rte_errno));
+		return -rte_errno;
+	}
+	ret = mlx5_rxq_start(dev);
+	if (ret) {
+		DRV_LOG(ERR, "port %u Rx queue allocation failed: %s",
+			dev->data->port_id, strerror(rte_errno));
+		mlx5_txq_stop(dev);
+		return -rte_errno;
+	}
+	dev->data->dev_started = 1;
+	ret = mlx5_rx_intr_vec_enable(dev);
+	if (ret) {
+		DRV_LOG(ERR, "port %u Rx interrupt vector creation failed",
+			dev->data->port_id);
+		goto error;
+	}
+	mlx5_xstats_init(dev);
+	ret = mlx5_traffic_enable(dev);
+	if (ret) {
+		DRV_LOG(DEBUG, "port %u failed to set defaults flows",
+			dev->data->port_id);
+		goto error;
+	}
+	ret = mlx5_flow_start(dev, &priv->flows);
+	if (ret) {
+		DRV_LOG(DEBUG, "port %u failed to set flows",
+			dev->data->port_id);
+		goto error;
+	}
+	dev->tx_pkt_burst = mlx5_select_tx_function(dev);
+	dev->rx_pkt_burst = mlx5_select_rx_function(dev);
+	mlx5_dev_interrupt_handler_install(dev);
+	return 0;
+error:
+	ret = rte_errno; /* Save rte_errno before cleanup. */
+	/* Rollback. */
+	dev->data->dev_started = 0;
+	mlx5_flow_stop(dev, &priv->flows);
+	mlx5_traffic_disable(dev);
+	mlx5_txq_stop(dev);
+	mlx5_rxq_stop(dev);
+	rte_errno = ret; /* Restore rte_errno. */
+	return -rte_errno;
+}
+
+/**
+ * DPDK callback to stop the device.
+ *
+ * Simulate device stop by detaching all configured flows.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ */
+void
+mlx5_dev_stop(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+
+	dev->data->dev_started = 0;
+	/* Prevent crashes when queues are still in use. */
+	dev->rx_pkt_burst = removed_rx_burst;
+	dev->tx_pkt_burst = removed_tx_burst;
+	rte_wmb();
+	usleep(1000 * priv->rxqs_n);
+	DRV_LOG(DEBUG, "port %u stopping device", dev->data->port_id);
+	mlx5_flow_stop(dev, &priv->flows);
+	mlx5_traffic_disable(dev);
+	mlx5_rx_intr_vec_disable(dev);
+	mlx5_dev_interrupt_handler_uninstall(dev);
+	mlx5_txq_stop(dev);
+	mlx5_rxq_stop(dev);
+}
+
+/**
+ * Enable traffic flows configured by control plane
+ *
+ * @param dev
+ *   Pointer to Ethernet device private data.
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_traffic_enable(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct rte_flow_item_eth bcast = {
+		.dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
+	};
+	struct rte_flow_item_eth ipv6_multi_spec = {
+		.dst.addr_bytes = "\x33\x33\x00\x00\x00\x00",
+	};
+	struct rte_flow_item_eth ipv6_multi_mask = {
+		.dst.addr_bytes = "\xff\xff\x00\x00\x00\x00",
+	};
+	struct rte_flow_item_eth unicast = {
+		.src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
+	};
+	struct rte_flow_item_eth unicast_mask = {
+		.dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
+	};
+	const unsigned int vlan_filter_n = priv->vlan_filter_n;
+	const struct ether_addr cmp = {
+		.addr_bytes = "\x00\x00\x00\x00\x00\x00",
+	};
+	unsigned int i;
+	unsigned int j;
+	int ret;
+
+	if (priv->isolated)
+		return 0;
+	if (dev->data->promiscuous) {
+		struct rte_flow_item_eth promisc = {
+			.dst.addr_bytes = "\x00\x00\x00\x00\x00\x00",
+			.src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
+			.type = 0,
+		};
+
+		ret = mlx5_ctrl_flow(dev, &promisc, &promisc);
+		if (ret)
+			goto error;
+	}
+	if (dev->data->all_multicast) {
+		struct rte_flow_item_eth multicast = {
+			.dst.addr_bytes = "\x01\x00\x00\x00\x00\x00",
+			.src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
+			.type = 0,
+		};
+
+		ret = mlx5_ctrl_flow(dev, &multicast, &multicast);
+		if (ret)
+			goto error;
+	} else {
+		/* Add broadcast/multicast flows. */
+		for (i = 0; i != vlan_filter_n; ++i) {
+			uint16_t vlan = priv->vlan_filter[i];
+
+			struct rte_flow_item_vlan vlan_spec = {
+				.tci = rte_cpu_to_be_16(vlan),
+			};
+			struct rte_flow_item_vlan vlan_mask =
+				rte_flow_item_vlan_mask;
+
+			ret = mlx5_ctrl_flow_vlan(dev, &bcast, &bcast,
+						  &vlan_spec, &vlan_mask);
+			if (ret)
+				goto error;
+			ret = mlx5_ctrl_flow_vlan(dev, &ipv6_multi_spec,
+						  &ipv6_multi_mask,
+						  &vlan_spec, &vlan_mask);
+			if (ret)
+				goto error;
+		}
+		if (!vlan_filter_n) {
+			ret = mlx5_ctrl_flow(dev, &bcast, &bcast);
+			if (ret)
+				goto error;
+			ret = mlx5_ctrl_flow(dev, &ipv6_multi_spec,
+					     &ipv6_multi_mask);
+			if (ret)
+				goto error;
+		}
+	}
+	/* Add MAC address flows. */
+	for (i = 0; i != MLX5_MAX_MAC_ADDRESSES; ++i) {
+		struct ether_addr *mac = &dev->data->mac_addrs[i];
+
+		if (!memcmp(mac, &cmp, sizeof(*mac)))
+			continue;
+		memcpy(&unicast.dst.addr_bytes,
+		       mac->addr_bytes,
+		       ETHER_ADDR_LEN);
+		for (j = 0; j != vlan_filter_n; ++j) {
+			uint16_t vlan = priv->vlan_filter[j];
+
+			struct rte_flow_item_vlan vlan_spec = {
+				.tci = rte_cpu_to_be_16(vlan),
+			};
+			struct rte_flow_item_vlan vlan_mask =
+				rte_flow_item_vlan_mask;
+
+			ret = mlx5_ctrl_flow_vlan(dev, &unicast,
+						  &unicast_mask,
+						  &vlan_spec,
+						  &vlan_mask);
+			if (ret)
+				goto error;
+		}
+		if (!vlan_filter_n) {
+			ret = mlx5_ctrl_flow(dev, &unicast, &unicast_mask);
+			if (ret)
+				goto error;
+		}
+	}
+	return 0;
+error:
+	ret = rte_errno; /* Save rte_errno before cleanup. */
+	mlx5_flow_list_flush(dev, &priv->ctrl_flows);
+	rte_errno = ret; /* Restore rte_errno. */
+	return -rte_errno;
+}
+
+
+/**
+ * Disable traffic flows configured by control plane
+ *
+ * @param dev
+ *   Pointer to Ethernet device private data.
+ */
+void
+mlx5_traffic_disable(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+
+	mlx5_flow_list_flush(dev, &priv->ctrl_flows);
+}
+
+/**
+ * Restart traffic flows configured by control plane
+ *
+ * @param dev
+ *   Pointer to Ethernet device private data.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_traffic_restart(struct rte_eth_dev *dev)
+{
+	if (dev->data->dev_started) {
+		mlx5_traffic_disable(dev);
+		return mlx5_traffic_enable(dev);
+	}
+	return 0;
+}
diff --git a/src/spdk/dpdk/drivers/net/mlx5/mlx5_txq.c b/src/spdk/dpdk/drivers/net/mlx5/mlx5_txq.c
new file mode 100644
index 00000000..f9bc4739
--- /dev/null
+++ b/src/spdk/dpdk/drivers/net/mlx5/mlx5_txq.c
@@ -0,0 +1,903 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2015 6WIND S.A.
+ * Copyright 2015 Mellanox Technologies, Ltd
+ */
+
+#include <stddef.h>
+#include <assert.h>
+#include <errno.h>
+#include <string.h>
+#include <stdint.h>
+#include <unistd.h>
+#include <sys/mman.h>
+
+/* Verbs header. */
+/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <infiniband/verbs.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+#include <rte_mbuf.h>
+#include <rte_malloc.h>
+#include <rte_ethdev_driver.h>
+#include <rte_common.h>
+
+#include "mlx5_utils.h"
+#include "mlx5_defs.h"
+#include "mlx5.h"
+#include "mlx5_rxtx.h"
+#include "mlx5_autoconf.h"
+#include "mlx5_glue.h"
+
+/**
+ * Allocate TX queue elements.
+ *
+ * @param txq_ctrl
+ *   Pointer to TX queue structure.
+ */
+void
+txq_alloc_elts(struct mlx5_txq_ctrl *txq_ctrl)
+{
+	const unsigned int elts_n = 1 << txq_ctrl->txq.elts_n;
+	unsigned int i;
+
+	for (i = 0; (i != elts_n); ++i)
+		(*txq_ctrl->txq.elts)[i] = NULL;
+	DRV_LOG(DEBUG, "port %u Tx queue %u allocated and configured %u WRs",
+		PORT_ID(txq_ctrl->priv), txq_ctrl->idx, elts_n);
+	txq_ctrl->txq.elts_head = 0;
+	txq_ctrl->txq.elts_tail = 0;
+	txq_ctrl->txq.elts_comp = 0;
+}
+
+/**
+ * Free TX queue elements.
+ *
+ * @param txq_ctrl
+ *   Pointer to TX queue structure.
+ */
+static void
+txq_free_elts(struct mlx5_txq_ctrl *txq_ctrl)
+{
+	const uint16_t elts_n = 1 << txq_ctrl->txq.elts_n;
+	const uint16_t elts_m = elts_n - 1;
+	uint16_t elts_head = txq_ctrl->txq.elts_head;
+	uint16_t elts_tail = txq_ctrl->txq.elts_tail;
+	struct rte_mbuf *(*elts)[elts_n] = txq_ctrl->txq.elts;
+
+	DRV_LOG(DEBUG, "port %u Tx queue %u freeing WRs",
+		PORT_ID(txq_ctrl->priv), txq_ctrl->idx);
+	txq_ctrl->txq.elts_head = 0;
+	txq_ctrl->txq.elts_tail = 0;
+	txq_ctrl->txq.elts_comp = 0;
+
+	while (elts_tail != elts_head) {
+		struct rte_mbuf *elt = (*elts)[elts_tail & elts_m];
+
+		assert(elt != NULL);
+		rte_pktmbuf_free_seg(elt);
+#ifndef NDEBUG
+		/* Poisoning. */
+		memset(&(*elts)[elts_tail & elts_m],
+		       0x77,
+		       sizeof((*elts)[elts_tail & elts_m]));
+#endif
+		++elts_tail;
+	}
+}
+
+/**
+ * Returns the per-port supported offloads.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   Supported Tx offloads.
+ */
+uint64_t
+mlx5_get_tx_port_offloads(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	uint64_t offloads = (DEV_TX_OFFLOAD_MULTI_SEGS |
+			     DEV_TX_OFFLOAD_VLAN_INSERT);
+	struct mlx5_dev_config *config = &priv->config;
+
+	if (config->hw_csum)
+		offloads |= (DEV_TX_OFFLOAD_IPV4_CKSUM |
+			     DEV_TX_OFFLOAD_UDP_CKSUM |
+			     DEV_TX_OFFLOAD_TCP_CKSUM);
+	if (config->tso)
+		offloads |= DEV_TX_OFFLOAD_TCP_TSO;
+	if (config->swp) {
+		if (config->hw_csum)
+			offloads |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM;
+		if (config->tso)
+			offloads |= (DEV_TX_OFFLOAD_IP_TNL_TSO |
+				     DEV_TX_OFFLOAD_UDP_TNL_TSO);
+	}
+
+	if (config->tunnel_en) {
+		if (config->hw_csum)
+			offloads |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM;
+		if (config->tso)
+			offloads |= (DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
+				     DEV_TX_OFFLOAD_GRE_TNL_TSO);
+	}
+	return offloads;
+}
+
+/**
+ * DPDK callback to configure a TX queue.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param idx
+ *   TX queue index.
+ * @param desc
+ *   Number of descriptors to configure in queue.
+ * @param socket
+ *   NUMA socket on which memory must be allocated.
+ * @param[in] conf
+ *   Thresholds parameters.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
+		    unsigned int socket, const struct rte_eth_txconf *conf)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_txq_data *txq = (*priv->txqs)[idx];
+	struct mlx5_txq_ctrl *txq_ctrl =
+		container_of(txq, struct mlx5_txq_ctrl, txq);
+
+	if (desc <= MLX5_TX_COMP_THRESH) {
+		DRV_LOG(WARNING,
+			"port %u number of descriptors requested for Tx queue"
+			" %u must be higher than MLX5_TX_COMP_THRESH, using %u"
+			" instead of %u",
+			dev->data->port_id, idx, MLX5_TX_COMP_THRESH + 1, desc);
+		desc = MLX5_TX_COMP_THRESH + 1;
+	}
+	if (!rte_is_power_of_2(desc)) {
+		desc = 1 << log2above(desc);
+		DRV_LOG(WARNING,
+			"port %u increased number of descriptors in Tx queue"
+			" %u to the next power of two (%d)",
+			dev->data->port_id, idx, desc);
+	}
+	DRV_LOG(DEBUG, "port %u configuring queue %u for %u descriptors",
+		dev->data->port_id, idx, desc);
+	if (idx >= priv->txqs_n) {
+		DRV_LOG(ERR, "port %u Tx queue index out of range (%u >= %u)",
+			dev->data->port_id, idx, priv->txqs_n);
+		rte_errno = EOVERFLOW;
+		return -rte_errno;
+	}
+	if (!mlx5_txq_releasable(dev, idx)) {
+		rte_errno = EBUSY;
+		DRV_LOG(ERR, "port %u unable to release queue index %u",
+			dev->data->port_id, idx);
+		return -rte_errno;
+	}
+	mlx5_txq_release(dev, idx);
+	txq_ctrl = mlx5_txq_new(dev, idx, desc, socket, conf);
+	if (!txq_ctrl) {
+		DRV_LOG(ERR, "port %u unable to allocate queue index %u",
+			dev->data->port_id, idx);
+		return -rte_errno;
+	}
+	DRV_LOG(DEBUG, "port %u adding Tx queue %u to list",
+		dev->data->port_id, idx);
+	(*priv->txqs)[idx] = &txq_ctrl->txq;
+	return 0;
+}
+
+/**
+ * DPDK callback to release a TX queue.
+ *
+ * @param dpdk_txq
+ *   Generic TX queue pointer.
+ */
+void
+mlx5_tx_queue_release(void *dpdk_txq)
+{
+	struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq;
+	struct mlx5_txq_ctrl *txq_ctrl;
+	struct priv *priv;
+	unsigned int i;
+
+	if (txq == NULL)
+		return;
+	txq_ctrl = container_of(txq, struct mlx5_txq_ctrl, txq);
+	priv = txq_ctrl->priv;
+	for (i = 0; (i != priv->txqs_n); ++i)
+		if ((*priv->txqs)[i] == txq) {
+			mlx5_txq_release(ETH_DEV(priv), i);
+			DRV_LOG(DEBUG, "port %u removing Tx queue %u from list",
+				PORT_ID(priv), txq_ctrl->idx);
+			break;
+		}
+}
+
+
+/**
+ * Mmap TX UAR(HW doorbell) pages into reserved UAR address space.
+ * Both primary and secondary process do mmap to make UAR address
+ * aligned.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ * @param fd
+ *   Verbs file descriptor to map UAR pages.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_tx_uar_remap(struct rte_eth_dev *dev, int fd)
+{
+	struct priv *priv = dev->data->dev_private;
+	unsigned int i, j;
+	uintptr_t pages[priv->txqs_n];
+	unsigned int pages_n = 0;
+	uintptr_t uar_va;
+	uintptr_t off;
+	void *addr;
+	void *ret;
+	struct mlx5_txq_data *txq;
+	struct mlx5_txq_ctrl *txq_ctrl;
+	int already_mapped;
+	size_t page_size = sysconf(_SC_PAGESIZE);
+#ifndef RTE_ARCH_64
+	unsigned int lock_idx;
+#endif
+
+	memset(pages, 0, priv->txqs_n * sizeof(uintptr_t));
+	/*
+	 * As rdma-core, UARs are mapped in size of OS page size.
+	 * Use aligned address to avoid duplicate mmap.
+	 * Ref to libmlx5 function: mlx5_init_context()
+	 */
+	for (i = 0; i != priv->txqs_n; ++i) {
+		if (!(*priv->txqs)[i])
+			continue;
+		txq = (*priv->txqs)[i];
+		txq_ctrl = container_of(txq, struct mlx5_txq_ctrl, txq);
+		assert(txq_ctrl->idx == (uint16_t)i);
+		/* UAR addr form verbs used to find dup and offset in page. */
+		uar_va = (uintptr_t)txq_ctrl->bf_reg_orig;
+		off = uar_va & (page_size - 1); /* offset in page. */
+		uar_va = RTE_ALIGN_FLOOR(uar_va, page_size); /* page addr. */
+		already_mapped = 0;
+		for (j = 0; j != pages_n; ++j) {
+			if (pages[j] == uar_va) {
+				already_mapped = 1;
+				break;
+			}
+		}
+		/* new address in reserved UAR address space. */
+		addr = RTE_PTR_ADD(priv->uar_base,
+				   uar_va & (uintptr_t)(MLX5_UAR_SIZE - 1));
+		if (!already_mapped) {
+			pages[pages_n++] = uar_va;
+			/* fixed mmap to specified address in reserved
+			 * address space.
+			 */
+			ret = mmap(addr, page_size,
+				   PROT_WRITE, MAP_FIXED | MAP_SHARED, fd,
+				   txq_ctrl->uar_mmap_offset);
+			if (ret != addr) {
+				/* fixed mmap have to return same address */
+				DRV_LOG(ERR,
+					"port %u call to mmap failed on UAR"
+					" for txq %u",
+					dev->data->port_id, txq_ctrl->idx);
+				rte_errno = ENXIO;
+				return -rte_errno;
+			}
+		}
+		if (rte_eal_process_type() == RTE_PROC_PRIMARY) /* save once */
+			txq_ctrl->txq.bf_reg = RTE_PTR_ADD((void *)addr, off);
+		else
+			assert(txq_ctrl->txq.bf_reg ==
+			       RTE_PTR_ADD((void *)addr, off));
+#ifndef RTE_ARCH_64
+		/* Assign a UAR lock according to UAR page number */
+		lock_idx = (txq_ctrl->uar_mmap_offset / page_size) &
+			   MLX5_UAR_PAGE_NUM_MASK;
+		txq->uar_lock = &priv->uar_lock[lock_idx];
+#endif
+	}
+	return 0;
+}
+
+/**
+ * Check if the burst function is using eMPW.
+ *
+ * @param tx_pkt_burst
+ *   Tx burst function pointer.
+ *
+ * @return
+ *   1 if the burst function is using eMPW, 0 otherwise.
+ */
+static int
+is_empw_burst_func(eth_tx_burst_t tx_pkt_burst)
+{
+	if (tx_pkt_burst == mlx5_tx_burst_raw_vec ||
+	    tx_pkt_burst == mlx5_tx_burst_vec ||
+	    tx_pkt_burst == mlx5_tx_burst_empw)
+		return 1;
+	return 0;
+}
+
+/**
+ * Create the Tx queue Verbs object.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param idx
+ *   Queue index in DPDK Rx queue array
+ *
+ * @return
+ *   The Verbs object initialised, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_txq_ibv *
+mlx5_txq_ibv_new(struct rte_eth_dev *dev, uint16_t idx)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_txq_data *txq_data = (*priv->txqs)[idx];
+	struct mlx5_txq_ctrl *txq_ctrl =
+		container_of(txq_data, struct mlx5_txq_ctrl, txq);
+	struct mlx5_txq_ibv tmpl;
+	struct mlx5_txq_ibv *txq_ibv;
+	union {
+		struct ibv_qp_init_attr_ex init;
+		struct ibv_cq_init_attr_ex cq;
+		struct ibv_qp_attr mod;
+		struct ibv_cq_ex cq_attr;
+	} attr;
+	unsigned int cqe_n;
+	struct mlx5dv_qp qp = { .comp_mask = MLX5DV_QP_MASK_UAR_MMAP_OFFSET };
+	struct mlx5dv_cq cq_info;
+	struct mlx5dv_obj obj;
+	const int desc = 1 << txq_data->elts_n;
+	eth_tx_burst_t tx_pkt_burst = mlx5_select_tx_function(dev);
+	int ret = 0;
+
+	assert(txq_data);
+	priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_TX_QUEUE;
+	priv->verbs_alloc_ctx.obj = txq_ctrl;
+	if (mlx5_getenv_int("MLX5_ENABLE_CQE_COMPRESSION")) {
+		DRV_LOG(ERR,
+			"port %u MLX5_ENABLE_CQE_COMPRESSION must never be set",
+			dev->data->port_id);
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	memset(&tmpl, 0, sizeof(struct mlx5_txq_ibv));
+	attr.cq = (struct ibv_cq_init_attr_ex){
+		.comp_mask = 0,
+	};
+	cqe_n = ((desc / MLX5_TX_COMP_THRESH) - 1) ?
+		((desc / MLX5_TX_COMP_THRESH) - 1) : 1;
+	if (is_empw_burst_func(tx_pkt_burst))
+		cqe_n += MLX5_TX_COMP_THRESH_INLINE_DIV;
+	tmpl.cq = mlx5_glue->create_cq(priv->ctx, cqe_n, NULL, NULL, 0);
+	if (tmpl.cq == NULL) {
+		DRV_LOG(ERR, "port %u Tx queue %u CQ creation failure",
+			dev->data->port_id, idx);
+		rte_errno = errno;
+		goto error;
+	}
+	attr.init = (struct ibv_qp_init_attr_ex){
+		/* CQ to be associated with the send queue. */
+		.send_cq = tmpl.cq,
+		/* CQ to be associated with the receive queue. */
+		.recv_cq = tmpl.cq,
+		.cap = {
+			/* Max number of outstanding WRs. */
+			.max_send_wr =
+				((priv->device_attr.orig_attr.max_qp_wr <
+				  desc) ?
+				 priv->device_attr.orig_attr.max_qp_wr :
+				 desc),
+			/*
+			 * Max number of scatter/gather elements in a WR,
+			 * must be 1 to prevent libmlx5 from trying to affect
+			 * too much memory. TX gather is not impacted by the
+			 * priv->device_attr.max_sge limit and will still work
+			 * properly.
+			 */
+			.max_send_sge = 1,
+		},
+		.qp_type = IBV_QPT_RAW_PACKET,
+		/*
+		 * Do *NOT* enable this, completions events are managed per
+		 * Tx burst.
+		 */
+		.sq_sig_all = 0,
+		.pd = priv->pd,
+		.comp_mask = IBV_QP_INIT_ATTR_PD,
+	};
+	if (txq_data->max_inline)
+		attr.init.cap.max_inline_data = txq_ctrl->max_inline_data;
+	if (txq_data->tso_en) {
+		attr.init.max_tso_header = txq_ctrl->max_tso_header;
+		attr.init.comp_mask |= IBV_QP_INIT_ATTR_MAX_TSO_HEADER;
+	}
+	tmpl.qp = mlx5_glue->create_qp_ex(priv->ctx, &attr.init);
+	if (tmpl.qp == NULL) {
+		DRV_LOG(ERR, "port %u Tx queue %u QP creation failure",
+			dev->data->port_id, idx);
+		rte_errno = errno;
+		goto error;
+	}
+	attr.mod = (struct ibv_qp_attr){
+		/* Move the QP to this state. */
+		.qp_state = IBV_QPS_INIT,
+		/* Primary port number. */
+		.port_num = 1,
+	};
+	ret = mlx5_glue->modify_qp(tmpl.qp, &attr.mod,
+				   (IBV_QP_STATE | IBV_QP_PORT));
+	if (ret) {
+		DRV_LOG(ERR,
+			"port %u Tx queue %u QP state to IBV_QPS_INIT failed",
+			dev->data->port_id, idx);
+		rte_errno = errno;
+		goto error;
+	}
+	attr.mod = (struct ibv_qp_attr){
+		.qp_state = IBV_QPS_RTR
+	};
+	ret = mlx5_glue->modify_qp(tmpl.qp, &attr.mod, IBV_QP_STATE);
+	if (ret) {
+		DRV_LOG(ERR,
+			"port %u Tx queue %u QP state to IBV_QPS_RTR failed",
+			dev->data->port_id, idx);
+		rte_errno = errno;
+		goto error;
+	}
+	attr.mod.qp_state = IBV_QPS_RTS;
+	ret = mlx5_glue->modify_qp(tmpl.qp, &attr.mod, IBV_QP_STATE);
+	if (ret) {
+		DRV_LOG(ERR,
+			"port %u Tx queue %u QP state to IBV_QPS_RTS failed",
+			dev->data->port_id, idx);
+		rte_errno = errno;
+		goto error;
+	}
+	txq_ibv = rte_calloc_socket(__func__, 1, sizeof(struct mlx5_txq_ibv), 0,
+				    txq_ctrl->socket);
+	if (!txq_ibv) {
+		DRV_LOG(ERR, "port %u Tx queue %u cannot allocate memory",
+			dev->data->port_id, idx);
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	obj.cq.in = tmpl.cq;
+	obj.cq.out = &cq_info;
+	obj.qp.in = tmpl.qp;
+	obj.qp.out = &qp;
+	ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_CQ | MLX5DV_OBJ_QP);
+	if (ret != 0) {
+		rte_errno = errno;
+		goto error;
+	}
+	if (cq_info.cqe_size != RTE_CACHE_LINE_SIZE) {
+		DRV_LOG(ERR,
+			"port %u wrong MLX5_CQE_SIZE environment variable"
+			" value: it should be set to %u",
+			dev->data->port_id, RTE_CACHE_LINE_SIZE);
+		rte_errno = EINVAL;
+		goto error;
+	}
+	txq_data->cqe_n = log2above(cq_info.cqe_cnt);
+	txq_data->qp_num_8s = tmpl.qp->qp_num << 8;
+	txq_data->wqes = qp.sq.buf;
+	txq_data->wqe_n = log2above(qp.sq.wqe_cnt);
+	txq_data->qp_db = &qp.dbrec[MLX5_SND_DBR];
+	txq_ctrl->bf_reg_orig = qp.bf.reg;
+	txq_data->cq_db = cq_info.dbrec;
+	txq_data->cqes =
+		(volatile struct mlx5_cqe (*)[])
+		(uintptr_t)cq_info.buf;
+	txq_data->cq_ci = 0;
+#ifndef NDEBUG
+	txq_data->cq_pi = 0;
+#endif
+	txq_data->wqe_ci = 0;
+	txq_data->wqe_pi = 0;
+	txq_ibv->qp = tmpl.qp;
+	txq_ibv->cq = tmpl.cq;
+	rte_atomic32_inc(&txq_ibv->refcnt);
+	if (qp.comp_mask & MLX5DV_QP_MASK_UAR_MMAP_OFFSET) {
+		txq_ctrl->uar_mmap_offset = qp.uar_mmap_offset;
+		DRV_LOG(DEBUG, "port %u: uar_mmap_offset 0x%lx",
+			dev->data->port_id, txq_ctrl->uar_mmap_offset);
+	} else {
+		DRV_LOG(ERR,
+			"port %u failed to retrieve UAR info, invalid"
+			" libmlx5.so",
+			dev->data->port_id);
+		rte_errno = EINVAL;
+		goto error;
+	}
+	LIST_INSERT_HEAD(&priv->txqsibv, txq_ibv, next);
+	txq_ibv->txq_ctrl = txq_ctrl;
+	priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE;
+	return txq_ibv;
+error:
+	ret = rte_errno; /* Save rte_errno before cleanup. */
+	if (tmpl.cq)
+		claim_zero(mlx5_glue->destroy_cq(tmpl.cq));
+	if (tmpl.qp)
+		claim_zero(mlx5_glue->destroy_qp(tmpl.qp));
+	priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE;
+	rte_errno = ret; /* Restore rte_errno. */
+	return NULL;
+}
+
+/**
+ * Get an Tx queue Verbs object.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param idx
+ *   Queue index in DPDK Rx queue array
+ *
+ * @return
+ *   The Verbs object if it exists.
+ */
+struct mlx5_txq_ibv *
+mlx5_txq_ibv_get(struct rte_eth_dev *dev, uint16_t idx)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_txq_ctrl *txq_ctrl;
+
+	if (idx >= priv->txqs_n)
+		return NULL;
+	if (!(*priv->txqs)[idx])
+		return NULL;
+	txq_ctrl = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq);
+	if (txq_ctrl->ibv)
+		rte_atomic32_inc(&txq_ctrl->ibv->refcnt);
+	return txq_ctrl->ibv;
+}
+
+/**
+ * Release an Tx verbs queue object.
+ *
+ * @param txq_ibv
+ *   Verbs Tx queue object.
+ *
+ * @return
+ *   1 while a reference on it exists, 0 when freed.
+ */
+int
+mlx5_txq_ibv_release(struct mlx5_txq_ibv *txq_ibv)
+{
+	assert(txq_ibv);
+	if (rte_atomic32_dec_and_test(&txq_ibv->refcnt)) {
+		claim_zero(mlx5_glue->destroy_qp(txq_ibv->qp));
+		claim_zero(mlx5_glue->destroy_cq(txq_ibv->cq));
+		LIST_REMOVE(txq_ibv, next);
+		rte_free(txq_ibv);
+		return 0;
+	}
+	return 1;
+}
+
+/**
+ * Return true if a single reference exists on the object.
+ *
+ * @param txq_ibv
+ *   Verbs Tx queue object.
+ */
+int
+mlx5_txq_ibv_releasable(struct mlx5_txq_ibv *txq_ibv)
+{
+	assert(txq_ibv);
+	return (rte_atomic32_read(&txq_ibv->refcnt) == 1);
+}
+
+/**
+ * Verify the Verbs Tx queue list is empty
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   The number of object not released.
+ */
+int
+mlx5_txq_ibv_verify(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	int ret = 0;
+	struct mlx5_txq_ibv *txq_ibv;
+
+	LIST_FOREACH(txq_ibv, &priv->txqsibv, next) {
+		DRV_LOG(DEBUG, "port %u Verbs Tx queue %u still referenced",
+			dev->data->port_id, txq_ibv->txq_ctrl->idx);
+		++ret;
+	}
+	return ret;
+}
+
+/**
+ * Set Tx queue parameters from device configuration.
+ *
+ * @param txq_ctrl
+ *   Pointer to Tx queue control structure.
+ */
+static void
+txq_set_params(struct mlx5_txq_ctrl *txq_ctrl)
+{
+	struct priv *priv = txq_ctrl->priv;
+	struct mlx5_dev_config *config = &priv->config;
+	const unsigned int max_tso_inline =
+		((MLX5_MAX_TSO_HEADER + (RTE_CACHE_LINE_SIZE - 1)) /
+		 RTE_CACHE_LINE_SIZE);
+	unsigned int txq_inline;
+	unsigned int txqs_inline;
+	unsigned int inline_max_packet_sz;
+	eth_tx_burst_t tx_pkt_burst =
+		mlx5_select_tx_function(ETH_DEV(priv));
+	int is_empw_func = is_empw_burst_func(tx_pkt_burst);
+	int tso = !!(txq_ctrl->txq.offloads & (DEV_TX_OFFLOAD_TCP_TSO |
+					       DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
+					       DEV_TX_OFFLOAD_GRE_TNL_TSO |
+					       DEV_TX_OFFLOAD_IP_TNL_TSO |
+					       DEV_TX_OFFLOAD_UDP_TNL_TSO));
+
+	txq_inline = (config->txq_inline == MLX5_ARG_UNSET) ?
+		0 : config->txq_inline;
+	txqs_inline = (config->txqs_inline == MLX5_ARG_UNSET) ?
+		0 : config->txqs_inline;
+	inline_max_packet_sz =
+		(config->inline_max_packet_sz == MLX5_ARG_UNSET) ?
+		0 : config->inline_max_packet_sz;
+	if (is_empw_func) {
+		if (config->txq_inline == MLX5_ARG_UNSET)
+			txq_inline = MLX5_WQE_SIZE_MAX - MLX5_WQE_SIZE;
+		if (config->txqs_inline == MLX5_ARG_UNSET)
+			txqs_inline = MLX5_EMPW_MIN_TXQS;
+		if (config->inline_max_packet_sz == MLX5_ARG_UNSET)
+			inline_max_packet_sz = MLX5_EMPW_MAX_INLINE_LEN;
+		txq_ctrl->txq.mpw_hdr_dseg = config->mpw_hdr_dseg;
+		txq_ctrl->txq.inline_max_packet_sz = inline_max_packet_sz;
+	}
+	if (txq_inline && priv->txqs_n >= txqs_inline) {
+		unsigned int ds_cnt;
+
+		txq_ctrl->txq.max_inline =
+			((txq_inline + (RTE_CACHE_LINE_SIZE - 1)) /
+			 RTE_CACHE_LINE_SIZE);
+		if (is_empw_func) {
+			/* To minimize the size of data set, avoid requesting
+			 * too large WQ.
+			 */
+			txq_ctrl->max_inline_data =
+				((RTE_MIN(txq_inline,
+					  inline_max_packet_sz) +
+				  (RTE_CACHE_LINE_SIZE - 1)) /
+				 RTE_CACHE_LINE_SIZE) * RTE_CACHE_LINE_SIZE;
+		} else {
+			txq_ctrl->max_inline_data =
+				txq_ctrl->txq.max_inline * RTE_CACHE_LINE_SIZE;
+		}
+		/*
+		 * Check if the inline size is too large in a way which
+		 * can make the WQE DS to overflow.
+		 * Considering in calculation:
+		 *      WQE CTRL (1 DS)
+		 *      WQE ETH  (1 DS)
+		 *      Inline part (N DS)
+		 */
+		ds_cnt = 2 + (txq_ctrl->txq.max_inline / MLX5_WQE_DWORD_SIZE);
+		if (ds_cnt > MLX5_DSEG_MAX) {
+			unsigned int max_inline = (MLX5_DSEG_MAX - 2) *
+						  MLX5_WQE_DWORD_SIZE;
+
+			max_inline = max_inline - (max_inline %
+						   RTE_CACHE_LINE_SIZE);
+			DRV_LOG(WARNING,
+				"port %u txq inline is too large (%d) setting"
+				" it to the maximum possible: %d\n",
+				PORT_ID(priv), txq_inline, max_inline);
+			txq_ctrl->txq.max_inline = max_inline /
+						   RTE_CACHE_LINE_SIZE;
+		}
+	}
+	if (tso) {
+		txq_ctrl->max_tso_header = max_tso_inline * RTE_CACHE_LINE_SIZE;
+		txq_ctrl->txq.max_inline = RTE_MAX(txq_ctrl->txq.max_inline,
+						   max_tso_inline);
+		txq_ctrl->txq.tso_en = 1;
+	}
+	txq_ctrl->txq.tunnel_en = config->tunnel_en | config->swp;
+	txq_ctrl->txq.swp_en = ((DEV_TX_OFFLOAD_IP_TNL_TSO |
+				 DEV_TX_OFFLOAD_UDP_TNL_TSO |
+				 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM) &
+				txq_ctrl->txq.offloads) && config->swp;
+}
+
+/**
+ * Create a DPDK Tx queue.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param idx
+ *   TX queue index.
+ * @param desc
+ *   Number of descriptors to configure in queue.
+ * @param socket
+ *   NUMA socket on which memory must be allocated.
+ * @param[in] conf
+ *  Thresholds parameters.
+ *
+ * @return
+ *   A DPDK queue object on success, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_txq_ctrl *
+mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
+	     unsigned int socket, const struct rte_eth_txconf *conf)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_txq_ctrl *tmpl;
+
+	tmpl = rte_calloc_socket("TXQ", 1,
+				 sizeof(*tmpl) +
+				 desc * sizeof(struct rte_mbuf *),
+				 0, socket);
+	if (!tmpl) {
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+	if (mlx5_mr_btree_init(&tmpl->txq.mr_ctrl.cache_bh,
+			       MLX5_MR_BTREE_CACHE_N, socket)) {
+		/* rte_errno is already set. */
+		goto error;
+	}
+	/* Save pointer of global generation number to check memory event. */
+	tmpl->txq.mr_ctrl.dev_gen_ptr = &priv->mr.dev_gen;
+	assert(desc > MLX5_TX_COMP_THRESH);
+	tmpl->txq.offloads = conf->offloads |
+			     dev->data->dev_conf.txmode.offloads;
+	tmpl->priv = priv;
+	tmpl->socket = socket;
+	tmpl->txq.elts_n = log2above(desc);
+	tmpl->idx = idx;
+	txq_set_params(tmpl);
+	DRV_LOG(DEBUG, "port %u priv->device_attr.max_qp_wr is %d",
+		dev->data->port_id, priv->device_attr.orig_attr.max_qp_wr);
+	DRV_LOG(DEBUG, "port %u priv->device_attr.max_sge is %d",
+		dev->data->port_id, priv->device_attr.orig_attr.max_sge);
+	tmpl->txq.elts =
+		(struct rte_mbuf *(*)[1 << tmpl->txq.elts_n])(tmpl + 1);
+	tmpl->txq.stats.idx = idx;
+	rte_atomic32_inc(&tmpl->refcnt);
+	LIST_INSERT_HEAD(&priv->txqsctrl, tmpl, next);
+	return tmpl;
+error:
+	rte_free(tmpl);
+	return NULL;
+}
+
+/**
+ * Get a Tx queue.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param idx
+ *   TX queue index.
+ *
+ * @return
+ *   A pointer to the queue if it exists.
+ */
+struct mlx5_txq_ctrl *
+mlx5_txq_get(struct rte_eth_dev *dev, uint16_t idx)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_txq_ctrl *ctrl = NULL;
+
+	if ((*priv->txqs)[idx]) {
+		ctrl = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl,
+				    txq);
+		mlx5_txq_ibv_get(dev, idx);
+		rte_atomic32_inc(&ctrl->refcnt);
+	}
+	return ctrl;
+}
+
+/**
+ * Release a Tx queue.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param idx
+ *   TX queue index.
+ *
+ * @return
+ *   1 while a reference on it exists, 0 when freed.
+ */
+int
+mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_txq_ctrl *txq;
+	size_t page_size = sysconf(_SC_PAGESIZE);
+
+	if (!(*priv->txqs)[idx])
+		return 0;
+	txq = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq);
+	if (txq->ibv && !mlx5_txq_ibv_release(txq->ibv))
+		txq->ibv = NULL;
+	if (priv->uar_base)
+		munmap((void *)RTE_ALIGN_FLOOR((uintptr_t)txq->txq.bf_reg,
+		       page_size), page_size);
+	if (rte_atomic32_dec_and_test(&txq->refcnt)) {
+		txq_free_elts(txq);
+		mlx5_mr_btree_free(&txq->txq.mr_ctrl.cache_bh);
+		LIST_REMOVE(txq, next);
+		rte_free(txq);
+		(*priv->txqs)[idx] = NULL;
+		return 0;
+	}
+	return 1;
+}
+
+/**
+ * Verify if the queue can be released.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param idx
+ *   TX queue index.
+ *
+ * @return
+ *   1 if the queue can be released.
+ */
+int
+mlx5_txq_releasable(struct rte_eth_dev *dev, uint16_t idx)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_txq_ctrl *txq;
+
+	if (!(*priv->txqs)[idx])
+		return -1;
+	txq = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq);
+	return (rte_atomic32_read(&txq->refcnt) == 1);
+}
+
+/**
+ * Verify the Tx Queue list is empty
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   The number of object not released.
+ */
+int
+mlx5_txq_verify(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_txq_ctrl *txq;
+	int ret = 0;
+
+	LIST_FOREACH(txq, &priv->txqsctrl, next) {
+		DRV_LOG(DEBUG, "port %u Tx queue %u still referenced",
+			dev->data->port_id, txq->idx);
+		++ret;
+	}
+	return ret;
+}
diff --git a/src/spdk/dpdk/drivers/net/mlx5/mlx5_utils.h b/src/spdk/dpdk/drivers/net/mlx5/mlx5_utils.h
new file mode 100644
index 00000000..886f60e6
--- /dev/null
+++ b/src/spdk/dpdk/drivers/net/mlx5/mlx5_utils.h
@@ -0,0 +1,163 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2015 6WIND S.A.
+ * Copyright 2015 Mellanox Technologies, Ltd
+ */
+
+#ifndef RTE_PMD_MLX5_UTILS_H_
+#define RTE_PMD_MLX5_UTILS_H_
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <limits.h>
+#include <assert.h>
+#include <errno.h>
+
+#include "mlx5_defs.h"
+
+/* Bit-field manipulation. */
+#define BITFIELD_DECLARE(bf, type, size) \
+	type bf[(((size_t)(size) / (sizeof(type) * CHAR_BIT)) + \
+		 !!((size_t)(size) % (sizeof(type) * CHAR_BIT)))]
+#define BITFIELD_DEFINE(bf, type, size) \
+	BITFIELD_DECLARE((bf), type, (size)) = { 0 }
+#define BITFIELD_SET(bf, b) \
+	(assert((size_t)(b) < (sizeof(bf) * CHAR_BIT)), \
+	 (void)((bf)[((b) / (sizeof((bf)[0]) * CHAR_BIT))] |= \
+		((size_t)1 << ((b) % (sizeof((bf)[0]) * CHAR_BIT)))))
+#define BITFIELD_RESET(bf, b) \
+	(assert((size_t)(b) < (sizeof(bf) * CHAR_BIT)), \
+	 (void)((bf)[((b) / (sizeof((bf)[0]) * CHAR_BIT))] &= \
+		~((size_t)1 << ((b) % (sizeof((bf)[0]) * CHAR_BIT)))))
+#define BITFIELD_ISSET(bf, b) \
+	(assert((size_t)(b) < (sizeof(bf) * CHAR_BIT)), \
+	 !!(((bf)[((b) / (sizeof((bf)[0]) * CHAR_BIT))] & \
+	     ((size_t)1 << ((b) % (sizeof((bf)[0]) * CHAR_BIT))))))
+
+/* Convert a bit number to the corresponding 64-bit mask */
+#define MLX5_BITSHIFT(v) (UINT64_C(1) << (v))
+
+/* Save and restore errno around argument evaluation. */
+#define ERRNO_SAFE(x) ((errno = (int []){ errno, ((x), 0) }[0]))
+
+/*
+ * Helper macros to work around __VA_ARGS__ limitations in a C99 compliant
+ * manner.
+ */
+#define PMD_DRV_LOG_STRIP(a, b) a
+#define PMD_DRV_LOG_OPAREN (
+#define PMD_DRV_LOG_CPAREN )
+#define PMD_DRV_LOG_COMMA ,
+
+/* Return the file name part of a path. */
+static inline const char *
+pmd_drv_log_basename(const char *s)
+{
+	const char *n = s;
+
+	while (*n)
+		if (*(n++) == '/')
+			s = n;
+	return s;
+}
+
+extern int mlx5_logtype;
+
+#define PMD_DRV_LOG___(level, ...) \
+	rte_log(RTE_LOG_ ## level, \
+		mlx5_logtype, \
+		RTE_FMT(MLX5_DRIVER_NAME ": " \
+			RTE_FMT_HEAD(__VA_ARGS__,), \
+		RTE_FMT_TAIL(__VA_ARGS__,)))
+
+/*
+ * When debugging is enabled (NDEBUG not defined), file, line and function
+ * information replace the driver name (MLX5_DRIVER_NAME) in log messages.
+ */
+#ifndef NDEBUG
+
+#define PMD_DRV_LOG__(level, ...) \
+	PMD_DRV_LOG___(level, "%s:%u: %s(): " __VA_ARGS__)
+#define PMD_DRV_LOG_(level, s, ...) \
+	PMD_DRV_LOG__(level, \
+		s "\n" PMD_DRV_LOG_COMMA \
+		pmd_drv_log_basename(__FILE__) PMD_DRV_LOG_COMMA \
+		__LINE__ PMD_DRV_LOG_COMMA \
+		__func__, \
+		__VA_ARGS__)
+
+#else /* NDEBUG */
+#define PMD_DRV_LOG__(level, ...) \
+	PMD_DRV_LOG___(level, __VA_ARGS__)
+#define PMD_DRV_LOG_(level, s, ...) \
+	PMD_DRV_LOG__(level, s "\n", __VA_ARGS__)
+
+#endif /* NDEBUG */
+
+/* Generic printf()-like logging macro with automatic line feed. */
+#define DRV_LOG(level, ...) \
+	PMD_DRV_LOG_(level, \
+		__VA_ARGS__ PMD_DRV_LOG_STRIP PMD_DRV_LOG_OPAREN, \
+		PMD_DRV_LOG_CPAREN)
+
+/* claim_zero() does not perform any check when debugging is disabled. */
+#ifndef NDEBUG
+
+#define DEBUG(...) DRV_LOG(DEBUG, __VA_ARGS__)
+#define claim_zero(...) assert((__VA_ARGS__) == 0)
+#define claim_nonzero(...) assert((__VA_ARGS__) != 0)
+
+#else /* NDEBUG */
+
+#define DEBUG(...) (void)0
+#define claim_zero(...) (__VA_ARGS__)
+#define claim_nonzero(...) (__VA_ARGS__)
+
+#endif /* NDEBUG */
+
+#define INFO(...) DRV_LOG(INFO, __VA_ARGS__)
+#define WARN(...) DRV_LOG(WARNING, __VA_ARGS__)
+#define ERROR(...) DRV_LOG(ERR, __VA_ARGS__)
+
+/* Convenience macros for accessing mbuf fields. */
+#define NEXT(m) ((m)->next)
+#define DATA_LEN(m) ((m)->data_len)
+#define PKT_LEN(m) ((m)->pkt_len)
+#define DATA_OFF(m) ((m)->data_off)
+#define SET_DATA_OFF(m, o) ((m)->data_off = (o))
+#define NB_SEGS(m) ((m)->nb_segs)
+#define PORT(m) ((m)->port)
+
+/* Transpose flags. Useful to convert IBV to DPDK flags. */
+#define TRANSPOSE(val, from, to) \
+	(((from) >= (to)) ? \
+	 (((val) & (from)) / ((from) / (to))) : \
+	 (((val) & (from)) * ((to) / (from))))
+
+/* Allocate a buffer on the stack and fill it with a printf format string. */
+#define MKSTR(name, ...) \
+	char name[snprintf(NULL, 0, __VA_ARGS__) + 1]; \
+	\
+	snprintf(name, sizeof(name), __VA_ARGS__)
+
+/**
+ * Return nearest power of two above input value.
+ *
+ * @param v
+ *   Input value.
+ *
+ * @return
+ *   Nearest power of two above input value.
+ */
+static inline unsigned int
+log2above(unsigned int v)
+{
+	unsigned int l;
+	unsigned int r;
+
+	for (l = 0, r = 0; (v >> 1); ++l, v >>= 1)
+		r |= (v & 1);
+	return l + r;
+}
+
+#endif /* RTE_PMD_MLX5_UTILS_H_ */
diff --git a/src/spdk/dpdk/drivers/net/mlx5/mlx5_vlan.c b/src/spdk/dpdk/drivers/net/mlx5/mlx5_vlan.c
new file mode 100644
index 00000000..c91d08be
--- /dev/null
+++ b/src/spdk/dpdk/drivers/net/mlx5/mlx5_vlan.c
@@ -0,0 +1,180 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2015 6WIND S.A.
+ * Copyright 2015 Mellanox Technologies, Ltd
+ */
+
+#include <stddef.h>
+#include <errno.h>
+#include <assert.h>
+#include <stdint.h>
+
+/*
+ * Not needed by this file; included to work around the lack of off_t
+ * definition for mlx5dv.h with unpatched rdma-core versions.
+ */
+#include <sys/types.h>
+
+/* Verbs headers do not support -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <infiniband/mlx5dv.h>
+#include <infiniband/verbs.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+#include <rte_ethdev_driver.h>
+#include <rte_common.h>
+
+#include "mlx5_utils.h"
+#include "mlx5.h"
+#include "mlx5_autoconf.h"
+#include "mlx5_glue.h"
+
+/**
+ * DPDK callback to configure a VLAN filter.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param vlan_id
+ *   VLAN ID to filter.
+ * @param on
+ *   Toggle filter.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
+{
+	struct priv *priv = dev->data->dev_private;
+	unsigned int i;
+
+	DRV_LOG(DEBUG, "port %u %s VLAN filter ID %" PRIu16,
+		dev->data->port_id, (on ? "enable" : "disable"), vlan_id);
+	assert(priv->vlan_filter_n <= RTE_DIM(priv->vlan_filter));
+	for (i = 0; (i != priv->vlan_filter_n); ++i)
+		if (priv->vlan_filter[i] == vlan_id)
+			break;
+	/* Check if there's room for another VLAN filter. */
+	if (i == RTE_DIM(priv->vlan_filter)) {
+		rte_errno = ENOMEM;
+		return -rte_errno;
+	}
+	if (i < priv->vlan_filter_n) {
+		assert(priv->vlan_filter_n != 0);
+		/* Enabling an existing VLAN filter has no effect. */
+		if (on)
+			goto out;
+		/* Remove VLAN filter from list. */
+		--priv->vlan_filter_n;
+		memmove(&priv->vlan_filter[i],
+			&priv->vlan_filter[i + 1],
+			sizeof(priv->vlan_filter[i]) *
+			(priv->vlan_filter_n - i));
+		priv->vlan_filter[priv->vlan_filter_n] = 0;
+	} else {
+		assert(i == priv->vlan_filter_n);
+		/* Disabling an unknown VLAN filter has no effect. */
+		if (!on)
+			goto out;
+		/* Add new VLAN filter. */
+		priv->vlan_filter[priv->vlan_filter_n] = vlan_id;
+		++priv->vlan_filter_n;
+	}
+out:
+	if (dev->data->dev_started)
+		return mlx5_traffic_restart(dev);
+	return 0;
+}
+
+/**
+ * Callback to set/reset VLAN stripping for a specific queue.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param queue
+ *   RX queue index.
+ * @param on
+ *   Enable/disable VLAN stripping.
+ */
+void
+mlx5_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_rxq_data *rxq = (*priv->rxqs)[queue];
+	struct mlx5_rxq_ctrl *rxq_ctrl =
+		container_of(rxq, struct mlx5_rxq_ctrl, rxq);
+	struct ibv_wq_attr mod;
+	uint16_t vlan_offloads =
+		(on ? IBV_WQ_FLAGS_CVLAN_STRIPPING : 0) |
+		0;
+	int ret;
+
+	/* Validate hw support */
+	if (!priv->config.hw_vlan_strip) {
+		DRV_LOG(ERR, "port %u VLAN stripping is not supported",
+			dev->data->port_id);
+		return;
+	}
+	/* Validate queue number */
+	if (queue >= priv->rxqs_n) {
+		DRV_LOG(ERR, "port %u VLAN stripping, invalid queue number %d",
+			dev->data->port_id, queue);
+		return;
+	}
+	DRV_LOG(DEBUG, "port %u set VLAN offloads 0x%x for port %uqueue %d",
+		dev->data->port_id, vlan_offloads, rxq->port_id, queue);
+	if (!rxq_ctrl->ibv) {
+		/* Update related bits in RX queue. */
+		rxq->vlan_strip = !!on;
+		return;
+	}
+	mod = (struct ibv_wq_attr){
+		.attr_mask = IBV_WQ_ATTR_FLAGS,
+		.flags_mask = IBV_WQ_FLAGS_CVLAN_STRIPPING,
+		.flags = vlan_offloads,
+	};
+	ret = mlx5_glue->modify_wq(rxq_ctrl->ibv->wq, &mod);
+	if (ret) {
+		DRV_LOG(ERR, "port %u failed to modified stripping mode: %s",
+			dev->data->port_id, strerror(rte_errno));
+		return;
+	}
+	/* Update related bits in RX queue. */
+	rxq->vlan_strip = !!on;
+}
+
+/**
+ * Callback to set/reset VLAN offloads for a port.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param mask
+ *   VLAN offload bit mask.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_vlan_offload_set(struct rte_eth_dev *dev, int mask)
+{
+	struct priv *priv = dev->data->dev_private;
+	unsigned int i;
+
+	if (mask & ETH_VLAN_STRIP_MASK) {
+		int hw_vlan_strip = !!(dev->data->dev_conf.rxmode.offloads &
+				       DEV_RX_OFFLOAD_VLAN_STRIP);
+
+		if (!priv->config.hw_vlan_strip) {
+			DRV_LOG(ERR, "port %u VLAN stripping is not supported",
+				dev->data->port_id);
+			return 0;
+		}
+		/* Run on every RX queue and set/reset VLAN stripping. */
+		for (i = 0; (i != priv->rxqs_n); i++)
+			mlx5_vlan_strip_queue_set(dev, i, hw_vlan_strip);
+	}
+	return 0;
+}
diff --git a/src/spdk/dpdk/drivers/net/mlx5/rte_pmd_mlx5_version.map b/src/spdk/dpdk/drivers/net/mlx5/rte_pmd_mlx5_version.map
new file mode 100644
index 00000000..ad607bbe
--- /dev/null
+++ b/src/spdk/dpdk/drivers/net/mlx5/rte_pmd_mlx5_version.map
@@ -0,0 +1,3 @@
+DPDK_2.2 {
+	local: *;
+};