13 files changed, 912 insertions, 0 deletions

diff --git a/src/spdk/dpdk/lib/librte_gso/Makefile b/src/spdk/dpdk/lib/librte_gso/Makefile
new file mode 100644
index 00000000..1fac53a8
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_gso/Makefile
@@ -0,0 +1,27 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2017 Intel Corporation
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+# library name
+LIB = librte_gso.a
+
+CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -O3
+LDLIBS += -lrte_eal -lrte_mbuf -lrte_ethdev -lrte_net
+LDLIBS += -lrte_mempool
+
+EXPORT_MAP := rte_gso_version.map
+
+LIBABIVER := 1
+
+#source files
+SRCS-$(CONFIG_RTE_LIBRTE_GSO) += rte_gso.c
+SRCS-$(CONFIG_RTE_LIBRTE_GSO) += gso_common.c
+SRCS-$(CONFIG_RTE_LIBRTE_GSO) += gso_tcp4.c
+SRCS-$(CONFIG_RTE_LIBRTE_GSO) += gso_tunnel_tcp4.c
+SRCS-$(CONFIG_RTE_LIBRTE_GSO) += gso_udp4.c
+
+# install this header file
+SYMLINK-$(CONFIG_RTE_LIBRTE_GSO)-include += rte_gso.h
+
+include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/src/spdk/dpdk/lib/librte_gso/gso_common.c b/src/spdk/dpdk/lib/librte_gso/gso_common.c
new file mode 100644
index 00000000..0fad1132
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_gso/gso_common.c
@@ -0,0 +1,124 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017 Intel Corporation
+ */
+
+#include <stdbool.h>
+#include <errno.h>
+
+#include <rte_memcpy.h>
+#include <rte_mempool.h>
+
+#include "gso_common.h"
+
+static inline void
+hdr_segment_init(struct rte_mbuf *hdr_segment, struct rte_mbuf *pkt,
+		uint16_t pkt_hdr_offset)
+{
+	/* Copy MBUF metadata */
+	hdr_segment->nb_segs = 1;
+	hdr_segment->port = pkt->port;
+	hdr_segment->ol_flags = pkt->ol_flags;
+	hdr_segment->packet_type = pkt->packet_type;
+	hdr_segment->pkt_len = pkt_hdr_offset;
+	hdr_segment->data_len = pkt_hdr_offset;
+	hdr_segment->tx_offload = pkt->tx_offload;
+
+	/* Copy the packet header */
+	rte_memcpy(rte_pktmbuf_mtod(hdr_segment, char *),
+			rte_pktmbuf_mtod(pkt, char *),
+			pkt_hdr_offset);
+}
+
+static inline void
+free_gso_segment(struct rte_mbuf **pkts, uint16_t nb_pkts)
+{
+	uint16_t i;
+
+	for (i = 0; i < nb_pkts; i++)
+		rte_pktmbuf_free(pkts[i]);
+}
+
+int
+gso_do_segment(struct rte_mbuf *pkt,
+		uint16_t pkt_hdr_offset,
+		uint16_t pyld_unit_size,
+		struct rte_mempool *direct_pool,
+		struct rte_mempool *indirect_pool,
+		struct rte_mbuf **pkts_out,
+		uint16_t nb_pkts_out)
+{
+	struct rte_mbuf *pkt_in;
+	struct rte_mbuf *hdr_segment, *pyld_segment, *prev_segment;
+	uint16_t pkt_in_data_pos, segment_bytes_remaining;
+	uint16_t pyld_len, nb_segs;
+	bool more_in_pkt, more_out_segs;
+
+	pkt_in = pkt;
+	nb_segs = 0;
+	more_in_pkt = 1;
+	pkt_in_data_pos = pkt_hdr_offset;
+
+	while (more_in_pkt) {
+		if (unlikely(nb_segs >= nb_pkts_out)) {
+			free_gso_segment(pkts_out, nb_segs);
+			return -EINVAL;
+		}
+
+		/* Allocate a direct MBUF */
+		hdr_segment = rte_pktmbuf_alloc(direct_pool);
+		if (unlikely(hdr_segment == NULL)) {
+			free_gso_segment(pkts_out, nb_segs);
+			return -ENOMEM;
+		}
+		/* Fill the packet header */
+		hdr_segment_init(hdr_segment, pkt, pkt_hdr_offset);
+
+		prev_segment = hdr_segment;
+		segment_bytes_remaining = pyld_unit_size;
+		more_out_segs = 1;
+
+		while (more_out_segs && more_in_pkt) {
+			/* Allocate an indirect MBUF */
+			pyld_segment = rte_pktmbuf_alloc(indirect_pool);
+			if (unlikely(pyld_segment == NULL)) {
+				rte_pktmbuf_free(hdr_segment);
+				free_gso_segment(pkts_out, nb_segs);
+				return -ENOMEM;
+			}
+			/* Attach to current MBUF segment of pkt */
+			rte_pktmbuf_attach(pyld_segment, pkt_in);
+
+			prev_segment->next = pyld_segment;
+			prev_segment = pyld_segment;
+
+			pyld_len = segment_bytes_remaining;
+			if (pyld_len + pkt_in_data_pos > pkt_in->data_len)
+				pyld_len = pkt_in->data_len - pkt_in_data_pos;
+
+			pyld_segment->data_off = pkt_in_data_pos +
+				pkt_in->data_off;
+			pyld_segment->data_len = pyld_len;
+
+			/* Update header segment */
+			hdr_segment->pkt_len += pyld_len;
+			hdr_segment->nb_segs++;
+
+			pkt_in_data_pos += pyld_len;
+			segment_bytes_remaining -= pyld_len;
+
+			/* Finish processing a MBUF segment of pkt */
+			if (pkt_in_data_pos == pkt_in->data_len) {
+				pkt_in = pkt_in->next;
+				pkt_in_data_pos = 0;
+				if (pkt_in == NULL)
+					more_in_pkt = 0;
+			}
+
+			/* Finish generating a GSO segment */
+			if (segment_bytes_remaining == 0)
+				more_out_segs = 0;
+		}
+		pkts_out[nb_segs++] = hdr_segment;
+	}
+	return nb_segs;
+}
diff --git a/src/spdk/dpdk/lib/librte_gso/gso_common.h b/src/spdk/dpdk/lib/librte_gso/gso_common.h
new file mode 100644
index 00000000..6cd764ff
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_gso/gso_common.h
@@ -0,0 +1,145 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017 Intel Corporation
+ */
+
+#ifndef _GSO_COMMON_H_
+#define _GSO_COMMON_H_
+
+#include <stdint.h>
+
+#include <rte_mbuf.h>
+#include <rte_ip.h>
+#include <rte_tcp.h>
+#include <rte_udp.h>
+
+#define IS_FRAGMENTED(frag_off) (((frag_off) & IPV4_HDR_OFFSET_MASK) != 0 \
+		|| ((frag_off) & IPV4_HDR_MF_FLAG) == IPV4_HDR_MF_FLAG)
+
+#define TCP_HDR_PSH_MASK ((uint8_t)0x08)
+#define TCP_HDR_FIN_MASK ((uint8_t)0x01)
+
+#define IS_IPV4_TCP(flag) (((flag) & (PKT_TX_TCP_SEG | PKT_TX_IPV4)) == \
+		(PKT_TX_TCP_SEG | PKT_TX_IPV4))
+
+#define IS_IPV4_VXLAN_TCP4(flag) (((flag) & (PKT_TX_TCP_SEG | PKT_TX_IPV4 | \
+				PKT_TX_OUTER_IPV4 | PKT_TX_TUNNEL_VXLAN)) == \
+		(PKT_TX_TCP_SEG | PKT_TX_IPV4 | PKT_TX_OUTER_IPV4 | \
+		 PKT_TX_TUNNEL_VXLAN))
+
+#define IS_IPV4_GRE_TCP4(flag) (((flag) & (PKT_TX_TCP_SEG | PKT_TX_IPV4 | \
+				PKT_TX_OUTER_IPV4 | PKT_TX_TUNNEL_GRE)) == \
+		(PKT_TX_TCP_SEG | PKT_TX_IPV4 | PKT_TX_OUTER_IPV4 | \
+		 PKT_TX_TUNNEL_GRE))
+
+#define IS_IPV4_UDP(flag) (((flag) & (PKT_TX_UDP_SEG | PKT_TX_IPV4)) == \
+		(PKT_TX_UDP_SEG | PKT_TX_IPV4))
+
+/**
+ * Internal function which updates the UDP header of a packet, following
+ * segmentation. This is required to update the header's datagram length field.
+ *
+ * @param pkt
+ *  The packet containing the UDP header.
+ * @param udp_offset
+ *  The offset of the UDP header from the start of the packet.
+ */
+static inline void
+update_udp_header(struct rte_mbuf *pkt, uint16_t udp_offset)
+{
+	struct udp_hdr *udp_hdr;
+
+	udp_hdr = (struct udp_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
+			udp_offset);
+	udp_hdr->dgram_len = rte_cpu_to_be_16(pkt->pkt_len - udp_offset);
+}
+
+/**
+ * Internal function which updates the TCP header of a packet, following
+ * segmentation. This is required to update the header's 'sent' sequence
+ * number, and also to clear 'PSH' and 'FIN' flags for non-tail segments.
+ *
+ * @param pkt
+ *  The packet containing the TCP header.
+ * @param l4_offset
+ *  The offset of the TCP header from the start of the packet.
+ * @param sent_seq
+ *  The sent sequence number.
+ * @param non-tail
+ *  Indicates whether or not this is a tail segment.
+ */
+static inline void
+update_tcp_header(struct rte_mbuf *pkt, uint16_t l4_offset, uint32_t sent_seq,
+		uint8_t non_tail)
+{
+	struct tcp_hdr *tcp_hdr;
+
+	tcp_hdr = (struct tcp_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
+			l4_offset);
+	tcp_hdr->sent_seq = rte_cpu_to_be_32(sent_seq);
+	if (likely(non_tail))
+		tcp_hdr->tcp_flags &= (~(TCP_HDR_PSH_MASK |
+					TCP_HDR_FIN_MASK));
+}
+
+/**
+ * Internal function which updates the IPv4 header of a packet, following
+ * segmentation. This is required to update the header's 'total_length' field,
+ * to reflect the reduced length of the now-segmented packet. Furthermore, the
+ * header's 'packet_id' field must be updated to reflect the new ID of the
+ * now-segmented packet.
+ *
+ * @param pkt
+ *  The packet containing the IPv4 header.
+ * @param l3_offset
+ *  The offset of the IPv4 header from the start of the packet.
+ * @param id
+ *  The new ID of the packet.
+ */
+static inline void
+update_ipv4_header(struct rte_mbuf *pkt, uint16_t l3_offset, uint16_t id)
+{
+	struct ipv4_hdr *ipv4_hdr;
+
+	ipv4_hdr = (struct ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
+			l3_offset);
+	ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len - l3_offset);
+	ipv4_hdr->packet_id = rte_cpu_to_be_16(id);
+}
+
+/**
+ * Internal function which divides the input packet into small segments.
+ * Each of the newly-created segments is organized as a two-segment MBUF,
+ * where the first segment is a standard mbuf, which stores a copy of
+ * packet header, and the second is an indirect mbuf which points to a
+ * section of data in the input packet.
+ *
+ * @param pkt
+ *  Packet to segment.
+ * @param pkt_hdr_offset
+ *  Packet header offset, measured in bytes.
+ * @param pyld_unit_size
+ *  The max payload length of a GSO segment.
+ * @param direct_pool
+ *  MBUF pool used for allocating direct buffers for output segments.
+ * @param indirect_pool
+ *  MBUF pool used for allocating indirect buffers for output segments.
+ * @param pkts_out
+ *  Pointer array used to keep the mbuf addresses of output segments. If
+ *  the memory space in pkts_out is insufficient, gso_do_segment() fails
+ *  and returns -EINVAL.
+ * @param nb_pkts_out
+ *  The max number of items that pkts_out can keep.
+ *
+ * @return
+ *  - The number of segments created in the event of success.
+ *  - Return -ENOMEM if run out of memory in MBUF pools.
+ *  - Return -EINVAL for invalid parameters.
+ */
+int gso_do_segment(struct rte_mbuf *pkt,
+		uint16_t pkt_hdr_offset,
+		uint16_t pyld_unit_size,
+		struct rte_mempool *direct_pool,
+		struct rte_mempool *indirect_pool,
+		struct rte_mbuf **pkts_out,
+		uint16_t nb_pkts_out);
+#endif
diff --git a/src/spdk/dpdk/lib/librte_gso/gso_tcp4.c b/src/spdk/dpdk/lib/librte_gso/gso_tcp4.c
new file mode 100644
index 00000000..fbd95f8f
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_gso/gso_tcp4.c
@@ -0,0 +1,73 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017 Intel Corporation
+ */
+
+#include "gso_common.h"
+#include "gso_tcp4.h"
+
+static void
+update_ipv4_tcp_headers(struct rte_mbuf *pkt, uint8_t ipid_delta,
+		struct rte_mbuf **segs, uint16_t nb_segs)
+{
+	struct ipv4_hdr *ipv4_hdr;
+	struct tcp_hdr *tcp_hdr;
+	uint32_t sent_seq;
+	uint16_t id, tail_idx, i;
+	uint16_t l3_offset = pkt->l2_len;
+	uint16_t l4_offset = l3_offset + pkt->l3_len;
+
+	ipv4_hdr = (struct ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char*) +
+			l3_offset);
+	tcp_hdr = (struct tcp_hdr *)((char *)ipv4_hdr + pkt->l3_len);
+	id = rte_be_to_cpu_16(ipv4_hdr->packet_id);
+	sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
+	tail_idx = nb_segs - 1;
+
+	for (i = 0; i < nb_segs; i++) {
+		update_ipv4_header(segs[i], l3_offset, id);
+		update_tcp_header(segs[i], l4_offset, sent_seq, i < tail_idx);
+		id += ipid_delta;
+		sent_seq += (segs[i]->pkt_len - segs[i]->data_len);
+	}
+}
+
+int
+gso_tcp4_segment(struct rte_mbuf *pkt,
+		uint16_t gso_size,
+		uint8_t ipid_delta,
+		struct rte_mempool *direct_pool,
+		struct rte_mempool *indirect_pool,
+		struct rte_mbuf **pkts_out,
+		uint16_t nb_pkts_out)
+{
+	struct ipv4_hdr *ipv4_hdr;
+	uint16_t pyld_unit_size, hdr_offset;
+	uint16_t frag_off;
+	int ret;
+
+	/* Don't process the fragmented packet */
+	ipv4_hdr = (struct ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
+			pkt->l2_len);
+	frag_off = rte_be_to_cpu_16(ipv4_hdr->fragment_offset);
+	if (unlikely(IS_FRAGMENTED(frag_off))) {
+		pkts_out[0] = pkt;
+		return 1;
+	}
+
+	/* Don't process the packet without data */
+	hdr_offset = pkt->l2_len + pkt->l3_len + pkt->l4_len;
+	if (unlikely(hdr_offset >= pkt->pkt_len)) {
+		pkts_out[0] = pkt;
+		return 1;
+	}
+
+	pyld_unit_size = gso_size - hdr_offset;
+
+	/* Segment the payload */
+	ret = gso_do_segment(pkt, hdr_offset, pyld_unit_size, direct_pool,
+			indirect_pool, pkts_out, nb_pkts_out);
+	if (ret > 1)
+		update_ipv4_tcp_headers(pkt, ipid_delta, pkts_out, ret);
+
+	return ret;
+}
diff --git a/src/spdk/dpdk/lib/librte_gso/gso_tcp4.h b/src/spdk/dpdk/lib/librte_gso/gso_tcp4.h
new file mode 100644
index 00000000..10d4b3ac
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_gso/gso_tcp4.h
@@ -0,0 +1,45 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017 Intel Corporation
+ */
+
+#ifndef _GSO_TCP4_H_
+#define _GSO_TCP4_H_
+
+#include <stdint.h>
+#include <rte_mbuf.h>
+
+/**
+ * Segment an IPv4/TCP packet. This function doesn't check if the input
+ * packet has correct checksums, and doesn't update checksums for output
+ * GSO segments. Furthermore, it doesn't process IP fragment packets.
+ *
+ * @param pkt
+ *  The packet mbuf to segment.
+ * @param gso_size
+ *  The max length of a GSO segment, measured in bytes.
+ * @param ipid_delta
+ *  The increasing unit of IP ids.
+ * @param direct_pool
+ *  MBUF pool used for allocating direct buffers for output segments.
+ * @param indirect_pool
+ *  MBUF pool used for allocating indirect buffers for output segments.
+ * @param pkts_out
+ *  Pointer array used to store the MBUF addresses of output GSO
+ *  segments, when the function succeeds. If the memory space in
+ *  pkts_out is insufficient, it fails and returns -EINVAL.
+ * @param nb_pkts_out
+ *  The max number of items that 'pkts_out' can keep.
+ *
+ * @return
+ *   - The number of GSO segments filled in pkts_out on success.
+ *   - Return -ENOMEM if run out of memory in MBUF pools.
+ *   - Return -EINVAL for invalid parameters.
+ */
+int gso_tcp4_segment(struct rte_mbuf *pkt,
+		uint16_t gso_size,
+		uint8_t ip_delta,
+		struct rte_mempool *direct_pool,
+		struct rte_mempool *indirect_pool,
+		struct rte_mbuf **pkts_out,
+		uint16_t nb_pkts_out);
+#endif
diff --git a/src/spdk/dpdk/lib/librte_gso/gso_tunnel_tcp4.c b/src/spdk/dpdk/lib/librte_gso/gso_tunnel_tcp4.c
new file mode 100644
index 00000000..d39b4686
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_gso/gso_tunnel_tcp4.c
@@ -0,0 +1,97 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017 Intel Corporation
+ */
+
+#include "gso_common.h"
+#include "gso_tunnel_tcp4.h"
+
+static void
+update_tunnel_ipv4_tcp_headers(struct rte_mbuf *pkt, uint8_t ipid_delta,
+		struct rte_mbuf **segs, uint16_t nb_segs)
+{
+	struct ipv4_hdr *ipv4_hdr;
+	struct tcp_hdr *tcp_hdr;
+	uint32_t sent_seq;
+	uint16_t outer_id, inner_id, tail_idx, i;
+	uint16_t outer_ipv4_offset, inner_ipv4_offset;
+	uint16_t udp_gre_offset, tcp_offset;
+	uint8_t update_udp_hdr;
+
+	outer_ipv4_offset = pkt->outer_l2_len;
+	udp_gre_offset = outer_ipv4_offset + pkt->outer_l3_len;
+	inner_ipv4_offset = udp_gre_offset + pkt->l2_len;
+	tcp_offset = inner_ipv4_offset + pkt->l3_len;
+
+	/* Outer IPv4 header. */
+	ipv4_hdr = (struct ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
+			outer_ipv4_offset);
+	outer_id = rte_be_to_cpu_16(ipv4_hdr->packet_id);
+
+	/* Inner IPv4 header. */
+	ipv4_hdr = (struct ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
+			inner_ipv4_offset);
+	inner_id = rte_be_to_cpu_16(ipv4_hdr->packet_id);
+
+	tcp_hdr = (struct tcp_hdr *)((char *)ipv4_hdr + pkt->l3_len);
+	sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
+	tail_idx = nb_segs - 1;
+
+	/* Only update UDP header for VxLAN packets. */
+	update_udp_hdr = (pkt->ol_flags & PKT_TX_TUNNEL_VXLAN) ? 1 : 0;
+
+	for (i = 0; i < nb_segs; i++) {
+		update_ipv4_header(segs[i], outer_ipv4_offset, outer_id);
+		if (update_udp_hdr)
+			update_udp_header(segs[i], udp_gre_offset);
+		update_ipv4_header(segs[i], inner_ipv4_offset, inner_id);
+		update_tcp_header(segs[i], tcp_offset, sent_seq, i < tail_idx);
+		outer_id++;
+		inner_id += ipid_delta;
+		sent_seq += (segs[i]->pkt_len - segs[i]->data_len);
+	}
+}
+
+int
+gso_tunnel_tcp4_segment(struct rte_mbuf *pkt,
+		uint16_t gso_size,
+		uint8_t ipid_delta,
+		struct rte_mempool *direct_pool,
+		struct rte_mempool *indirect_pool,
+		struct rte_mbuf **pkts_out,
+		uint16_t nb_pkts_out)
+{
+	struct ipv4_hdr *inner_ipv4_hdr;
+	uint16_t pyld_unit_size, hdr_offset, frag_off;
+	int ret = 1;
+
+	hdr_offset = pkt->outer_l2_len + pkt->outer_l3_len + pkt->l2_len;
+	inner_ipv4_hdr = (struct ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
+			hdr_offset);
+	/*
+	 * Don't process the packet whose MF bit or offset in the inner
+	 * IPv4 header are non-zero.
+	 */
+	frag_off = rte_be_to_cpu_16(inner_ipv4_hdr->fragment_offset);
+	if (unlikely(IS_FRAGMENTED(frag_off))) {
+		pkts_out[0] = pkt;
+		return 1;
+	}
+
+	hdr_offset += pkt->l3_len + pkt->l4_len;
+	/* Don't process the packet without data */
+	if (hdr_offset >= pkt->pkt_len) {
+		pkts_out[0] = pkt;
+		return 1;
+	}
+	pyld_unit_size = gso_size - hdr_offset;
+
+	/* Segment the payload */
+	ret = gso_do_segment(pkt, hdr_offset, pyld_unit_size, direct_pool,
+			indirect_pool, pkts_out, nb_pkts_out);
+	if (ret <= 1)
+		return ret;
+
+	update_tunnel_ipv4_tcp_headers(pkt, ipid_delta, pkts_out, ret);
+
+	return ret;
+}
diff --git a/src/spdk/dpdk/lib/librte_gso/gso_tunnel_tcp4.h b/src/spdk/dpdk/lib/librte_gso/gso_tunnel_tcp4.h
new file mode 100644
index 00000000..c522ed27
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_gso/gso_tunnel_tcp4.h
@@ -0,0 +1,46 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017 Intel Corporation
+ */
+
+#ifndef _GSO_TUNNEL_TCP4_H_
+#define _GSO_TUNNEL_TCP4_H_
+
+#include <stdint.h>
+#include <rte_mbuf.h>
+
+/**
+ * Segment a tunneling packet with inner TCP/IPv4 headers. This function
+ * doesn't check if the input packet has correct checksums, and doesn't
+ * update checksums for output GSO segments. Furthermore, it doesn't
+ * process IP fragment packets.
+ *
+ * @param pkt
+ *  The packet mbuf to segment.
+ * @param gso_size
+ *  The max length of a GSO segment, measured in bytes.
+ * @param ipid_delta
+ *  The increasing unit of IP ids.
+ * @param direct_pool
+ *  MBUF pool used for allocating direct buffers for output segments.
+ * @param indirect_pool
+ *  MBUF pool used for allocating indirect buffers for output segments.
+ * @param pkts_out
+ *  Pointer array used to store the MBUF addresses of output GSO
+ *  segments, when it succeeds. If the memory space in pkts_out is
+ *  insufficient, it fails and returns -EINVAL.
+ * @param nb_pkts_out
+ *  The max number of items that 'pkts_out' can keep.
+ *
+ * @return
+ *   - The number of GSO segments filled in pkts_out on success.
+ *   - Return -ENOMEM if run out of memory in MBUF pools.
+ *   - Return -EINVAL for invalid parameters.
+ */
+int gso_tunnel_tcp4_segment(struct rte_mbuf *pkt,
+		uint16_t gso_size,
+		uint8_t ipid_delta,
+		struct rte_mempool *direct_pool,
+		struct rte_mempool *indirect_pool,
+		struct rte_mbuf **pkts_out,
+		uint16_t nb_pkts_out);
+#endif
diff --git a/src/spdk/dpdk/lib/librte_gso/gso_udp4.c b/src/spdk/dpdk/lib/librte_gso/gso_udp4.c
new file mode 100644
index 00000000..927dee12
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_gso/gso_udp4.c
@@ -0,0 +1,81 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#include "gso_common.h"
+#include "gso_udp4.h"
+
+#define IPV4_HDR_MF_BIT (1U << 13)
+
+static inline void
+update_ipv4_udp_headers(struct rte_mbuf *pkt, struct rte_mbuf **segs,
+		uint16_t nb_segs)
+{
+	struct ipv4_hdr *ipv4_hdr;
+	uint16_t frag_offset = 0, is_mf;
+	uint16_t l2_hdrlen = pkt->l2_len, l3_hdrlen = pkt->l3_len;
+	uint16_t tail_idx = nb_segs - 1, length, i;
+
+	/*
+	 * Update IP header fields for output segments. Specifically,
+	 * keep the same IP id, update fragment offset and total
+	 * length.
+	 */
+	for (i = 0; i < nb_segs; i++) {
+		ipv4_hdr = rte_pktmbuf_mtod_offset(segs[i], struct ipv4_hdr *,
+			l2_hdrlen);
+		length = segs[i]->pkt_len - l2_hdrlen;
+		ipv4_hdr->total_length = rte_cpu_to_be_16(length);
+
+		is_mf = i < tail_idx ? IPV4_HDR_MF_BIT : 0;
+		ipv4_hdr->fragment_offset =
+			rte_cpu_to_be_16(frag_offset | is_mf);
+		frag_offset += ((length - l3_hdrlen) >> 3);
+	}
+}
+
+int
+gso_udp4_segment(struct rte_mbuf *pkt,
+		uint16_t gso_size,
+		struct rte_mempool *direct_pool,
+		struct rte_mempool *indirect_pool,
+		struct rte_mbuf **pkts_out,
+		uint16_t nb_pkts_out)
+{
+	struct ipv4_hdr *ipv4_hdr;
+	uint16_t pyld_unit_size, hdr_offset;
+	uint16_t frag_off;
+	int ret;
+
+	/* Don't process the fragmented packet */
+	ipv4_hdr = rte_pktmbuf_mtod_offset(pkt, struct ipv4_hdr *,
+			pkt->l2_len);
+	frag_off = rte_be_to_cpu_16(ipv4_hdr->fragment_offset);
+	if (unlikely(IS_FRAGMENTED(frag_off))) {
+		pkts_out[0] = pkt;
+		return 1;
+	}
+
+	/*
+	 * UDP fragmentation is the same as IP fragmentation.
+	 * Except the first one, other output packets just have l2
+	 * and l3 headers.
+	 */
+	hdr_offset = pkt->l2_len + pkt->l3_len;
+
+	/* Don't process the packet without data. */
+	if (unlikely(hdr_offset + pkt->l4_len >= pkt->pkt_len)) {
+		pkts_out[0] = pkt;
+		return 1;
+	}
+
+	pyld_unit_size = gso_size - hdr_offset;
+
+	/* Segment the payload */
+	ret = gso_do_segment(pkt, hdr_offset, pyld_unit_size, direct_pool,
+			indirect_pool, pkts_out, nb_pkts_out);
+	if (ret > 1)
+		update_ipv4_udp_headers(pkt, pkts_out, ret);
+
+	return ret;
+}
diff --git a/src/spdk/dpdk/lib/librte_gso/gso_udp4.h b/src/spdk/dpdk/lib/librte_gso/gso_udp4.h
new file mode 100644
index 00000000..b2a2908e
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_gso/gso_udp4.h
@@ -0,0 +1,42 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#ifndef _GSO_UDP4_H_
+#define _GSO_UDP4_H_
+
+#include <stdint.h>
+#include <rte_mbuf.h>
+
+/**
+ * Segment an UDP/IPv4 packet. This function doesn't check if the input
+ * packet has correct checksums, and doesn't update checksums for output
+ * GSO segments. Furthermore, it doesn't process IP fragment packets.
+ *
+ * @param pkt
+ *  The packet mbuf to segment.
+ * @param gso_size
+ *  The max length of a GSO segment, measured in bytes.
+ * @param direct_pool
+ *  MBUF pool used for allocating direct buffers for output segments.
+ * @param indirect_pool
+ *  MBUF pool used for allocating indirect buffers for output segments.
+ * @param pkts_out
+ *  Pointer array used to store the MBUF addresses of output GSO
+ *  segments, when the function succeeds. If the memory space in
+ *  pkts_out is insufficient, it fails and returns -EINVAL.
+ * @param nb_pkts_out
+ *  The max number of items that 'pkts_out' can keep.
+ *
+ * @return
+ *   - The number of GSO segments filled in pkts_out on success.
+ *   - Return -ENOMEM if run out of memory in MBUF pools.
+ *   - Return -EINVAL for invalid parameters.
+ */
+int gso_udp4_segment(struct rte_mbuf *pkt,
+		uint16_t gso_size,
+		struct rte_mempool *direct_pool,
+		struct rte_mempool *indirect_pool,
+		struct rte_mbuf **pkts_out,
+		uint16_t nb_pkts_out);
+#endif
diff --git a/src/spdk/dpdk/lib/librte_gso/meson.build b/src/spdk/dpdk/lib/librte_gso/meson.build
new file mode 100644
index 00000000..ad8dd858
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_gso/meson.build
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2017 Intel Corporation
+
+sources = files('gso_common.c', 'gso_tcp4.c', 'gso_udp4.c',
+ 		'gso_tunnel_tcp4.c', 'rte_gso.c')
+headers = files('rte_gso.h')
+deps += ['ethdev']
diff --git a/src/spdk/dpdk/lib/librte_gso/rte_gso.c b/src/spdk/dpdk/lib/librte_gso/rte_gso.c
new file mode 100644
index 00000000..751b5b62
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_gso/rte_gso.c
@@ -0,0 +1,95 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017 Intel Corporation
+ */
+
+#include <errno.h>
+
+#include <rte_log.h>
+#include <rte_ethdev.h>
+
+#include "rte_gso.h"
+#include "gso_common.h"
+#include "gso_tcp4.h"
+#include "gso_tunnel_tcp4.h"
+#include "gso_udp4.h"
+
+#define ILLEGAL_UDP_GSO_CTX(ctx) \
+	((((ctx)->gso_types & DEV_TX_OFFLOAD_UDP_TSO) == 0) || \
+	 (ctx)->gso_size < RTE_GSO_UDP_SEG_SIZE_MIN)
+
+#define ILLEGAL_TCP_GSO_CTX(ctx) \
+	((((ctx)->gso_types & (DEV_TX_OFFLOAD_TCP_TSO | \
+		DEV_TX_OFFLOAD_VXLAN_TNL_TSO | \
+		DEV_TX_OFFLOAD_GRE_TNL_TSO)) == 0) || \
+		(ctx)->gso_size < RTE_GSO_SEG_SIZE_MIN)
+
+int
+rte_gso_segment(struct rte_mbuf *pkt,
+		const struct rte_gso_ctx *gso_ctx,
+		struct rte_mbuf **pkts_out,
+		uint16_t nb_pkts_out)
+{
+	struct rte_mempool *direct_pool, *indirect_pool;
+	struct rte_mbuf *pkt_seg;
+	uint64_t ol_flags;
+	uint16_t gso_size;
+	uint8_t ipid_delta;
+	int ret = 1;
+
+	if (pkt == NULL || pkts_out == NULL || gso_ctx == NULL ||
+			nb_pkts_out < 1 ||
+			(ILLEGAL_UDP_GSO_CTX(gso_ctx) &&
+			 ILLEGAL_TCP_GSO_CTX(gso_ctx)))
+		return -EINVAL;
+
+	if (gso_ctx->gso_size >= pkt->pkt_len) {
+		pkt->ol_flags &= (~(PKT_TX_TCP_SEG | PKT_TX_UDP_SEG));
+		pkts_out[0] = pkt;
+		return 1;
+	}
+
+	direct_pool = gso_ctx->direct_pool;
+	indirect_pool = gso_ctx->indirect_pool;
+	gso_size = gso_ctx->gso_size;
+	ipid_delta = (gso_ctx->flag != RTE_GSO_FLAG_IPID_FIXED);
+	ol_flags = pkt->ol_flags;
+
+	if ((IS_IPV4_VXLAN_TCP4(pkt->ol_flags) &&
+			(gso_ctx->gso_types & DEV_TX_OFFLOAD_VXLAN_TNL_TSO)) ||
+			((IS_IPV4_GRE_TCP4(pkt->ol_flags) &&
+			 (gso_ctx->gso_types & DEV_TX_OFFLOAD_GRE_TNL_TSO)))) {
+		pkt->ol_flags &= (~PKT_TX_TCP_SEG);
+		ret = gso_tunnel_tcp4_segment(pkt, gso_size, ipid_delta,
+				direct_pool, indirect_pool,
+				pkts_out, nb_pkts_out);
+	} else if (IS_IPV4_TCP(pkt->ol_flags) &&
+			(gso_ctx->gso_types & DEV_TX_OFFLOAD_TCP_TSO)) {
+		pkt->ol_flags &= (~PKT_TX_TCP_SEG);
+		ret = gso_tcp4_segment(pkt, gso_size, ipid_delta,
+				direct_pool, indirect_pool,
+				pkts_out, nb_pkts_out);
+	} else if (IS_IPV4_UDP(pkt->ol_flags) &&
+			(gso_ctx->gso_types & DEV_TX_OFFLOAD_UDP_TSO)) {
+		pkt->ol_flags &= (~PKT_TX_UDP_SEG);
+		ret = gso_udp4_segment(pkt, gso_size, direct_pool,
+				indirect_pool, pkts_out, nb_pkts_out);
+	} else {
+		/* unsupported packet, skip */
+		pkts_out[0] = pkt;
+		RTE_LOG(DEBUG, GSO, "Unsupported packet type\n");
+		return 1;
+	}
+
+	if (ret > 1) {
+		pkt_seg = pkt;
+		while (pkt_seg) {
+			rte_mbuf_refcnt_update(pkt_seg, -1);
+			pkt_seg = pkt_seg->next;
+		}
+	} else if (ret < 0) {
+		/* Revert the ol_flags in the event of failure. */
+		pkt->ol_flags = ol_flags;
+	}
+
+	return ret;
+}
diff --git a/src/spdk/dpdk/lib/librte_gso/rte_gso.h b/src/spdk/dpdk/lib/librte_gso/rte_gso.h
new file mode 100644
index 00000000..a626a11e
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_gso/rte_gso.h
@@ -0,0 +1,123 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017 Intel Corporation
+ */
+
+#ifndef _RTE_GSO_H_
+#define _RTE_GSO_H_
+
+/**
+ * @file
+ * Interface to GSO library
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+#include <rte_mbuf.h>
+
+/* Minimum GSO segment size for TCP based packets. */
+#define RTE_GSO_SEG_SIZE_MIN (sizeof(struct ether_hdr) + \
+		sizeof(struct ipv4_hdr) + sizeof(struct tcp_hdr) + 1)
+
+/* Minimum GSO segment size for UDP based packets. */
+#define RTE_GSO_UDP_SEG_SIZE_MIN (sizeof(struct ether_hdr) + \
+		sizeof(struct ipv4_hdr) + sizeof(struct udp_hdr) + 1)
+
+/* GSO flags for rte_gso_ctx. */
+#define RTE_GSO_FLAG_IPID_FIXED (1ULL << 0)
+/**< Use fixed IP ids for output GSO segments. Setting
+ * 0 indicates using incremental IP ids.
+ */
+
+/**
+ * GSO context structure.
+ */
+struct rte_gso_ctx {
+	struct rte_mempool *direct_pool;
+	/**< MBUF pool for allocating direct buffers, which are used
+	 * to store packet headers for GSO segments.
+	 */
+	struct rte_mempool *indirect_pool;
+	/**< MBUF pool for allocating indirect buffers, which are used
+	 * to locate packet payloads for GSO segments. The indirect
+	 * buffer doesn't contain any data, but simply points to an
+	 * offset within the packet to segment.
+	 */
+	uint64_t flag;
+	/**< flag that controls specific attributes of output segments,
+	 * such as the type of IP ID generated (i.e. fixed or incremental).
+	 */
+	uint32_t gso_types;
+	/**< the bit mask of required GSO types. The GSO library
+	 * uses the same macros as that of describing device TX
+	 * offloading capabilities (i.e. DEV_TX_OFFLOAD_*_TSO) for
+	 * gso_types.
+	 *
+	 * For example, if applications want to segment TCP/IPv4
+	 * packets, set DEV_TX_OFFLOAD_TCP_TSO in gso_types.
+	 */
+	uint16_t gso_size;
+	/**< maximum size of an output GSO segment, including packet
+	 * header and payload, measured in bytes. Must exceed
+	 * RTE_GSO_SEG_SIZE_MIN.
+	 */
+};
+
+/**
+ * Segmentation function, which supports processing of both single- and
+ * multi- MBUF packets.
+ *
+ * Note that we refer to the packets that are segmented from the input
+ * packet as 'GSO segments'. rte_gso_segment() doesn't check if the
+ * input packet has correct checksums, and doesn't update checksums for
+ * output GSO segments. Additionally, it doesn't process IP fragment
+ * packets.
+ *
+ * Before calling rte_gso_segment(), applications must set proper ol_flags
+ * for the packet. The GSO library uses the same macros as that of TSO.
+ * For example, set PKT_TX_TCP_SEG and PKT_TX_IPV4 in ol_flags to segment
+ * a TCP/IPv4 packet. If rte_gso_segment() succeeds, the PKT_TX_TCP_SEG
+ * flag is removed for all GSO segments and the input packet.
+ *
+ * Each of the newly-created GSO segments is organized as a two-segment
+ * MBUF, where the first segment is a standard MBUF, which stores a copy
+ * of packet header, and the second is an indirect MBUF which points to
+ * a section of data in the input packet. Since each GSO segment has
+ * multiple MBUFs (i.e. typically 2 MBUFs), the driver of the interface which
+ * the GSO segments are sent to should support transmission of multi-segment
+ * packets.
+ *
+ * If the input packet is GSO'd, its mbuf refcnt reduces by 1. Therefore,
+ * when all GSO segments are freed, the input packet is freed automatically.
+ *
+ * If the memory space in pkts_out or MBUF pools is insufficient, this
+ * function fails, and it returns (-1) * errno. Otherwise, GSO succeeds,
+ * and this function returns the number of output GSO segments filled in
+ * pkts_out.
+ *
+ * @param pkt
+ *  The packet mbuf to segment.
+ * @param ctx
+ *  GSO context object pointer.
+ * @param pkts_out
+ *  Pointer array used to store the MBUF addresses of output GSO
+ *  segments, when rte_gso_segment() succeeds.
+ * @param nb_pkts_out
+ *  The max number of items that pkts_out can keep.
+ *
+ * @return
+ *  - The number of GSO segments filled in pkts_out on success.
+ *  - Return -ENOMEM if run out of memory in MBUF pools.
+ *  - Return -EINVAL for invalid parameters.
+ */
+int rte_gso_segment(struct rte_mbuf *pkt,
+		const struct rte_gso_ctx *ctx,
+		struct rte_mbuf **pkts_out,
+		uint16_t nb_pkts_out);
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_GSO_H_ */
diff --git a/src/spdk/dpdk/lib/librte_gso/rte_gso_version.map b/src/spdk/dpdk/lib/librte_gso/rte_gso_version.map
new file mode 100644
index 00000000..e1fd453e
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_gso/rte_gso_version.map
@@ -0,0 +1,7 @@
+DPDK_17.11 {
+	global:
+
+	rte_gso_segment;
+
+	local: *;
+};