Adding upstream version 18.2.2.upstream/18.2.2

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

12 files changed, 3512 insertions, 0 deletions

diff --git a/src/spdk/dpdk/lib/librte_graph/Makefile b/src/spdk/dpdk/lib/librte_graph/Makefile
new file mode 100644
index 000000000..b66279c67
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_graph/Makefile
@@ -0,0 +1,28 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(C) 2020 Marvell International Ltd.
+#
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+# library name
+LIB = librte_graph.a
+
+CFLAGS += -O3
+CFLAGS += $(WERROR_FLAGS)
+LDLIBS += -lrte_eal
+
+EXPORT_MAP := rte_graph_version.map
+
+# all source are stored in SRCS-y
+SRCS-$(CONFIG_RTE_LIBRTE_GRAPH) += node.c
+SRCS-$(CONFIG_RTE_LIBRTE_GRAPH) += graph.c
+SRCS-$(CONFIG_RTE_LIBRTE_GRAPH) += graph_ops.c
+SRCS-$(CONFIG_RTE_LIBRTE_GRAPH) += graph_debug.c
+SRCS-$(CONFIG_RTE_LIBRTE_GRAPH) += graph_stats.c
+SRCS-$(CONFIG_RTE_LIBRTE_GRAPH) += graph_populate.c
+
+# install header files
+SYMLINK-$(CONFIG_RTE_LIBRTE_GRAPH)-include += rte_graph.h
+SYMLINK-$(CONFIG_RTE_LIBRTE_GRAPH)-include += rte_graph_worker.h
+
+include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/src/spdk/dpdk/lib/librte_graph/graph.c b/src/spdk/dpdk/lib/librte_graph/graph.c
new file mode 100644
index 000000000..0ea83df3d
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_graph/graph.c
@@ -0,0 +1,587 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2020 Marvell International Ltd.
+ */
+
+#include <fnmatch.h>
+#include <stdbool.h>
+
+#include <rte_common.h>
+#include <rte_debug.h>
+#include <rte_errno.h>
+#include <rte_malloc.h>
+#include <rte_memzone.h>
+#include <rte_spinlock.h>
+#include <rte_string_fns.h>
+
+#include "graph_private.h"
+
+static struct graph_head graph_list = STAILQ_HEAD_INITIALIZER(graph_list);
+static rte_spinlock_t graph_lock = RTE_SPINLOCK_INITIALIZER;
+static rte_graph_t graph_id;
+int rte_graph_logtype;
+
+#define GRAPH_ID_CHECK(id) ID_CHECK(id, graph_id)
+
+/* Private functions */
+struct graph_head *
+graph_list_head_get(void)
+{
+	return &graph_list;
+}
+
+void
+graph_spinlock_lock(void)
+{
+	rte_spinlock_lock(&graph_lock);
+}
+
+void
+graph_spinlock_unlock(void)
+{
+	rte_spinlock_unlock(&graph_lock);
+}
+
+static int
+graph_node_add(struct graph *graph, struct node *node)
+{
+	struct graph_node *graph_node;
+	size_t sz;
+
+	/* Skip the duplicate nodes */
+	STAILQ_FOREACH(graph_node, &graph->node_list, next)
+		if (strncmp(node->name, graph_node->node->name,
+			    RTE_NODE_NAMESIZE) == 0)
+			return 0;
+
+	/* Allocate new graph node object */
+	sz = sizeof(*graph_node) + node->nb_edges * sizeof(struct node *);
+	graph_node = calloc(1, sz);
+
+	if (graph_node == NULL)
+		SET_ERR_JMP(ENOMEM, free, "Failed to calloc %s object",
+			    node->name);
+
+	/* Initialize the graph node */
+	graph_node->node = node;
+
+	/* Add to graph node list */
+	STAILQ_INSERT_TAIL(&graph->node_list, graph_node, next);
+	return 0;
+
+free:
+	free(graph_node);
+	return -rte_errno;
+}
+
+static struct graph_node *
+node_to_graph_node(struct graph *graph, struct node *node)
+{
+	struct graph_node *graph_node;
+
+	STAILQ_FOREACH(graph_node, &graph->node_list, next)
+		if (graph_node->node == node)
+			return graph_node;
+
+	SET_ERR_JMP(ENODEV, fail, "Found isolated node %s", node->name);
+fail:
+	return NULL;
+}
+
+static int
+graph_node_edges_add(struct graph *graph)
+{
+	struct graph_node *graph_node;
+	struct node *adjacency;
+	const char *next;
+	rte_edge_t i;
+
+	STAILQ_FOREACH(graph_node, &graph->node_list, next) {
+		for (i = 0; i < graph_node->node->nb_edges; i++) {
+			next = graph_node->node->next_nodes[i];
+			adjacency = node_from_name(next);
+			if (adjacency == NULL)
+				SET_ERR_JMP(EINVAL, fail,
+					    "Node %s not registered", next);
+			if (graph_node_add(graph, adjacency))
+				goto fail;
+		}
+	}
+	return 0;
+fail:
+	return -rte_errno;
+}
+
+static int
+graph_adjacency_list_update(struct graph *graph)
+{
+	struct graph_node *graph_node, *tmp;
+	struct node *adjacency;
+	const char *next;
+	rte_edge_t i;
+
+	STAILQ_FOREACH(graph_node, &graph->node_list, next) {
+		for (i = 0; i < graph_node->node->nb_edges; i++) {
+			next = graph_node->node->next_nodes[i];
+			adjacency = node_from_name(next);
+			if (adjacency == NULL)
+				SET_ERR_JMP(EINVAL, fail,
+					    "Node %s not registered", next);
+			tmp = node_to_graph_node(graph, adjacency);
+			if (tmp == NULL)
+				goto fail;
+			graph_node->adjacency_list[i] = tmp;
+		}
+	}
+
+	return 0;
+fail:
+	return -rte_errno;
+}
+
+static int
+expand_pattern_to_node(struct graph *graph, const char *pattern)
+{
+	struct node_head *node_head = node_list_head_get();
+	bool found = false;
+	struct node *node;
+
+	/* Check for pattern match */
+	STAILQ_FOREACH(node, node_head, next) {
+		if (fnmatch(pattern, node->name, 0) == 0) {
+			if (graph_node_add(graph, node))
+				goto fail;
+			found = true;
+		}
+	}
+	if (found == false)
+		SET_ERR_JMP(EFAULT, fail, "Pattern %s node not found", pattern);
+
+	return 0;
+fail:
+	return -rte_errno;
+}
+
+static void
+graph_cleanup(struct graph *graph)
+{
+	struct graph_node *graph_node;
+
+	while (!STAILQ_EMPTY(&graph->node_list)) {
+		graph_node = STAILQ_FIRST(&graph->node_list);
+		STAILQ_REMOVE_HEAD(&graph->node_list, next);
+		free(graph_node);
+	}
+}
+
+static int
+graph_node_init(struct graph *graph)
+{
+	struct graph_node *graph_node;
+	const char *name;
+	int rc;
+
+	STAILQ_FOREACH(graph_node, &graph->node_list, next) {
+		if (graph_node->node->init) {
+			name = graph_node->node->name;
+			rc = graph_node->node->init(
+				graph->graph,
+				graph_node_name_to_ptr(graph->graph, name));
+			if (rc)
+				SET_ERR_JMP(rc, err, "Node %s init() failed",
+					    name);
+		}
+	}
+
+	return 0;
+err:
+	return -rte_errno;
+}
+
+static void
+graph_node_fini(struct graph *graph)
+{
+	struct graph_node *graph_node;
+
+	STAILQ_FOREACH(graph_node, &graph->node_list, next)
+		if (graph_node->node->fini)
+			graph_node->node->fini(
+				graph->graph,
+				graph_node_name_to_ptr(graph->graph,
+						       graph_node->node->name));
+}
+
+static struct rte_graph *
+graph_mem_fixup_node_ctx(struct rte_graph *graph)
+{
+	struct rte_node *node;
+	struct node *node_db;
+	rte_graph_off_t off;
+	rte_node_t count;
+	const char *name;
+
+	rte_graph_foreach_node(count, off, graph, node) {
+		if (node->parent_id == RTE_NODE_ID_INVALID) /* Static node */
+			name = node->name;
+		else /* Cloned node */
+			name = node->parent;
+
+		node_db = node_from_name(name);
+		if (node_db == NULL)
+			SET_ERR_JMP(ENOLINK, fail, "Node %s not found", name);
+		node->process = node_db->process;
+	}
+
+	return graph;
+fail:
+	return NULL;
+}
+
+static struct rte_graph *
+graph_mem_fixup_secondary(struct rte_graph *graph)
+{
+	if (graph == NULL || rte_eal_process_type() == RTE_PROC_PRIMARY)
+		return graph;
+
+	return graph_mem_fixup_node_ctx(graph);
+}
+
+struct rte_graph *
+rte_graph_lookup(const char *name)
+{
+	const struct rte_memzone *mz;
+	struct rte_graph *rc = NULL;
+
+	mz = rte_memzone_lookup(name);
+	if (mz)
+		rc = mz->addr;
+
+	return graph_mem_fixup_secondary(rc);
+}
+
+rte_graph_t
+rte_graph_create(const char *name, struct rte_graph_param *prm)
+{
+	rte_node_t src_node_count;
+	struct graph *graph;
+	const char *pattern;
+	uint16_t i;
+
+	graph_spinlock_lock();
+
+	/* Check arguments sanity */
+	if (prm == NULL)
+		SET_ERR_JMP(EINVAL, fail, "Param should not be NULL");
+
+	if (name == NULL)
+		SET_ERR_JMP(EINVAL, fail, "Graph name should not be NULL");
+
+	/* Check for existence of duplicate graph */
+	STAILQ_FOREACH(graph, &graph_list, next)
+		if (strncmp(name, graph->name, RTE_GRAPH_NAMESIZE) == 0)
+			SET_ERR_JMP(EEXIST, fail, "Found duplicate graph %s",
+				    name);
+
+	/* Create graph object */
+	graph = calloc(1, sizeof(*graph));
+	if (graph == NULL)
+		SET_ERR_JMP(ENOMEM, fail, "Failed to calloc graph object");
+
+	/* Initialize the graph object */
+	STAILQ_INIT(&graph->node_list);
+	if (rte_strscpy(graph->name, name, RTE_GRAPH_NAMESIZE) < 0)
+		SET_ERR_JMP(E2BIG, free, "Too big name=%s", name);
+
+	/* Expand node pattern and add the nodes to the graph */
+	for (i = 0; i < prm->nb_node_patterns; i++) {
+		pattern = prm->node_patterns[i];
+		if (expand_pattern_to_node(graph, pattern))
+			goto graph_cleanup;
+	}
+
+	/* Go over all the nodes edges and add them to the graph */
+	if (graph_node_edges_add(graph))
+		goto graph_cleanup;
+
+	/* Update adjacency list of all nodes in the graph */
+	if (graph_adjacency_list_update(graph))
+		goto graph_cleanup;
+
+	/* Make sure at least a source node present in the graph */
+	src_node_count = graph_src_nodes_count(graph);
+	if (src_node_count == 0)
+		goto graph_cleanup;
+
+	/* Make sure no node is pointing to source node */
+	if (graph_node_has_edge_to_src_node(graph))
+		goto graph_cleanup;
+
+	/* Don't allow node has loop to self */
+	if (graph_node_has_loop_edge(graph))
+		goto graph_cleanup;
+
+	/* Do BFS from src nodes on the graph to find isolated nodes */
+	if (graph_has_isolated_node(graph))
+		goto graph_cleanup;
+
+	/* Initialize graph object */
+	graph->socket = prm->socket_id;
+	graph->src_node_count = src_node_count;
+	graph->node_count = graph_nodes_count(graph);
+	graph->id = graph_id;
+
+	/* Allocate the Graph fast path memory and populate the data */
+	if (graph_fp_mem_create(graph))
+		goto graph_cleanup;
+
+	/* Call init() of the all the nodes in the graph */
+	if (graph_node_init(graph))
+		goto graph_mem_destroy;
+
+	/* All good, Lets add the graph to the list */
+	graph_id++;
+	STAILQ_INSERT_TAIL(&graph_list, graph, next);
+
+	graph_spinlock_unlock();
+	return graph->id;
+
+graph_mem_destroy:
+	graph_fp_mem_destroy(graph);
+graph_cleanup:
+	graph_cleanup(graph);
+free:
+	free(graph);
+fail:
+	graph_spinlock_unlock();
+	return RTE_GRAPH_ID_INVALID;
+}
+
+int
+rte_graph_destroy(rte_graph_t id)
+{
+	struct graph *graph, *tmp;
+	int rc = -ENOENT;
+
+	graph_spinlock_lock();
+
+	graph = STAILQ_FIRST(&graph_list);
+	while (graph != NULL) {
+		tmp = STAILQ_NEXT(graph, next);
+		if (graph->id == id) {
+			/* Call fini() of the all the nodes in the graph */
+			graph_node_fini(graph);
+			/* Destroy graph fast path memory */
+			rc = graph_fp_mem_destroy(graph);
+			if (rc)
+				SET_ERR_JMP(rc, done, "Graph %s destroy failed",
+					    graph->name);
+
+			graph_cleanup(graph);
+			STAILQ_REMOVE(&graph_list, graph, graph, next);
+			free(graph);
+			graph_id--;
+			goto done;
+		}
+		graph = tmp;
+	}
+done:
+	graph_spinlock_unlock();
+	return rc;
+}
+
+rte_graph_t
+rte_graph_from_name(const char *name)
+{
+	struct graph *graph;
+
+	STAILQ_FOREACH(graph, &graph_list, next)
+		if (strncmp(graph->name, name, RTE_GRAPH_NAMESIZE) == 0)
+			return graph->id;
+
+	return RTE_GRAPH_ID_INVALID;
+}
+
+char *
+rte_graph_id_to_name(rte_graph_t id)
+{
+	struct graph *graph;
+
+	GRAPH_ID_CHECK(id);
+	STAILQ_FOREACH(graph, &graph_list, next)
+		if (graph->id == id)
+			return graph->name;
+
+fail:
+	return NULL;
+}
+
+struct rte_node *
+rte_graph_node_get(rte_graph_t gid, uint32_t nid)
+{
+	struct rte_node *node;
+	struct graph *graph;
+	rte_graph_off_t off;
+	rte_node_t count;
+
+	GRAPH_ID_CHECK(gid);
+	STAILQ_FOREACH(graph, &graph_list, next)
+		if (graph->id == gid) {
+			rte_graph_foreach_node(count, off, graph->graph,
+						node) {
+				if (node->id == nid)
+					return node;
+			}
+			break;
+		}
+fail:
+	return NULL;
+}
+
+struct rte_node *
+rte_graph_node_get_by_name(const char *graph_name, const char *node_name)
+{
+	struct rte_node *node;
+	struct graph *graph;
+	rte_graph_off_t off;
+	rte_node_t count;
+
+	STAILQ_FOREACH(graph, &graph_list, next)
+		if (!strncmp(graph->name, graph_name, RTE_GRAPH_NAMESIZE)) {
+			rte_graph_foreach_node(count, off, graph->graph,
+						node) {
+				if (!strncmp(node->name, node_name,
+					     RTE_NODE_NAMESIZE))
+					return node;
+			}
+			break;
+		}
+
+	return NULL;
+}
+
+void __rte_noinline
+__rte_node_stream_alloc(struct rte_graph *graph, struct rte_node *node)
+{
+	uint16_t size = node->size;
+
+	RTE_VERIFY(size != UINT16_MAX);
+	/* Allocate double amount of size to avoid immediate realloc */
+	size = RTE_MIN(UINT16_MAX, RTE_MAX(RTE_GRAPH_BURST_SIZE, size * 2));
+	node->objs = rte_realloc_socket(node->objs, size * sizeof(void *),
+					RTE_CACHE_LINE_SIZE, graph->socket);
+	RTE_VERIFY(node->objs);
+	node->size = size;
+	node->realloc_count++;
+}
+
+void __rte_noinline
+__rte_node_stream_alloc_size(struct rte_graph *graph, struct rte_node *node,
+			     uint16_t req_size)
+{
+	uint16_t size = node->size;
+
+	RTE_VERIFY(size != UINT16_MAX);
+	/* Allocate double amount of size to avoid immediate realloc */
+	size = RTE_MIN(UINT16_MAX, RTE_MAX(RTE_GRAPH_BURST_SIZE, req_size * 2));
+	node->objs = rte_realloc_socket(node->objs, size * sizeof(void *),
+					RTE_CACHE_LINE_SIZE, graph->socket);
+	RTE_VERIFY(node->objs);
+	node->size = size;
+	node->realloc_count++;
+}
+
+static int
+graph_to_dot(FILE *f, struct graph *graph)
+{
+	const char *src_edge_color = " [color=blue]\n";
+	const char *edge_color = "\n";
+	struct graph_node *graph_node;
+	char *node_name;
+	rte_edge_t i;
+	int rc;
+
+	rc = fprintf(f, "Digraph %s {\n\trankdir=LR;\n", graph->name);
+	if (rc < 0)
+		goto end;
+
+	STAILQ_FOREACH(graph_node, &graph->node_list, next) {
+		node_name = graph_node->node->name;
+		for (i = 0; i < graph_node->node->nb_edges; i++) {
+			rc = fprintf(f, "\t\"%s\"->\"%s\"%s", node_name,
+				     graph_node->adjacency_list[i]->node->name,
+				     graph_node->node->flags & RTE_NODE_SOURCE_F
+					     ? src_edge_color
+					     : edge_color);
+			if (rc < 0)
+				goto end;
+		}
+	}
+	rc = fprintf(f, "}\n");
+	if (rc < 0)
+		goto end;
+
+	return 0;
+end:
+	rte_errno = EBADF;
+	return -rte_errno;
+}
+
+int
+rte_graph_export(const char *name, FILE *f)
+{
+	struct graph *graph;
+	int rc = ENOENT;
+
+	STAILQ_FOREACH(graph, &graph_list, next) {
+		if (strncmp(graph->name, name, RTE_GRAPH_NAMESIZE) == 0) {
+			rc = graph_to_dot(f, graph);
+			goto end;
+		}
+	}
+end:
+	return -rc;
+}
+
+static void
+graph_scan_dump(FILE *f, rte_graph_t id, bool all)
+{
+	struct graph *graph;
+
+	RTE_VERIFY(f);
+	GRAPH_ID_CHECK(id);
+
+	STAILQ_FOREACH(graph, &graph_list, next) {
+		if (all == true) {
+			graph_dump(f, graph);
+		} else if (graph->id == id) {
+			graph_dump(f, graph);
+			return;
+		}
+	}
+fail:
+	return;
+}
+
+void
+rte_graph_dump(FILE *f, rte_graph_t id)
+{
+	graph_scan_dump(f, id, false);
+}
+
+void
+rte_graph_list_dump(FILE *f)
+{
+	graph_scan_dump(f, 0, true);
+}
+
+rte_graph_t
+rte_graph_max_count(void)
+{
+	return graph_id;
+}
+
+RTE_INIT(rte_graph_init_log)
+{
+	rte_graph_logtype = rte_log_register("lib.graph");
+	if (rte_graph_logtype >= 0)
+		rte_log_set_level(rte_graph_logtype, RTE_LOG_INFO);
+}
diff --git a/src/spdk/dpdk/lib/librte_graph/graph_debug.c b/src/spdk/dpdk/lib/librte_graph/graph_debug.c
new file mode 100644
index 000000000..f8aea16ac
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_graph/graph_debug.c
@@ -0,0 +1,84 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2020 Marvell International Ltd.
+ */
+
+#include <rte_common.h>
+#include <rte_debug.h>
+
+#include "graph_private.h"
+
+void
+graph_dump(FILE *f, struct graph *g)
+{
+	struct graph_node *graph_node;
+	rte_edge_t i = 0;
+
+	fprintf(f, "graph <%s>\n", g->name);
+	fprintf(f, "  id=%" PRIu32 "\n", g->id);
+	fprintf(f, "  cir_start=%" PRIu32 "\n", g->cir_start);
+	fprintf(f, "  cir_mask=%" PRIu32 "\n", g->cir_mask);
+	fprintf(f, "  addr=%p\n", g);
+	fprintf(f, "  graph=%p\n", g->graph);
+	fprintf(f, "  mem_sz=%zu\n", g->mem_sz);
+	fprintf(f, "  node_count=%" PRIu32 "\n", g->node_count);
+	fprintf(f, "  src_node_count=%" PRIu32 "\n", g->src_node_count);
+
+	STAILQ_FOREACH(graph_node, &g->node_list, next)
+		fprintf(f, "     node[%d] <%s>\n", i++, graph_node->node->name);
+}
+
+void
+node_dump(FILE *f, struct node *n)
+{
+	rte_edge_t i;
+
+	fprintf(f, "node <%s>\n", n->name);
+	fprintf(f, "  id=%" PRIu32 "\n", n->id);
+	fprintf(f, "  flags=0x%" PRIx64 "\n", n->flags);
+	fprintf(f, "  addr=%p\n", n);
+	fprintf(f, "  process=%p\n", n->process);
+	fprintf(f, "  nb_edges=%d\n", n->nb_edges);
+
+	for (i = 0; i < n->nb_edges; i++)
+		fprintf(f, "     edge[%d] <%s>\n", i, n->next_nodes[i]);
+}
+
+void
+rte_graph_obj_dump(FILE *f, struct rte_graph *g, bool all)
+{
+	rte_node_t count;
+	rte_graph_off_t off;
+	struct rte_node *n;
+	rte_edge_t i;
+
+	fprintf(f, "graph <%s> @ %p\n", g->name, g);
+	fprintf(f, "  id=%" PRIu32 "\n", g->id);
+	fprintf(f, "  head=%" PRId32 "\n", (int32_t)g->head);
+	fprintf(f, "  tail=%" PRId32 "\n", (int32_t)g->tail);
+	fprintf(f, "  cir_mask=0x%" PRIx32 "\n", g->cir_mask);
+	fprintf(f, "  nb_nodes=%" PRId32 "\n", g->nb_nodes);
+	fprintf(f, "  socket=%d\n", g->socket);
+	fprintf(f, "  fence=0x%" PRIx64 "\n", g->fence);
+	fprintf(f, "  nodes_start=0x%" PRIx32 "\n", g->nodes_start);
+	fprintf(f, "  cir_start=%p\n", g->cir_start);
+
+	rte_graph_foreach_node(count, off, g, n) {
+		if (!all && n->idx == 0)
+			continue;
+		fprintf(f, "     node[%d] <%s>\n", count, n->name);
+		fprintf(f, "       fence=0x%" PRIx64 "\n", n->fence);
+		fprintf(f, "       objs=%p\n", n->objs);
+		fprintf(f, "       process=%p\n", n->process);
+		fprintf(f, "       id=0x%" PRIx32 "\n", n->id);
+		fprintf(f, "       offset=0x%" PRIx32 "\n", n->off);
+		fprintf(f, "       nb_edges=%" PRId32 "\n", n->nb_edges);
+		fprintf(f, "       realloc_count=%d\n", n->realloc_count);
+		fprintf(f, "       size=%d\n", n->size);
+		fprintf(f, "       idx=%d\n", n->idx);
+		fprintf(f, "       total_objs=%" PRId64 "\n", n->total_objs);
+		fprintf(f, "       total_calls=%" PRId64 "\n", n->total_calls);
+		for (i = 0; i < n->nb_edges; i++)
+			fprintf(f, "          edge[%d] <%s>\n", i,
+				n->nodes[i]->name);
+	}
+}
diff --git a/src/spdk/dpdk/lib/librte_graph/graph_ops.c b/src/spdk/dpdk/lib/librte_graph/graph_ops.c
new file mode 100644
index 000000000..335595311
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_graph/graph_ops.c
@@ -0,0 +1,169 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2020 Marvell International Ltd.
+ */
+
+#include <stdbool.h>
+#include <string.h>
+
+#include <rte_common.h>
+#include <rte_errno.h>
+
+#include "graph_private.h"
+
+/* Check whether a node has next_node to itself */
+static inline int
+node_has_loop_edge(struct node *node)
+{
+	rte_edge_t i;
+	char *name;
+	int rc = 0;
+
+	for (i = 0; i < node->nb_edges; i++) {
+		if (strncmp(node->name, node->next_nodes[i],
+			    RTE_NODE_NAMESIZE) == 0) {
+			name = node->name;
+			rc = 1;
+			SET_ERR_JMP(EINVAL, fail, "Node %s has loop to self",
+				    name);
+		}
+	}
+fail:
+	return rc;
+}
+
+int
+graph_node_has_loop_edge(struct graph *graph)
+{
+	struct graph_node *graph_node;
+
+	STAILQ_FOREACH(graph_node, &graph->node_list, next)
+		if (node_has_loop_edge(graph_node->node))
+			return 1;
+
+	return 0;
+}
+
+rte_node_t
+graph_src_nodes_count(struct graph *graph)
+{
+	struct graph_node *graph_node;
+	rte_node_t rc = 0;
+
+	STAILQ_FOREACH(graph_node, &graph->node_list, next)
+		if (graph_node->node->flags & RTE_NODE_SOURCE_F)
+			rc++;
+
+	if (rc == 0)
+		SET_ERR_JMP(EINVAL, fail, "Graph needs at least a source node");
+fail:
+	return rc;
+}
+
+/* Check whether a node has next_node to a source node */
+int
+graph_node_has_edge_to_src_node(struct graph *graph)
+{
+	struct graph_node *graph_node;
+	struct node *node;
+	rte_edge_t i;
+
+	STAILQ_FOREACH(graph_node, &graph->node_list, next) {
+		for (i = 0; i < graph_node->node->nb_edges; i++) {
+			node = graph_node->adjacency_list[i]->node;
+			if (node->flags & RTE_NODE_SOURCE_F)
+				SET_ERR_JMP(
+					EEXIST, fail,
+					"Node %s points to the source node %s",
+					graph_node->node->name, node->name);
+		}
+	}
+
+	return 0;
+fail:
+	return 1;
+}
+
+rte_node_t
+graph_nodes_count(struct graph *graph)
+{
+	struct graph_node *graph_node;
+	rte_node_t count = 0;
+
+	STAILQ_FOREACH(graph_node, &graph->node_list, next)
+		count++;
+
+	return count;
+}
+
+void
+graph_mark_nodes_as_not_visited(struct graph *graph)
+{
+	struct graph_node *graph_node;
+
+	STAILQ_FOREACH(graph_node, &graph->node_list, next)
+		graph_node->visited = false;
+}
+
+int
+graph_bfs(struct graph *graph, struct graph_node *start)
+{
+	struct graph_node **queue, *v, *tmp;
+	uint16_t head = 0, tail = 0;
+	rte_edge_t i;
+	size_t sz;
+
+	sz = sizeof(struct graph_node *) * graph_nodes_count(graph);
+	queue = malloc(sz);
+	if (queue == NULL)
+		SET_ERR_JMP(ENOMEM, fail, "Failed to alloc BFS queue of %zu",
+			    sz);
+
+	/* BFS algorithm */
+	queue[tail++] = start;
+	start->visited = true;
+	while (head != tail) {
+		v = queue[head++];
+		for (i = 0; i < v->node->nb_edges; i++) {
+			tmp = v->adjacency_list[i];
+			if (tmp->visited == false) {
+				queue[tail++] = tmp;
+				tmp->visited = true;
+			}
+		}
+	}
+
+	free(queue);
+	return 0;
+
+fail:
+	return -rte_errno;
+}
+
+/* Check whether a node has connected path or parent node */
+int
+graph_has_isolated_node(struct graph *graph)
+{
+	struct graph_node *graph_node;
+
+	graph_mark_nodes_as_not_visited(graph);
+
+	STAILQ_FOREACH(graph_node, &graph->node_list, next) {
+		if (graph_node->node->flags & RTE_NODE_SOURCE_F) {
+			if (graph_node->node->nb_edges == 0)
+				SET_ERR_JMP(EINVAL, fail,
+					    "%s node needs minimum one edge",
+					    graph_node->node->name);
+			if (graph_bfs(graph, graph_node))
+				goto fail;
+		}
+	}
+
+	STAILQ_FOREACH(graph_node, &graph->node_list, next)
+		if (graph_node->visited == false)
+			SET_ERR_JMP(EINVAL, fail, "Found isolated node %s",
+				    graph_node->node->name);
+
+	return 0;
+fail:
+	return 1;
+}
diff --git a/src/spdk/dpdk/lib/librte_graph/graph_populate.c b/src/spdk/dpdk/lib/librte_graph/graph_populate.c
new file mode 100644
index 000000000..093512efa
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_graph/graph_populate.c
@@ -0,0 +1,234 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2020 Marvell International Ltd.
+ */
+
+#include <fnmatch.h>
+#include <stdbool.h>
+
+#include <rte_common.h>
+#include <rte_errno.h>
+#include <rte_malloc.h>
+#include <rte_memzone.h>
+
+#include "graph_private.h"
+
+static size_t
+graph_fp_mem_calc_size(struct graph *graph)
+{
+	struct graph_node *graph_node;
+	rte_node_t val;
+	size_t sz;
+
+	/* Graph header */
+	sz = sizeof(struct rte_graph);
+	/* Source nodes list */
+	sz += sizeof(rte_graph_off_t) * graph->src_node_count;
+	/* Circular buffer for pending streams of size number of nodes */
+	val = rte_align32pow2(graph->node_count * sizeof(rte_graph_off_t));
+	sz = RTE_ALIGN(sz, val);
+	graph->cir_start = sz;
+	graph->cir_mask = rte_align32pow2(graph->node_count) - 1;
+	sz += val;
+	/* Fence */
+	sz += sizeof(RTE_GRAPH_FENCE);
+	sz = RTE_ALIGN(sz, RTE_CACHE_LINE_SIZE);
+	graph->nodes_start = sz;
+	/* For 0..N node objects with fence */
+	STAILQ_FOREACH(graph_node, &graph->node_list, next) {
+		sz = RTE_ALIGN(sz, RTE_CACHE_LINE_SIZE);
+		sz += sizeof(struct rte_node);
+		/* Pointer to next nodes(edges) */
+		sz += sizeof(struct rte_node *) * graph_node->node->nb_edges;
+	}
+
+	graph->mem_sz = sz;
+	return sz;
+}
+
+static void
+graph_header_popluate(struct graph *_graph)
+{
+	struct rte_graph *graph = _graph->graph;
+
+	graph->tail = 0;
+	graph->head = (int32_t)-_graph->src_node_count;
+	graph->cir_mask = _graph->cir_mask;
+	graph->nb_nodes = _graph->node_count;
+	graph->cir_start = RTE_PTR_ADD(graph, _graph->cir_start);
+	graph->nodes_start = _graph->nodes_start;
+	graph->socket = _graph->socket;
+	graph->id = _graph->id;
+	memcpy(graph->name, _graph->name, RTE_GRAPH_NAMESIZE);
+	graph->fence = RTE_GRAPH_FENCE;
+}
+
+static void
+graph_nodes_populate(struct graph *_graph)
+{
+	rte_graph_off_t off = _graph->nodes_start;
+	struct rte_graph *graph = _graph->graph;
+	struct graph_node *graph_node;
+	rte_edge_t count, nb_edges;
+	const char *parent;
+	rte_node_t pid;
+
+	STAILQ_FOREACH(graph_node, &_graph->node_list, next) {
+		struct rte_node *node = RTE_PTR_ADD(graph, off);
+		memset(node, 0, sizeof(*node));
+		node->fence = RTE_GRAPH_FENCE;
+		node->off = off;
+		node->process = graph_node->node->process;
+		memcpy(node->name, graph_node->node->name, RTE_GRAPH_NAMESIZE);
+		pid = graph_node->node->parent_id;
+		if (pid != RTE_NODE_ID_INVALID) { /* Cloned node */
+			parent = rte_node_id_to_name(pid);
+			memcpy(node->parent, parent, RTE_GRAPH_NAMESIZE);
+		}
+		node->id = graph_node->node->id;
+		node->parent_id = pid;
+		nb_edges = graph_node->node->nb_edges;
+		node->nb_edges = nb_edges;
+		off += sizeof(struct rte_node);
+		/* Copy the name in first pass to replace with rte_node* later*/
+		for (count = 0; count < nb_edges; count++)
+			node->nodes[count] = (struct rte_node *)&graph_node
+						     ->adjacency_list[count]
+						     ->node->name[0];
+
+		off += sizeof(struct rte_node *) * nb_edges;
+		off = RTE_ALIGN(off, RTE_CACHE_LINE_SIZE);
+		node->next = off;
+		__rte_node_stream_alloc(graph, node);
+	}
+}
+
+struct rte_node *
+graph_node_id_to_ptr(const struct rte_graph *graph, rte_node_t id)
+{
+	rte_node_t count;
+	rte_graph_off_t off;
+	struct rte_node *node;
+
+	rte_graph_foreach_node(count, off, graph, node)
+		if (unlikely(node->id == id))
+			return node;
+
+	return NULL;
+}
+
+struct rte_node *
+graph_node_name_to_ptr(const struct rte_graph *graph, const char *name)
+{
+	rte_node_t count;
+	rte_graph_off_t off;
+	struct rte_node *node;
+
+	rte_graph_foreach_node(count, off, graph, node)
+		if (strncmp(name, node->name, RTE_NODE_NAMESIZE) == 0)
+			return node;
+
+	return NULL;
+}
+
+static int
+graph_node_nexts_populate(struct graph *_graph)
+{
+	rte_node_t count, val;
+	rte_graph_off_t off;
+	struct rte_node *node;
+	const struct rte_graph *graph = _graph->graph;
+	const char *name;
+
+	rte_graph_foreach_node(count, off, graph, node) {
+		for (val = 0; val < node->nb_edges; val++) {
+			name = (const char *)node->nodes[val];
+			node->nodes[val] = graph_node_name_to_ptr(graph, name);
+			if (node->nodes[val] == NULL)
+				SET_ERR_JMP(EINVAL, fail, "%s not found", name);
+		}
+	}
+
+	return 0;
+fail:
+	return -rte_errno;
+}
+
+static int
+graph_src_nodes_populate(struct graph *_graph)
+{
+	struct rte_graph *graph = _graph->graph;
+	struct graph_node *graph_node;
+	struct rte_node *node;
+	int32_t head = -1;
+	const char *name;
+
+	STAILQ_FOREACH(graph_node, &_graph->node_list, next) {
+		if (graph_node->node->flags & RTE_NODE_SOURCE_F) {
+			name = graph_node->node->name;
+			node = graph_node_name_to_ptr(graph, name);
+			if (node == NULL)
+				SET_ERR_JMP(EINVAL, fail, "%s not found", name);
+
+			__rte_node_stream_alloc(graph, node);
+			graph->cir_start[head--] = node->off;
+		}
+	}
+
+	return 0;
+fail:
+	return -rte_errno;
+}
+
+static int
+graph_fp_mem_populate(struct graph *graph)
+{
+	int rc;
+
+	graph_header_popluate(graph);
+	graph_nodes_populate(graph);
+	rc = graph_node_nexts_populate(graph);
+	rc |= graph_src_nodes_populate(graph);
+
+	return rc;
+}
+
+int
+graph_fp_mem_create(struct graph *graph)
+{
+	const struct rte_memzone *mz;
+	size_t sz;
+
+	sz = graph_fp_mem_calc_size(graph);
+	mz = rte_memzone_reserve(graph->name, sz, graph->socket, 0);
+	if (mz == NULL)
+		SET_ERR_JMP(ENOMEM, fail, "Memzone %s reserve failed",
+			    graph->name);
+
+	graph->graph = mz->addr;
+	graph->mz = mz;
+
+	return graph_fp_mem_populate(graph);
+fail:
+	return -rte_errno;
+}
+
+static void
+graph_nodes_mem_destroy(struct rte_graph *graph)
+{
+	rte_node_t count;
+	rte_graph_off_t off;
+	struct rte_node *node;
+
+	if (graph == NULL)
+		return;
+
+	rte_graph_foreach_node(count, off, graph, node)
+		rte_free(node->objs);
+}
+
+int
+graph_fp_mem_destroy(struct graph *graph)
+{
+	graph_nodes_mem_destroy(graph->graph);
+	return rte_memzone_free(graph->mz);
+}
diff --git a/src/spdk/dpdk/lib/librte_graph/graph_private.h b/src/spdk/dpdk/lib/librte_graph/graph_private.h
new file mode 100644
index 000000000..f9a85c892
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_graph/graph_private.h
@@ -0,0 +1,347 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2020 Marvell International Ltd.
+ */
+
+#ifndef _RTE_GRAPH_PRIVATE_H_
+#define _RTE_GRAPH_PRIVATE_H_
+
+#include <inttypes.h>
+#include <sys/queue.h>
+
+#include <rte_common.h>
+#include <rte_eal.h>
+
+#include "rte_graph.h"
+#include "rte_graph_worker.h"
+
+extern int rte_graph_logtype;
+
+#define GRAPH_LOG(level, ...)                                                  \
+	rte_log(RTE_LOG_##level, rte_graph_logtype,                            \
+		RTE_FMT("GRAPH: %s():%u " RTE_FMT_HEAD(__VA_ARGS__, ) "\n",    \
+			__func__, __LINE__, RTE_FMT_TAIL(__VA_ARGS__, )))
+
+#define graph_err(...) GRAPH_LOG(ERR, __VA_ARGS__)
+#define graph_info(...) GRAPH_LOG(INFO, __VA_ARGS__)
+#define graph_dbg(...) GRAPH_LOG(DEBUG, __VA_ARGS__)
+
+#define ID_CHECK(id, id_max)                                                   \
+	do {                                                                   \
+		if ((id) >= (id_max)) {                                        \
+			rte_errno = EINVAL;                                    \
+			goto fail;                                             \
+		}                                                              \
+	} while (0)
+
+#define SET_ERR_JMP(err, where, fmt, ...)                                      \
+	do {                                                                   \
+		graph_err(fmt, ##__VA_ARGS__);                                 \
+		rte_errno = err;                                               \
+		goto where;                                                    \
+	} while (0)
+
+/**
+ * @internal
+ *
+ * Structure that holds node internal data.
+ */
+struct node {
+	STAILQ_ENTRY(node) next;      /**< Next node in the list. */
+	char name[RTE_NODE_NAMESIZE]; /**< Name of the node. */
+	uint64_t flags;		      /**< Node configuration flag. */
+	rte_node_process_t process;   /**< Node process function. */
+	rte_node_init_t init;         /**< Node init function. */
+	rte_node_fini_t fini;	      /**< Node fini function. */
+	rte_node_t id;		      /**< Allocated identifier for the node. */
+	rte_node_t parent_id;	      /**< Parent node identifier. */
+	rte_edge_t nb_edges;	      /**< Number of edges from this node. */
+	char next_nodes[][RTE_NODE_NAMESIZE]; /**< Names of next nodes. */
+};
+
+/**
+ * @internal
+ *
+ * Structure that holds the graph node data.
+ */
+struct graph_node {
+	STAILQ_ENTRY(graph_node) next; /**< Next graph node in the list. */
+	struct node *node; /**< Pointer to internal node. */
+	bool visited;      /**< Flag used in BFS to mark node visited. */
+	struct graph_node *adjacency_list[]; /**< Adjacency list of the node. */
+};
+
+/**
+ * @internal
+ *
+ * Structure that holds graph internal data.
+ */
+struct graph {
+	STAILQ_ENTRY(graph) next;
+	/**< List of graphs. */
+	char name[RTE_GRAPH_NAMESIZE];
+	/**< Name of the graph. */
+	const struct rte_memzone *mz;
+	/**< Memzone to store graph data. */
+	rte_graph_off_t nodes_start;
+	/**< Node memory start offset in graph reel. */
+	rte_node_t src_node_count;
+	/**< Number of source nodes in a graph. */
+	struct rte_graph *graph;
+	/**< Pointer to graph data. */
+	rte_node_t node_count;
+	/**< Total number of nodes. */
+	uint32_t cir_start;
+	/**< Circular buffer start offset in graph reel. */
+	uint32_t cir_mask;
+	/**< Circular buffer mask for wrap around. */
+	rte_graph_t id;
+	/**< Graph identifier. */
+	size_t mem_sz;
+	/**< Memory size of the graph. */
+	int socket;
+	/**< Socket identifier where memory is allocated. */
+	STAILQ_HEAD(gnode_list, graph_node) node_list;
+	/**< Nodes in a graph. */
+};
+
+/* Node functions */
+STAILQ_HEAD(node_head, node);
+
+/**
+ * @internal
+ *
+ * Get the head of the node list.
+ *
+ * @return
+ *   Pointer to the node head.
+ */
+struct node_head *node_list_head_get(void);
+
+/**
+ * @internal
+ *
+ * Get node pointer from node name.
+ *
+ * @param name
+ *   Pointer to character string containing the node name.
+ *
+ * @return
+ *   Pointer to the node.
+ */
+struct node *node_from_name(const char *name);
+
+/* Graph list functions */
+STAILQ_HEAD(graph_head, graph);
+
+/**
+ * @internal
+ *
+ * Get the head of the graph list.
+ *
+ * @return
+ *   Pointer to the graph head.
+ */
+struct graph_head *graph_list_head_get(void);
+
+/* Lock functions */
+/**
+ * @internal
+ *
+ * Take a lock on the graph internal spin lock.
+ */
+void graph_spinlock_lock(void);
+
+/**
+ * @internal
+ *
+ * Release a lock on the graph internal spin lock.
+ */
+void graph_spinlock_unlock(void);
+
+/* Graph operations */
+/**
+ * @internal
+ *
+ * Run a BFS(Breadth First Search) on the graph marking all
+ * the graph nodes as visited.
+ *
+ * @param graph
+ *   Pointer to the internal graph object.
+ * @param start
+ *   Pointer to the starting graph node.
+ *
+ * @return
+ *   - 0: Success.
+ *   - -ENOMEM: Not enough memory for BFS.
+ */
+int graph_bfs(struct graph *graph, struct graph_node *start);
+
+/**
+ * @internal
+ *
+ * Check if there is an isolated node in the given graph.
+ *
+ * @param graph
+ *   Pointer to the internal graph object.
+ *
+ * @return
+ *   - 0: No isolated node found.
+ *   - 1: Isolated node found.
+ */
+int graph_has_isolated_node(struct graph *graph);
+
+/**
+ * @internal
+ *
+ * Check whether a node in the graph has next_node to a source node.
+ *
+ * @param graph
+ *   Pointer to the internal graph object.
+ *
+ * @return
+ *   - 0: Node has an edge to source node.
+ *   - 1: Node doesn't have an edge to source node.
+ */
+int graph_node_has_edge_to_src_node(struct graph *graph);
+
+/**
+ * @internal
+ *
+ * Checks whether node in the graph has a edge to itself i.e. forms a
+ * loop.
+ *
+ * @param graph
+ *   Pointer to the internal graph object.
+ *
+ * @return
+ *   - 0: Node has an edge to itself.
+ *   - 1: Node doesn't have an edge to itself.
+ */
+int graph_node_has_loop_edge(struct graph *graph);
+
+/**
+ * @internal
+ *
+ * Get the count of source nodes in the graph.
+ *
+ * @param graph
+ *   Pointer to the internal graph object.
+ *
+ * @return
+ *   Number of source nodes.
+ */
+rte_node_t graph_src_nodes_count(struct graph *graph);
+
+/**
+ * @internal
+ *
+ * Get the count of total number of nodes in the graph.
+ *
+ * @param graph
+ *   Pointer to the internal graph object.
+ *
+ * @return
+ *   Number of nodes.
+ */
+rte_node_t graph_nodes_count(struct graph *graph);
+
+/**
+ * @internal
+ *
+ * Clear the visited flag of all the nodes in the graph.
+ *
+ * @param graph
+ *   Pointer to the internal graph object.
+ */
+void graph_mark_nodes_as_not_visited(struct graph *graph);
+
+/* Fast path graph memory populate unctions */
+
+/**
+ * @internal
+ *
+ * Create fast-path memory for the graph and nodes.
+ *
+ * @param graph
+ *   Pointer to the internal graph object.
+ *
+ * @return
+ *   - 0: Success.
+ *   - -ENOMEM: Not enough for graph and nodes.
+ *   - -EINVAL: Graph nodes not found.
+ */
+int graph_fp_mem_create(struct graph *graph);
+
+/**
+ * @internal
+ *
+ * Free fast-path memory used by graph and nodes.
+ *
+ * @param graph
+ *   Pointer to the internal graph object.
+ *
+ * @return
+ *   - 0: Success.
+ *   - <0: Graph memzone related error.
+ */
+int graph_fp_mem_destroy(struct graph *graph);
+
+/* Lookup functions */
+/**
+ * @internal
+ *
+ * Get graph node object from node id.
+ *
+ * @param graph
+ *   Pointer to rte_graph object.
+ * @param id
+ *   Node Identifier.
+ *
+ * @return
+ *   Pointer to rte_node if identifier is valid else NULL.
+ */
+struct rte_node *graph_node_id_to_ptr(const struct rte_graph *graph,
+				      rte_node_t id);
+
+/**
+ * @internal
+ *
+ * Get graph node object from node name.
+ *
+ * @param graph
+ *   Pointer to rte_graph object.
+ * @param node_name
+ *   Pointer to character string holding the node name.
+ *
+ * @return
+ *   Pointer to rte_node if identifier is valid else NULL.
+ */
+struct rte_node *graph_node_name_to_ptr(const struct rte_graph *graph,
+					const char *node_name);
+
+/* Debug functions */
+/**
+ * @internal
+ *
+ * Dump internal graph object data.
+ *
+ * @param f
+ *   FILE pointer to dump the data.
+ * @param g
+ *   Pointer to the internal graph object.
+ */
+void graph_dump(FILE *f, struct graph *g);
+
+/**
+ * @internal
+ *
+ * Dump internal node object data.
+ *
+ * @param f
+ *   FILE pointer to dump the info.
+ * @param g
+ *   Pointer to the internal node object.
+ */
+void node_dump(FILE *f, struct node *n);
+
+#endif /* _RTE_GRAPH_PRIVATE_H_ */
diff --git a/src/spdk/dpdk/lib/librte_graph/graph_stats.c b/src/spdk/dpdk/lib/librte_graph/graph_stats.c
new file mode 100644
index 000000000..125e08d73
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_graph/graph_stats.c
@@ -0,0 +1,406 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2020 Marvell International Ltd.
+ */
+
+#include <fnmatch.h>
+#include <stdbool.h>
+
+#include <rte_common.h>
+#include <rte_errno.h>
+#include <rte_malloc.h>
+
+#include "graph_private.h"
+
+/* Capture all graphs of cluster */
+struct cluster {
+	rte_graph_t nb_graphs;
+	rte_graph_t size;
+
+	struct graph **graphs;
+};
+
+/* Capture same node ID across cluster  */
+struct cluster_node {
+	struct rte_graph_cluster_node_stats stat;
+	rte_node_t nb_nodes;
+
+	struct rte_node *nodes[];
+};
+
+struct rte_graph_cluster_stats {
+	/* Header */
+	rte_graph_cluster_stats_cb_t fn;
+	uint32_t cluster_node_size; /* Size of struct cluster_node */
+	rte_node_t max_nodes;
+	int socket_id;
+	void *cookie;
+	size_t sz;
+
+	struct cluster_node clusters[];
+} __rte_cache_aligned;
+
+#define boarder()                                                              \
+	fprintf(f, "+-------------------------------+---------------+--------" \
+		   "-------+---------------+---------------+---------------+-" \
+		   "----------+\n")
+
+static inline void
+print_banner(FILE *f)
+{
+	boarder();
+	fprintf(f, "%-32s%-16s%-16s%-16s%-16s%-16s%-16s\n", "|Node", "|calls",
+		"|objs", "|realloc_count", "|objs/call", "|objs/sec(10E6)",
+		"|cycles/call|");
+	boarder();
+}
+
+static inline void
+print_node(FILE *f, const struct rte_graph_cluster_node_stats *stat)
+{
+	double objs_per_call, objs_per_sec, cycles_per_call, ts_per_hz;
+	const uint64_t prev_calls = stat->prev_calls;
+	const uint64_t prev_objs = stat->prev_objs;
+	const uint64_t cycles = stat->cycles;
+	const uint64_t calls = stat->calls;
+	const uint64_t objs = stat->objs;
+	uint64_t call_delta;
+
+	call_delta = calls - prev_calls;
+	objs_per_call =
+		call_delta ? (double)((objs - prev_objs) / call_delta) : 0;
+	cycles_per_call =
+		call_delta ? (double)((cycles - stat->prev_cycles) / call_delta)
+			   : 0;
+	ts_per_hz = (double)((stat->ts - stat->prev_ts) / stat->hz);
+	objs_per_sec = ts_per_hz ? (objs - prev_objs) / ts_per_hz : 0;
+	objs_per_sec /= 1000000;
+
+	fprintf(f,
+		"|%-31s|%-15" PRIu64 "|%-15" PRIu64 "|%-15" PRIu64
+		"|%-15.3f|%-15.6f|%-11.4f|\n",
+		stat->name, calls, objs, stat->realloc_count, objs_per_call,
+		objs_per_sec, cycles_per_call);
+}
+
+static int
+graph_cluster_stats_cb(bool is_first, bool is_last, void *cookie,
+		       const struct rte_graph_cluster_node_stats *stat)
+{
+	FILE *f = cookie;
+
+	if (unlikely(is_first))
+		print_banner(f);
+	if (stat->objs)
+		print_node(f, stat);
+	if (unlikely(is_last))
+		boarder();
+
+	return 0;
+};
+
+static struct rte_graph_cluster_stats *
+stats_mem_init(struct cluster *cluster,
+	       const struct rte_graph_cluster_stats_param *prm)
+{
+	size_t sz = sizeof(struct rte_graph_cluster_stats);
+	struct rte_graph_cluster_stats *stats;
+	rte_graph_cluster_stats_cb_t fn;
+	int socket_id = prm->socket_id;
+	uint32_t cluster_node_size;
+
+	/* Fix up callback */
+	fn = prm->fn;
+	if (fn == NULL)
+		fn = graph_cluster_stats_cb;
+
+	cluster_node_size = sizeof(struct cluster_node);
+	/* For a given cluster, max nodes will be the max number of graphs */
+	cluster_node_size += cluster->nb_graphs * sizeof(struct rte_node *);
+	cluster_node_size = RTE_ALIGN(cluster_node_size, RTE_CACHE_LINE_SIZE);
+
+	stats = realloc(NULL, sz);
+	memset(stats, 0, sz);
+	if (stats) {
+		stats->fn = fn;
+		stats->cluster_node_size = cluster_node_size;
+		stats->max_nodes = 0;
+		stats->socket_id = socket_id;
+		stats->cookie = prm->cookie;
+		stats->sz = sz;
+	}
+
+	return stats;
+}
+
+static int
+stats_mem_populate(struct rte_graph_cluster_stats **stats_in,
+		   struct rte_graph *graph, struct graph_node *graph_node)
+{
+	struct rte_graph_cluster_stats *stats = *stats_in;
+	rte_node_t id = graph_node->node->id;
+	struct cluster_node *cluster;
+	struct rte_node *node;
+	rte_node_t count;
+
+	cluster = stats->clusters;
+
+	/* Iterate over cluster node array to find node ID match */
+	for (count = 0; count < stats->max_nodes; count++) {
+		/* Found an existing node in the reel */
+		if (cluster->stat.id == id) {
+			node = graph_node_id_to_ptr(graph, id);
+			if (node == NULL)
+				SET_ERR_JMP(
+					ENOENT, err,
+					"Failed to find node %s in graph %s",
+					graph_node->node->name, graph->name);
+
+			cluster->nodes[cluster->nb_nodes++] = node;
+			return 0;
+		}
+		cluster = RTE_PTR_ADD(cluster, stats->cluster_node_size);
+	}
+
+	/* Hey, it is a new node, allocate space for it in the reel */
+	stats = realloc(stats, stats->sz + stats->cluster_node_size);
+	if (stats == NULL)
+		SET_ERR_JMP(ENOMEM, err, "Realloc failed");
+
+	/* Clear the new struct cluster_node area */
+	cluster = RTE_PTR_ADD(stats, stats->sz),
+	memset(cluster, 0, stats->cluster_node_size);
+	memcpy(cluster->stat.name, graph_node->node->name, RTE_NODE_NAMESIZE);
+	cluster->stat.id = graph_node->node->id;
+	cluster->stat.hz = rte_get_timer_hz();
+	node = graph_node_id_to_ptr(graph, id);
+	if (node == NULL)
+		SET_ERR_JMP(ENOENT, err, "Failed to find node %s in graph %s",
+			    graph_node->node->name, graph->name);
+	cluster->nodes[cluster->nb_nodes++] = node;
+
+	stats->sz += stats->cluster_node_size;
+	stats->max_nodes++;
+	*stats_in = stats;
+
+	return 0;
+err:
+	return -rte_errno;
+}
+
+static void
+stats_mem_fini(struct rte_graph_cluster_stats *stats)
+{
+	free(stats);
+}
+
+static void
+cluster_init(struct cluster *cluster)
+{
+	memset(cluster, 0, sizeof(*cluster));
+}
+
+static int
+cluster_add(struct cluster *cluster, struct graph *graph)
+{
+	rte_graph_t count;
+	size_t sz;
+
+	/* Skip the if graph is already added to cluster */
+	for (count = 0; count < cluster->nb_graphs; count++)
+		if (cluster->graphs[count] == graph)
+			return 0;
+
+	/* Expand the cluster if required to store graph objects */
+	if (cluster->nb_graphs + 1 > cluster->size) {
+		cluster->size = RTE_MAX(1, cluster->size * 2);
+		sz = sizeof(struct graph *) * cluster->size;
+		cluster->graphs = realloc(cluster->graphs, sz);
+		if (cluster->graphs == NULL)
+			SET_ERR_JMP(ENOMEM, free, "Failed to realloc");
+	}
+
+	/* Add graph to cluster */
+	cluster->graphs[cluster->nb_graphs++] = graph;
+	return 0;
+
+free:
+	return -rte_errno;
+}
+
+static void
+cluster_fini(struct cluster *cluster)
+{
+	if (cluster->graphs)
+		free(cluster->graphs);
+}
+
+static int
+expand_pattern_to_cluster(struct cluster *cluster, const char *pattern)
+{
+	struct graph_head *graph_head = graph_list_head_get();
+	struct graph *graph;
+	bool found = false;
+
+	/* Check for pattern match */
+	STAILQ_FOREACH(graph, graph_head, next) {
+		if (fnmatch(pattern, graph->name, 0) == 0) {
+			if (cluster_add(cluster, graph))
+				goto fail;
+			found = true;
+		}
+	}
+	if (found == false)
+		SET_ERR_JMP(EFAULT, fail, "Pattern %s graph not found",
+			    pattern);
+
+	return 0;
+fail:
+	return -rte_errno;
+}
+
+struct rte_graph_cluster_stats *
+rte_graph_cluster_stats_create(const struct rte_graph_cluster_stats_param *prm)
+{
+	struct rte_graph_cluster_stats *stats, *rc = NULL;
+	struct graph_node *graph_node;
+	struct cluster cluster;
+	struct graph *graph;
+	const char *pattern;
+	rte_graph_t i;
+
+	/* Sanity checks */
+	if (!rte_graph_has_stats_feature())
+		SET_ERR_JMP(EINVAL, fail, "Stats feature is not enabled");
+
+	if (prm == NULL)
+		SET_ERR_JMP(EINVAL, fail, "Invalid param");
+
+	if (prm->graph_patterns == NULL || prm->nb_graph_patterns == 0)
+		SET_ERR_JMP(EINVAL, fail, "Invalid graph param");
+
+	cluster_init(&cluster);
+
+	graph_spinlock_lock();
+	/* Expand graph pattern and add the graph to the cluster */
+	for (i = 0; i < prm->nb_graph_patterns; i++) {
+		pattern = prm->graph_patterns[i];
+		if (expand_pattern_to_cluster(&cluster, pattern))
+			goto bad_pattern;
+	}
+
+	/* Alloc the stats memory */
+	stats = stats_mem_init(&cluster, prm);
+	if (stats == NULL)
+		SET_ERR_JMP(ENOMEM, bad_pattern, "Failed alloc stats memory");
+
+	/* Iterate over M(Graph) x N (Nodes in graph) */
+	for (i = 0; i < cluster.nb_graphs; i++) {
+		graph = cluster.graphs[i];
+		STAILQ_FOREACH(graph_node, &graph->node_list, next) {
+			struct rte_graph *graph_fp = graph->graph;
+			if (stats_mem_populate(&stats, graph_fp, graph_node))
+				goto realloc_fail;
+		}
+	}
+
+	/* Finally copy to hugepage memory to avoid pressure on rte_realloc */
+	rc = rte_malloc_socket(NULL, stats->sz, 0, stats->socket_id);
+	if (rc)
+		rte_memcpy(rc, stats, stats->sz);
+	else
+		SET_ERR_JMP(ENOMEM, realloc_fail, "rte_malloc failed");
+
+realloc_fail:
+	stats_mem_fini(stats);
+bad_pattern:
+	graph_spinlock_unlock();
+	cluster_fini(&cluster);
+fail:
+	return rc;
+}
+
+void
+rte_graph_cluster_stats_destroy(struct rte_graph_cluster_stats *stat)
+{
+	return rte_free(stat);
+}
+
+static inline void
+cluster_node_arregate_stats(struct cluster_node *cluster)
+{
+	uint64_t calls = 0, cycles = 0, objs = 0, realloc_count = 0;
+	struct rte_graph_cluster_node_stats *stat = &cluster->stat;
+	struct rte_node *node;
+	rte_node_t count;
+
+	for (count = 0; count < cluster->nb_nodes; count++) {
+		node = cluster->nodes[count];
+
+		calls += node->total_calls;
+		objs += node->total_objs;
+		cycles += node->total_cycles;
+		realloc_count += node->realloc_count;
+	}
+
+	stat->calls = calls;
+	stat->objs = objs;
+	stat->cycles = cycles;
+	stat->ts = rte_get_timer_cycles();
+	stat->realloc_count = realloc_count;
+}
+
+static inline void
+cluster_node_store_prev_stats(struct cluster_node *cluster)
+{
+	struct rte_graph_cluster_node_stats *stat = &cluster->stat;
+
+	stat->prev_ts = stat->ts;
+	stat->prev_calls = stat->calls;
+	stat->prev_objs = stat->objs;
+	stat->prev_cycles = stat->cycles;
+}
+
+void
+rte_graph_cluster_stats_get(struct rte_graph_cluster_stats *stat, bool skip_cb)
+{
+	struct cluster_node *cluster;
+	rte_node_t count;
+	int rc = 0;
+
+	cluster = stat->clusters;
+
+	for (count = 0; count < stat->max_nodes; count++) {
+		cluster_node_arregate_stats(cluster);
+		if (!skip_cb)
+			rc = stat->fn(!count, (count == stat->max_nodes - 1),
+				      stat->cookie, &cluster->stat);
+		cluster_node_store_prev_stats(cluster);
+		if (rc)
+			break;
+		cluster = RTE_PTR_ADD(cluster, stat->cluster_node_size);
+	}
+}
+
+void
+rte_graph_cluster_stats_reset(struct rte_graph_cluster_stats *stat)
+{
+	struct cluster_node *cluster;
+	rte_node_t count;
+
+	cluster = stat->clusters;
+
+	for (count = 0; count < stat->max_nodes; count++) {
+		struct rte_graph_cluster_node_stats *node = &cluster->stat;
+
+		node->ts = 0;
+		node->calls = 0;
+		node->objs = 0;
+		node->cycles = 0;
+		node->prev_ts = 0;
+		node->prev_calls = 0;
+		node->prev_objs = 0;
+		node->prev_cycles = 0;
+		node->realloc_count = 0;
+		cluster = RTE_PTR_ADD(cluster, stat->cluster_node_size);
+	}
+}
diff --git a/src/spdk/dpdk/lib/librte_graph/meson.build b/src/spdk/dpdk/lib/librte_graph/meson.build
new file mode 100644
index 000000000..8fa18b611
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_graph/meson.build
@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(C) 2020 Marvell International Ltd.
+
+name = 'graph'
+
+sources = files('node.c', 'graph.c', 'graph_ops.c', 'graph_debug.c', 'graph_stats.c', 'graph_populate.c')
+headers = files('rte_graph.h', 'rte_graph_worker.h')
+
+deps += ['eal']
+build = false
+reason = 'not needed by SPDK'
diff --git a/src/spdk/dpdk/lib/librte_graph/node.c b/src/spdk/dpdk/lib/librte_graph/node.c
new file mode 100644
index 000000000..873c9ab16
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_graph/node.c
@@ -0,0 +1,421 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2020 Marvell International Ltd.
+ */
+
+#include <stdbool.h>
+#include <stdio.h>
+#include <string.h>
+
+#include <rte_common.h>
+#include <rte_debug.h>
+#include <rte_eal.h>
+#include <rte_errno.h>
+#include <rte_string_fns.h>
+
+#include "graph_private.h"
+
+static struct node_head node_list = STAILQ_HEAD_INITIALIZER(node_list);
+static rte_node_t node_id;
+
+#define NODE_ID_CHECK(id) ID_CHECK(id, node_id)
+
+/* Private functions */
+struct node_head *
+node_list_head_get(void)
+{
+	return &node_list;
+}
+
+struct node *
+node_from_name(const char *name)
+{
+	struct node *node;
+
+	STAILQ_FOREACH(node, &node_list, next)
+		if (strncmp(node->name, name, RTE_NODE_NAMESIZE) == 0)
+			return node;
+
+	return NULL;
+}
+
+static bool
+node_has_duplicate_entry(const char *name)
+{
+	struct node *node;
+
+	/* Is duplicate name registered */
+	STAILQ_FOREACH(node, &node_list, next) {
+		if (strncmp(node->name, name, RTE_NODE_NAMESIZE) == 0) {
+			rte_errno = EEXIST;
+			return 1;
+		}
+	}
+	return 0;
+}
+
+/* Public functions */
+rte_node_t
+__rte_node_register(const struct rte_node_register *reg)
+{
+	struct node *node;
+	rte_edge_t i;
+	size_t sz;
+
+	/* Limit Node specific metadata to one cacheline on 64B CL machine */
+	RTE_BUILD_BUG_ON((offsetof(struct rte_node, nodes) -
+			  offsetof(struct rte_node, ctx)) !=
+			 RTE_CACHE_LINE_MIN_SIZE);
+
+	graph_spinlock_lock();
+
+	/* Check sanity */
+	if (reg == NULL || reg->process == NULL) {
+		rte_errno = EINVAL;
+		goto fail;
+	}
+
+	/* Check for duplicate name */
+	if (node_has_duplicate_entry(reg->name))
+		goto fail;
+
+	sz = sizeof(struct node) + (reg->nb_edges * RTE_NODE_NAMESIZE);
+	node = calloc(1, sz);
+	if (node == NULL) {
+		rte_errno = ENOMEM;
+		goto fail;
+	}
+
+	/* Initialize the node */
+	if (rte_strscpy(node->name, reg->name, RTE_NODE_NAMESIZE) < 0) {
+		rte_errno = E2BIG;
+		goto free;
+	}
+	node->flags = reg->flags;
+	node->process = reg->process;
+	node->init = reg->init;
+	node->fini = reg->fini;
+	node->nb_edges = reg->nb_edges;
+	node->parent_id = reg->parent_id;
+	for (i = 0; i < reg->nb_edges; i++) {
+		if (rte_strscpy(node->next_nodes[i], reg->next_nodes[i],
+				RTE_NODE_NAMESIZE) < 0) {
+			rte_errno = E2BIG;
+			goto free;
+		}
+	}
+
+	node->id = node_id++;
+
+	/* Add the node at tail */
+	STAILQ_INSERT_TAIL(&node_list, node, next);
+	graph_spinlock_unlock();
+
+	return node->id;
+free:
+	free(node);
+fail:
+	graph_spinlock_unlock();
+	return RTE_NODE_ID_INVALID;
+}
+
+static int
+clone_name(struct rte_node_register *reg, struct node *node, const char *name)
+{
+	ssize_t sz, rc;
+
+#define SZ RTE_NODE_NAMESIZE
+	rc = rte_strscpy(reg->name, node->name, SZ);
+	if (rc < 0)
+		goto fail;
+	sz = rc;
+	rc = rte_strscpy(reg->name + sz, "-", RTE_MAX((int16_t)(SZ - sz), 0));
+	if (rc < 0)
+		goto fail;
+	sz += rc;
+	sz = rte_strscpy(reg->name + sz, name, RTE_MAX((int16_t)(SZ - sz), 0));
+	if (sz < 0)
+		goto fail;
+
+	return 0;
+fail:
+	rte_errno = E2BIG;
+	return -rte_errno;
+}
+
+static rte_node_t
+node_clone(struct node *node, const char *name)
+{
+	rte_node_t rc = RTE_NODE_ID_INVALID;
+	struct rte_node_register *reg;
+	rte_edge_t i;
+
+	/* Don't allow to clone a node from a cloned node */
+	if (node->parent_id != RTE_NODE_ID_INVALID) {
+		rte_errno = EEXIST;
+		goto fail;
+	}
+
+	/* Check for duplicate name */
+	if (node_has_duplicate_entry(name))
+		goto fail;
+
+	reg = calloc(1, sizeof(*reg) + (sizeof(char *) * node->nb_edges));
+	if (reg == NULL) {
+		rte_errno = ENOMEM;
+		goto fail;
+	}
+
+	/* Clone the source node */
+	reg->flags = node->flags;
+	reg->process = node->process;
+	reg->init = node->init;
+	reg->fini = node->fini;
+	reg->nb_edges = node->nb_edges;
+	reg->parent_id = node->id;
+
+	for (i = 0; i < node->nb_edges; i++)
+		reg->next_nodes[i] = node->next_nodes[i];
+
+	/* Naming ceremony of the new node. name is node->name + "-" + name */
+	if (clone_name(reg, node, name))
+		goto free;
+
+	rc = __rte_node_register(reg);
+free:
+	free(reg);
+fail:
+	return rc;
+}
+
+rte_node_t
+rte_node_clone(rte_node_t id, const char *name)
+{
+	struct node *node;
+
+	NODE_ID_CHECK(id);
+	STAILQ_FOREACH(node, &node_list, next)
+		if (node->id == id)
+			return node_clone(node, name);
+
+fail:
+	return RTE_NODE_ID_INVALID;
+}
+
+rte_node_t
+rte_node_from_name(const char *name)
+{
+	struct node *node;
+
+	STAILQ_FOREACH(node, &node_list, next)
+		if (strncmp(node->name, name, RTE_NODE_NAMESIZE) == 0)
+			return node->id;
+
+	return RTE_NODE_ID_INVALID;
+}
+
+char *
+rte_node_id_to_name(rte_node_t id)
+{
+	struct node *node;
+
+	NODE_ID_CHECK(id);
+	STAILQ_FOREACH(node, &node_list, next)
+		if (node->id == id)
+			return node->name;
+
+fail:
+	return NULL;
+}
+
+rte_edge_t
+rte_node_edge_count(rte_node_t id)
+{
+	struct node *node;
+
+	NODE_ID_CHECK(id);
+	STAILQ_FOREACH(node, &node_list, next)
+		if (node->id == id)
+			return node->nb_edges;
+fail:
+	return RTE_EDGE_ID_INVALID;
+}
+
+static rte_edge_t
+edge_update(struct node *node, struct node *prev, rte_edge_t from,
+	    const char **next_nodes, rte_edge_t nb_edges)
+{
+	rte_edge_t i, max_edges, count = 0;
+	struct node *new_node;
+	bool need_realloc;
+	size_t sz;
+
+	if (from == RTE_EDGE_ID_INVALID)
+		from = node->nb_edges;
+
+	/* Don't create hole in next_nodes[] list */
+	if (from > node->nb_edges) {
+		rte_errno = ENOMEM;
+		goto fail;
+	}
+
+	/* Remove me from list */
+	STAILQ_REMOVE(&node_list, node, node, next);
+
+	/* Allocate the storage space for new node if required */
+	max_edges = from + nb_edges;
+	need_realloc = max_edges > node->nb_edges;
+	if (need_realloc) {
+		sz = sizeof(struct node) + (max_edges * RTE_NODE_NAMESIZE);
+		new_node = realloc(node, sz);
+		if (new_node == NULL) {
+			rte_errno = ENOMEM;
+			goto restore;
+		} else {
+			node = new_node;
+		}
+	}
+
+	/* Update the new nodes name */
+	for (i = from; i < max_edges; i++, count++) {
+		if (rte_strscpy(node->next_nodes[i], next_nodes[count],
+				RTE_NODE_NAMESIZE) < 0) {
+			rte_errno = E2BIG;
+			goto restore;
+		}
+	}
+restore:
+	/* Update the linked list to point new node address in prev node */
+	if (prev)
+		STAILQ_INSERT_AFTER(&node_list, prev, node, next);
+	else
+		STAILQ_INSERT_HEAD(&node_list, node, next);
+
+	if (need_realloc)
+		node->nb_edges = max_edges;
+
+fail:
+	return count;
+}
+
+rte_edge_t
+rte_node_edge_shrink(rte_node_t id, rte_edge_t size)
+{
+	rte_edge_t rc = RTE_EDGE_ID_INVALID;
+	struct node *node;
+
+	NODE_ID_CHECK(id);
+	graph_spinlock_lock();
+
+	STAILQ_FOREACH(node, &node_list, next) {
+		if (node->id == id) {
+			if (node->nb_edges < size) {
+				rte_errno = E2BIG;
+				goto fail;
+			}
+			node->nb_edges = size;
+			rc = size;
+			break;
+		}
+	}
+
+fail:
+	graph_spinlock_unlock();
+	return rc;
+}
+
+rte_edge_t
+rte_node_edge_update(rte_node_t id, rte_edge_t from, const char **next_nodes,
+		     uint16_t nb_edges)
+{
+	rte_edge_t rc = RTE_EDGE_ID_INVALID;
+	struct node *n, *prev;
+
+	NODE_ID_CHECK(id);
+	graph_spinlock_lock();
+
+	prev = NULL;
+	STAILQ_FOREACH(n, &node_list, next) {
+		if (n->id == id) {
+			rc = edge_update(n, prev, from, next_nodes, nb_edges);
+			break;
+		}
+		prev = n;
+	}
+
+	graph_spinlock_unlock();
+fail:
+	return rc;
+}
+
+static rte_node_t
+node_copy_edges(struct node *node, char *next_nodes[])
+{
+	rte_edge_t i;
+
+	for (i = 0; i < node->nb_edges; i++)
+		next_nodes[i] = node->next_nodes[i];
+
+	return i;
+}
+
+rte_node_t
+rte_node_edge_get(rte_node_t id, char *next_nodes[])
+{
+	rte_node_t rc = RTE_NODE_ID_INVALID;
+	struct node *node;
+
+	NODE_ID_CHECK(id);
+	graph_spinlock_lock();
+
+	STAILQ_FOREACH(node, &node_list, next) {
+		if (node->id == id) {
+			if (next_nodes == NULL)
+				rc = sizeof(char *) * node->nb_edges;
+			else
+				rc = node_copy_edges(node, next_nodes);
+			break;
+		}
+	}
+
+	graph_spinlock_unlock();
+fail:
+	return rc;
+}
+
+static void
+node_scan_dump(FILE *f, rte_node_t id, bool all)
+{
+	struct node *node;
+
+	RTE_ASSERT(f != NULL);
+	NODE_ID_CHECK(id);
+
+	STAILQ_FOREACH(node, &node_list, next) {
+		if (all == true) {
+			node_dump(f, node);
+		} else if (node->id == id) {
+			node_dump(f, node);
+			return;
+		}
+	}
+fail:
+	return;
+}
+
+void
+rte_node_dump(FILE *f, rte_node_t id)
+{
+	node_scan_dump(f, id, false);
+}
+
+void
+rte_node_list_dump(FILE *f)
+{
+	node_scan_dump(f, 0, true);
+}
+
+rte_node_t
+rte_node_max_count(void)
+{
+	return node_id;
+}
diff --git a/src/spdk/dpdk/lib/librte_graph/rte_graph.h b/src/spdk/dpdk/lib/librte_graph/rte_graph.h
new file mode 100644
index 000000000..9a26ffc18
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_graph/rte_graph.h
@@ -0,0 +1,668 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2020 Marvell International Ltd.
+ */
+
+#ifndef _RTE_GRAPH_H_
+#define _RTE_GRAPH_H_
+
+/**
+ * @file rte_graph.h
+ *
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Graph architecture abstracts the data processing functions as
+ * "node" and "link" them together to create a complex "graph" to enable
+ * reusable/modular data processing functions.
+ *
+ * This API enables graph framework operations such as create, lookup,
+ * dump and destroy on graph and node operations such as clone,
+ * edge update, and edge shrink, etc. The API also allows to create the stats
+ * cluster to monitor per graph and per node stats.
+ *
+ */
+
+#include <stdbool.h>
+#include <stdio.h>
+
+#include <rte_common.h>
+#include <rte_compat.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define RTE_GRAPH_NAMESIZE 64 /**< Max length of graph name. */
+#define RTE_NODE_NAMESIZE 64  /**< Max length of node name. */
+#define RTE_GRAPH_OFF_INVALID UINT32_MAX /**< Invalid graph offset. */
+#define RTE_NODE_ID_INVALID UINT32_MAX   /**< Invalid node id. */
+#define RTE_EDGE_ID_INVALID UINT16_MAX   /**< Invalid edge id. */
+#define RTE_GRAPH_ID_INVALID UINT16_MAX  /**< Invalid graph id. */
+#define RTE_GRAPH_FENCE 0xdeadbeef12345678ULL /**< Graph fence data. */
+
+typedef uint32_t rte_graph_off_t;  /**< Graph offset type. */
+typedef uint32_t rte_node_t;       /**< Node id type. */
+typedef uint16_t rte_edge_t;       /**< Edge id type. */
+typedef uint16_t rte_graph_t;      /**< Graph id type. */
+
+/** Burst size in terms of log2 */
+#if RTE_GRAPH_BURST_SIZE == 1
+#define RTE_GRAPH_BURST_SIZE_LOG2 0  /**< Object burst size of 1. */
+#elif RTE_GRAPH_BURST_SIZE == 2
+#define RTE_GRAPH_BURST_SIZE_LOG2 1  /**< Object burst size of 2. */
+#elif RTE_GRAPH_BURST_SIZE == 4
+#define RTE_GRAPH_BURST_SIZE_LOG2 2  /**< Object burst size of 4. */
+#elif RTE_GRAPH_BURST_SIZE == 8
+#define RTE_GRAPH_BURST_SIZE_LOG2 3  /**< Object burst size of 8. */
+#elif RTE_GRAPH_BURST_SIZE == 16
+#define RTE_GRAPH_BURST_SIZE_LOG2 4  /**< Object burst size of 16. */
+#elif RTE_GRAPH_BURST_SIZE == 32
+#define RTE_GRAPH_BURST_SIZE_LOG2 5  /**< Object burst size of 32. */
+#elif RTE_GRAPH_BURST_SIZE == 64
+#define RTE_GRAPH_BURST_SIZE_LOG2 6  /**< Object burst size of 64. */
+#elif RTE_GRAPH_BURST_SIZE == 128
+#define RTE_GRAPH_BURST_SIZE_LOG2 7  /**< Object burst size of 128. */
+#elif RTE_GRAPH_BURST_SIZE == 256
+#define RTE_GRAPH_BURST_SIZE_LOG2 8  /**< Object burst size of 256. */
+#else
+#error "Unsupported burst size"
+#endif
+
+/* Forward declaration */
+struct rte_node;  /**< Node object */
+struct rte_graph; /**< Graph object */
+struct rte_graph_cluster_stats;      /**< Stats for Cluster of graphs */
+struct rte_graph_cluster_node_stats; /**< Node stats within cluster of graphs */
+
+/**
+ * Node process function.
+ *
+ * The function invoked when the worker thread walks on nodes using
+ * rte_graph_walk().
+ *
+ * @param graph
+ *   Pointer to the graph object.
+ * @param node
+ *   Pointer to the node object.
+ * @param objs
+ *   Pointer to an array of objects to be processed.
+ * @param nb_objs
+ *   Number of objects in the array.
+ *
+ * @return
+ *   Number of objects processed.
+ *
+ * @see rte_graph_walk()
+ *
+ */
+typedef uint16_t (*rte_node_process_t)(struct rte_graph *graph,
+				       struct rte_node *node, void **objs,
+				       uint16_t nb_objs);
+
+/**
+ * Node initialization function.
+ *
+ * The function invoked when the user creates the graph using rte_graph_create()
+ *
+ * @param graph
+ *   Pointer to the graph object.
+ * @param node
+ *   Pointer to the node object.
+ *
+ * @return
+ *   - 0: Success.
+ *   -<0: Failure.
+ *
+ * @see rte_graph_create()
+ */
+typedef int (*rte_node_init_t)(const struct rte_graph *graph,
+			       struct rte_node *node);
+
+/**
+ * Node finalization function.
+ *
+ * The function invoked when the user destroys the graph using
+ * rte_graph_destroy().
+ *
+ * @param graph
+ *   Pointer to the graph object.
+ * @param node
+ *   Pointer to the node object.
+ *
+ * @see rte_graph_destroy()
+ */
+typedef void (*rte_node_fini_t)(const struct rte_graph *graph,
+				struct rte_node *node);
+
+/**
+ * Graph cluster stats callback.
+ *
+ * @param is_first
+ *   Flag to denote that stats are of the first node.
+ * @param is_last
+ *   Flag to denote that stats are of the last node.
+ * @param cookie
+ *   Cookie supplied during stats creation.
+ * @param stats
+ *   Node cluster stats data.
+ *
+ * @return
+ *   - 0: Success.
+ *   -<0: Failure.
+ */
+typedef int (*rte_graph_cluster_stats_cb_t)(bool is_first, bool is_last,
+	     void *cookie, const struct rte_graph_cluster_node_stats *stats);
+
+/**
+ * Structure to hold configuration parameters for creating the graph.
+ *
+ * @see rte_graph_create()
+ */
+struct rte_graph_param {
+	int socket_id; /**< Socket id where memory is allocated. */
+	uint16_t nb_node_patterns;  /**< Number of node patterns. */
+	const char **node_patterns;
+	/**< Array of node patterns based on shell pattern. */
+};
+
+/**
+ * Structure to hold configuration parameters for graph cluster stats create.
+ *
+ * @see rte_graph_cluster_stats_create()
+ */
+struct rte_graph_cluster_stats_param {
+	int socket_id;
+	/**< Socket id where memory is allocated */
+	rte_graph_cluster_stats_cb_t fn;
+	/**< Stats print callback function. NULL value allowed, in that case,
+	 *   default print stat function used.
+	 */
+	RTE_STD_C11
+	union {
+		void *cookie;
+		FILE *f; /**< File pointer to dump the stats when fn == NULL. */
+	};
+	uint16_t nb_graph_patterns;  /**< Number of graph patterns. */
+	const char **graph_patterns;
+	/**< Array of graph patterns based on shell pattern. */
+};
+
+/**
+ * Node cluster stats data structure.
+ *
+ * @see struct rte_graph_cluster_stats_param::fn
+ */
+struct rte_graph_cluster_node_stats {
+	uint64_t ts;	    /**< Current timestamp. */
+	uint64_t calls;	    /**< Current number of calls made. */
+	uint64_t objs;      /**< Current number of objs processed. */
+	uint64_t cycles;    /**< Current number of cycles. */
+
+	uint64_t prev_ts;	/**< Previous call timestamp. */
+	uint64_t prev_calls;	/**< Previous number of calls. */
+	uint64_t prev_objs;	/**< Previous number of processed objs. */
+	uint64_t prev_cycles;	/**< Previous number of cycles. */
+
+	uint64_t realloc_count; /**< Realloc count. */
+
+	rte_node_t id;	/**< Node identifier of stats. */
+	uint64_t hz;	/**< Cycles per seconds. */
+	char name[RTE_NODE_NAMESIZE];	/**< Name of the node. */
+} __rte_cache_aligned;
+
+/**
+ * Create Graph.
+ *
+ * Create memory reel, detect loops and find isolated nodes.
+ *
+ * @param name
+ *   Unique name for this graph.
+ * @param prm
+ *   Graph parameter, includes node names and count to be included
+ *   in this graph.
+ *
+ * @return
+ *   Unique graph id on success, RTE_GRAPH_ID_INVALID otherwise.
+ */
+__rte_experimental
+rte_graph_t rte_graph_create(const char *name, struct rte_graph_param *prm);
+
+/**
+ * Destroy Graph.
+ *
+ * Free Graph memory reel.
+ *
+ * @param id
+ *   id of the graph to destroy.
+ *
+ * @return
+ *   0 on success, error otherwise.
+ */
+__rte_experimental
+int rte_graph_destroy(rte_graph_t id);
+
+/**
+ * Get graph id from graph name.
+ *
+ * @param name
+ *   Name of the graph to get id.
+ *
+ * @return
+ *   Graph id on success, RTE_GRAPH_ID_INVALID otherwise.
+ */
+__rte_experimental
+rte_graph_t rte_graph_from_name(const char *name);
+
+/**
+ * Get graph name from graph id.
+ *
+ * @param id
+ *   id of the graph to get name.
+ *
+ * @return
+ *   Graph name on success, NULL otherwise.
+ */
+__rte_experimental
+char *rte_graph_id_to_name(rte_graph_t id);
+
+/**
+ * Export the graph as graph viz dot file
+ *
+ * @param name
+ *   Name of the graph to export.
+ * @param f
+ *   File pointer to export the graph.
+ *
+ * @return
+ *   0 on success, error otherwise.
+ */
+__rte_experimental
+int rte_graph_export(const char *name, FILE *f);
+
+/**
+ * Get graph object from its name.
+ *
+ * Typical usage of this API to get graph objects in the worker thread and
+ * followed calling rte_graph_walk() in a loop.
+ *
+ * @param name
+ *   Name of the graph.
+ *
+ * @return
+ *   Graph pointer on success, NULL otherwise.
+ *
+ * @see rte_graph_walk()
+ */
+__rte_experimental
+struct rte_graph *rte_graph_lookup(const char *name);
+
+/**
+ * Get maximum number of graph available.
+ *
+ * @return
+ *   Maximum graph count.
+ */
+__rte_experimental
+rte_graph_t rte_graph_max_count(void);
+
+/**
+ * Dump the graph information to file.
+ *
+ * @param f
+ *   File pointer to dump graph info.
+ * @param id
+ *   Graph id to get graph info.
+ */
+__rte_experimental
+void rte_graph_dump(FILE *f, rte_graph_t id);
+
+/**
+ * Dump all graphs information to file
+ *
+ * @param f
+ *   File pointer to dump graph info.
+ */
+__rte_experimental
+void rte_graph_list_dump(FILE *f);
+
+/**
+ * Dump graph information along with node info to file
+ *
+ * @param f
+ *   File pointer to dump graph info.
+ * @param graph
+ *   Graph pointer to get graph info.
+ * @param all
+ *   true to dump nodes in the graph.
+ */
+__rte_experimental
+void rte_graph_obj_dump(FILE *f, struct rte_graph *graph, bool all);
+
+/** Macro to browse rte_node object after the graph creation */
+#define rte_graph_foreach_node(count, off, graph, node)                        \
+	for (count = 0, off = graph->nodes_start,                              \
+	     node = RTE_PTR_ADD(graph, off);                                   \
+	     count < graph->nb_nodes;                                          \
+	     off = node->next, node = RTE_PTR_ADD(graph, off), count++)
+
+/**
+ * Get node object with in graph from id.
+ *
+ * @param graph_id
+ *   Graph id to get node pointer from.
+ * @param node_id
+ *   Node id to get node pointer.
+ *
+ * @return
+ *   Node pointer on success, NULL otherwise.
+ */
+__rte_experimental
+struct rte_node *rte_graph_node_get(rte_graph_t graph_id, rte_node_t node_id);
+
+/**
+ * Get node pointer with in graph from name.
+ *
+ * @param graph
+ *   Graph name to get node pointer from.
+ * @param name
+ *   Node name to get the node pointer.
+ *
+ * @return
+ *   Node pointer on success, NULL otherwise.
+ */
+__rte_experimental
+struct rte_node *rte_graph_node_get_by_name(const char *graph,
+					    const char *name);
+
+/**
+ * Create graph stats cluster to aggregate runtime node stats.
+ *
+ * @param prm
+ *   Parameters including file pointer to dump stats,
+ *   Graph pattern to create cluster and callback function.
+ *
+ * @return
+ *   Valid pointer on success, NULL otherwise.
+ */
+__rte_experimental
+struct rte_graph_cluster_stats *rte_graph_cluster_stats_create(
+			const struct rte_graph_cluster_stats_param *prm);
+
+/**
+ * Destroy cluster stats.
+ *
+ * @param stat
+ *    Valid cluster pointer to destroy.
+ */
+__rte_experimental
+void rte_graph_cluster_stats_destroy(struct rte_graph_cluster_stats *stat);
+
+/**
+ * Get stats to application.
+ *
+ * @param[out] stat
+ *   Cluster status.
+ * @param skip_cb
+ *   true to skip callback function invocation.
+ */
+__rte_experimental
+void rte_graph_cluster_stats_get(struct rte_graph_cluster_stats *stat,
+				 bool skip_cb);
+
+/**
+ * Reset cluster stats to zero.
+ *
+ * @param stat
+ *   Valid cluster stats pointer.
+ */
+__rte_experimental
+void rte_graph_cluster_stats_reset(struct rte_graph_cluster_stats *stat);
+
+/**
+ * Structure defines the node registration parameters.
+ *
+ * @see __rte_node_register(), RTE_NODE_REGISTER()
+ */
+struct rte_node_register {
+	char name[RTE_NODE_NAMESIZE]; /**< Name of the node. */
+	uint64_t flags;		      /**< Node configuration flag. */
+#define RTE_NODE_SOURCE_F (1ULL << 0) /**< Node type is source. */
+	rte_node_process_t process; /**< Node process function. */
+	rte_node_init_t init;       /**< Node init function. */
+	rte_node_fini_t fini;       /**< Node fini function. */
+	rte_node_t id;		    /**< Node Identifier. */
+	rte_node_t parent_id;       /**< Identifier of parent node. */
+	rte_edge_t nb_edges;        /**< Number of edges from this node. */
+	const char *next_nodes[];   /**< Names of next nodes. */
+};
+
+/**
+ * Register new packet processing node. Nodes can be registered
+ * dynamically via this call or statically via the RTE_NODE_REGISTER
+ * macro.
+ *
+ * @param node
+ *   Valid node pointer with name, process function and next_nodes.
+ *
+ * @return
+ *   Valid node id on success, RTE_NODE_ID_INVALID otherwise.
+ *
+ * @see RTE_NODE_REGISTER()
+ */
+__rte_experimental
+rte_node_t __rte_node_register(const struct rte_node_register *node);
+
+/**
+ * Register a static node.
+ *
+ * The static node is registered through the constructor scheme, thereby, it can
+ * be used in a multi-process scenario.
+ *
+ * @param node
+ *   Valid node pointer with name, process function, and next_nodes.
+ */
+#define RTE_NODE_REGISTER(node)                                                \
+	RTE_INIT(rte_node_register_##node)                                     \
+	{                                                                      \
+		node.parent_id = RTE_NODE_ID_INVALID;                          \
+		node.id = __rte_node_register(&node);                          \
+	}
+
+/**
+ * Clone a node from static node(node created from RTE_NODE_REGISTER).
+ *
+ * @param id
+ *   Static node id to clone from.
+ * @param name
+ *   Name of the new node. The library prepends the parent node name to the
+ * user-specified name. The final node name will be,
+ * "parent node name" + "-" + name.
+ *
+ * @return
+ *   Valid node id on success, RTE_NODE_ID_INVALID otherwise.
+ */
+__rte_experimental
+rte_node_t rte_node_clone(rte_node_t id, const char *name);
+
+/**
+ * Get node id from node name.
+ *
+ * @param name
+ *   Valid node name. In the case of the cloned node, the name will be
+ * "parent node name" + "-" + name.
+ *
+ * @return
+ *   Valid node id on success, RTE_NODE_ID_INVALID otherwise.
+ */
+__rte_experimental
+rte_node_t rte_node_from_name(const char *name);
+
+/**
+ * Get node name from node id.
+ *
+ * @param id
+ *   Valid node id.
+ *
+ * @return
+ *   Valid node name on success, NULL otherwise.
+ */
+__rte_experimental
+char *rte_node_id_to_name(rte_node_t id);
+
+/**
+ * Get the number of edges(next-nodes) for a node from node id.
+ *
+ * @param id
+ *   Valid node id.
+ *
+ * @return
+ *   Valid edge count on success, RTE_EDGE_ID_INVALID otherwise.
+ */
+__rte_experimental
+rte_edge_t rte_node_edge_count(rte_node_t id);
+
+/**
+ * Update the edges for a node from node id.
+ *
+ * @param id
+ *   Valid node id.
+ * @param from
+ *   Index to update the edges from. RTE_EDGE_ID_INVALID is valid,
+ * in that case, it will be added to the end of the list.
+ * @param next_nodes
+ *   Name of the edges to update.
+ * @param nb_edges
+ *   Number of edges to update.
+ *
+ * @return
+ *   Valid edge count on success, 0 otherwise.
+ */
+__rte_experimental
+rte_edge_t rte_node_edge_update(rte_node_t id, rte_edge_t from,
+				const char **next_nodes, uint16_t nb_edges);
+
+/**
+ * Shrink the edges to a given size.
+ *
+ * @param id
+ *   Valid node id.
+ * @param size
+ *   New size to shrink the edges.
+ *
+ * @return
+ *   New size on success, RTE_EDGE_ID_INVALID otherwise.
+ */
+__rte_experimental
+rte_edge_t rte_node_edge_shrink(rte_node_t id, rte_edge_t size);
+
+/**
+ * Get the edge names from a given node.
+ *
+ * @param id
+ *   Valid node id.
+ * @param[out] next_nodes
+ *   Buffer to copy the edge names. The NULL value is allowed in that case,
+ * the function returns the size of the array that needs to be allocated.
+ *
+ * @return
+ *   When next_nodes == NULL, it returns the size of the array else
+ *  number of item copied.
+ */
+__rte_experimental
+rte_node_t rte_node_edge_get(rte_node_t id, char *next_nodes[]);
+
+/**
+ * Get maximum nodes available.
+ *
+ * @return
+ *   Maximum nodes count.
+ */
+__rte_experimental
+rte_node_t rte_node_max_count(void);
+
+/**
+ * Dump node info to file.
+ *
+ * @param f
+ *   File pointer to dump the node info.
+ * @param id
+ *   Node id to get the info.
+ */
+__rte_experimental
+void rte_node_dump(FILE *f, rte_node_t id);
+
+/**
+ * Dump all node info to file.
+ *
+ * @param f
+ *   File pointer to dump the node info.
+ */
+__rte_experimental
+void rte_node_list_dump(FILE *f);
+
+/**
+ * Test the validity of node id.
+ *
+ * @param id
+ *   Node id to check.
+ *
+ * @return
+ *   1 if valid id, 0 otherwise.
+ */
+static __rte_always_inline int
+rte_node_is_invalid(rte_node_t id)
+{
+	return (id == RTE_NODE_ID_INVALID);
+}
+
+/**
+ * Test the validity of edge id.
+ *
+ * @param id
+ *   Edge node id to check.
+ *
+ * @return
+ *   1 if valid id, 0 otherwise.
+ */
+static __rte_always_inline int
+rte_edge_is_invalid(rte_edge_t id)
+{
+	return (id == RTE_EDGE_ID_INVALID);
+}
+
+/**
+ * Test the validity of graph id.
+ *
+ * @param id
+ *   Graph id to check.
+ *
+ * @return
+ *   1 if valid id, 0 otherwise.
+ */
+static __rte_always_inline int
+rte_graph_is_invalid(rte_graph_t id)
+{
+	return (id == RTE_GRAPH_ID_INVALID);
+}
+
+/**
+ * Test stats feature support.
+ *
+ * @return
+ *   1 if stats enabled, 0 otherwise.
+ */
+static __rte_always_inline int
+rte_graph_has_stats_feature(void)
+{
+#ifdef RTE_LIBRTE_GRAPH_STATS
+	return RTE_LIBRTE_GRAPH_STATS;
+#else
+	return 0;
+#endif
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_GRAPH_H_ */
diff --git a/src/spdk/dpdk/lib/librte_graph/rte_graph_version.map b/src/spdk/dpdk/lib/librte_graph/rte_graph_version.map
new file mode 100644
index 000000000..13b838752
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_graph/rte_graph_version.map
@@ -0,0 +1,47 @@
+EXPERIMENTAL {
+	global:
+
+	__rte_node_register;
+	__rte_node_stream_alloc;
+	__rte_node_stream_alloc_size;
+
+	rte_graph_create;
+	rte_graph_destroy;
+	rte_graph_dump;
+	rte_graph_export;
+	rte_graph_from_name;
+	rte_graph_id_to_name;
+	rte_graph_lookup;
+	rte_graph_list_dump;
+	rte_graph_max_count;
+	rte_graph_node_get;
+	rte_graph_node_get_by_name;
+	rte_graph_obj_dump;
+	rte_graph_walk;
+
+	rte_graph_cluster_stats_create;
+	rte_graph_cluster_stats_destroy;
+	rte_graph_cluster_stats_get;
+	rte_graph_cluster_stats_reset;
+
+	rte_node_clone;
+	rte_node_dump;
+	rte_node_edge_count;
+	rte_node_edge_get;
+	rte_node_edge_shrink;
+	rte_node_edge_update;
+	rte_node_enqueue;
+	rte_node_enqueue_x1;
+	rte_node_enqueue_x2;
+	rte_node_enqueue_x4;
+	rte_node_enqueue_next;
+	rte_node_from_name;
+	rte_node_id_to_name;
+	rte_node_list_dump;
+	rte_node_max_count;
+	rte_node_next_stream_get;
+	rte_node_next_stream_put;
+	rte_node_next_stream_move;
+
+	local: *;
+};
diff --git a/src/spdk/dpdk/lib/librte_graph/rte_graph_worker.h b/src/spdk/dpdk/lib/librte_graph/rte_graph_worker.h
new file mode 100644
index 000000000..4c3ddcbde
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_graph/rte_graph_worker.h
@@ -0,0 +1,510 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2020 Marvell International Ltd.
+ */
+
+#ifndef _RTE_GRAPH_WORKER_H_
+#define _RTE_GRAPH_WORKER_H_
+
+/**
+ * @file rte_graph_worker.h
+ *
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * This API allows a worker thread to walk over a graph and nodes to create,
+ * process, enqueue and move streams of objects to the next nodes.
+ */
+
+#include <rte_common.h>
+#include <rte_cycles.h>
+#include <rte_prefetch.h>
+#include <rte_memcpy.h>
+#include <rte_memory.h>
+
+#include "rte_graph.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * @internal
+ *
+ * Data structure to hold graph data.
+ */
+struct rte_graph {
+	uint32_t tail;		     /**< Tail of circular buffer. */
+	uint32_t head;		     /**< Head of circular buffer. */
+	uint32_t cir_mask;	     /**< Circular buffer wrap around mask. */
+	rte_node_t nb_nodes;	     /**< Number of nodes in the graph. */
+	rte_graph_off_t *cir_start;  /**< Pointer to circular buffer. */
+	rte_graph_off_t nodes_start; /**< Offset at which node memory starts. */
+	rte_graph_t id;	/**< Graph identifier. */
+	int socket;	/**< Socket ID where memory is allocated. */
+	char name[RTE_GRAPH_NAMESIZE];	/**< Name of the graph. */
+	uint64_t fence;			/**< Fence. */
+} __rte_cache_aligned;
+
+/**
+ * @internal
+ *
+ * Data structure to hold node data.
+ */
+struct rte_node {
+	/* Slow path area  */
+	uint64_t fence;		/**< Fence. */
+	rte_graph_off_t next;	/**< Index to next node. */
+	rte_node_t id;		/**< Node identifier. */
+	rte_node_t parent_id;	/**< Parent Node identifier. */
+	rte_edge_t nb_edges;	/**< Number of edges from this node. */
+	uint32_t realloc_count;	/**< Number of times realloced. */
+
+	char parent[RTE_NODE_NAMESIZE];	/**< Parent node name. */
+	char name[RTE_NODE_NAMESIZE];	/**< Name of the node. */
+
+	/* Fast path area  */
+#define RTE_NODE_CTX_SZ 16
+	uint8_t ctx[RTE_NODE_CTX_SZ] __rte_cache_aligned; /**< Node Context. */
+	uint16_t size;		/**< Total number of objects available. */
+	uint16_t idx;		/**< Number of objects used. */
+	rte_graph_off_t off;	/**< Offset of node in the graph reel. */
+	uint64_t total_cycles;	/**< Cycles spent in this node. */
+	uint64_t total_calls;	/**< Calls done to this node. */
+	uint64_t total_objs;	/**< Objects processed by this node. */
+	RTE_STD_C11
+		union {
+			void **objs;	   /**< Array of object pointers. */
+			uint64_t objs_u64;
+		};
+	RTE_STD_C11
+		union {
+			rte_node_process_t process; /**< Process function. */
+			uint64_t process_u64;
+		};
+	struct rte_node *nodes[] __rte_cache_min_aligned; /**< Next nodes. */
+} __rte_cache_aligned;
+
+/**
+ * @internal
+ *
+ * Allocate a stream of objects.
+ *
+ * If stream already exists then re-allocate it to a larger size.
+ *
+ * @param graph
+ *   Pointer to the graph object.
+ * @param node
+ *   Pointer to the node object.
+ */
+__rte_experimental
+void __rte_node_stream_alloc(struct rte_graph *graph, struct rte_node *node);
+
+/**
+ * @internal
+ *
+ * Allocate a stream with requested number of objects.
+ *
+ * If stream already exists then re-allocate it to a larger size.
+ *
+ * @param graph
+ *   Pointer to the graph object.
+ * @param node
+ *   Pointer to the node object.
+ * @param req_size
+ *   Number of objects to be allocated.
+ */
+__rte_experimental
+void __rte_node_stream_alloc_size(struct rte_graph *graph,
+				  struct rte_node *node, uint16_t req_size);
+
+/**
+ * Perform graph walk on the circular buffer and invoke the process function
+ * of the nodes and collect the stats.
+ *
+ * @param graph
+ *   Graph pointer returned from rte_graph_lookup function.
+ *
+ * @see rte_graph_lookup()
+ */
+__rte_experimental
+static inline void
+rte_graph_walk(struct rte_graph *graph)
+{
+	const rte_graph_off_t *cir_start = graph->cir_start;
+	const rte_node_t mask = graph->cir_mask;
+	uint32_t head = graph->head;
+	struct rte_node *node;
+	uint64_t start;
+	uint16_t rc;
+	void **objs;
+
+	/*
+	 * Walk on the source node(s) ((cir_start - head) -> cir_start) and then
+	 * on the pending streams (cir_start -> (cir_start + mask) -> cir_start)
+	 * in a circular buffer fashion.
+	 *
+	 *	+-----+ <= cir_start - head [number of source nodes]
+	 *	|     |
+	 *	| ... | <= source nodes
+	 *	|     |
+	 *	+-----+ <= cir_start [head = 0] [tail = 0]
+	 *	|     |
+	 *	| ... | <= pending streams
+	 *	|     |
+	 *	+-----+ <= cir_start + mask
+	 */
+	while (likely(head != graph->tail)) {
+		node = RTE_PTR_ADD(graph, cir_start[(int32_t)head++]);
+		RTE_ASSERT(node->fence == RTE_GRAPH_FENCE);
+		objs = node->objs;
+		rte_prefetch0(objs);
+
+		if (rte_graph_has_stats_feature()) {
+			start = rte_rdtsc();
+			rc = node->process(graph, node, objs, node->idx);
+			node->total_cycles += rte_rdtsc() - start;
+			node->total_calls++;
+			node->total_objs += rc;
+		} else {
+			node->process(graph, node, objs, node->idx);
+		}
+		node->idx = 0;
+		head = likely((int32_t)head > 0) ? head & mask : head;
+	}
+	graph->tail = 0;
+}
+
+/* Fast path helper functions */
+
+/**
+ * @internal
+ *
+ * Enqueue a given node to the tail of the graph reel.
+ *
+ * @param graph
+ *   Pointer Graph object.
+ * @param node
+ *   Pointer to node object to be enqueued.
+ */
+static __rte_always_inline void
+__rte_node_enqueue_tail_update(struct rte_graph *graph, struct rte_node *node)
+{
+	uint32_t tail;
+
+	tail = graph->tail;
+	graph->cir_start[tail++] = node->off;
+	graph->tail = tail & graph->cir_mask;
+}
+
+/**
+ * @internal
+ *
+ * Enqueue sequence prologue function.
+ *
+ * Updates the node to tail of graph reel and resizes the number of objects
+ * available in the stream as needed.
+ *
+ * @param graph
+ *   Pointer to the graph object.
+ * @param node
+ *   Pointer to the node object.
+ * @param idx
+ *   Index at which the object enqueue starts from.
+ * @param space
+ *   Space required for the object enqueue.
+ */
+static __rte_always_inline void
+__rte_node_enqueue_prologue(struct rte_graph *graph, struct rte_node *node,
+			    const uint16_t idx, const uint16_t space)
+{
+
+	/* Add to the pending stream list if the node is new */
+	if (idx == 0)
+		__rte_node_enqueue_tail_update(graph, node);
+
+	if (unlikely(node->size < (idx + space)))
+		__rte_node_stream_alloc(graph, node);
+}
+
+/**
+ * @internal
+ *
+ * Get the node pointer from current node edge id.
+ *
+ * @param node
+ *   Current node pointer.
+ * @param next
+ *   Edge id of the required node.
+ *
+ * @return
+ *   Pointer to the node denoted by the edge id.
+ */
+static __rte_always_inline struct rte_node *
+__rte_node_next_node_get(struct rte_node *node, rte_edge_t next)
+{
+	RTE_ASSERT(next < node->nb_edges);
+	RTE_ASSERT(node->fence == RTE_GRAPH_FENCE);
+	node = node->nodes[next];
+	RTE_ASSERT(node->fence == RTE_GRAPH_FENCE);
+
+	return node;
+}
+
+/**
+ * Enqueue the objs to next node for further processing and set
+ * the next node to pending state in the circular buffer.
+ *
+ * @param graph
+ *   Graph pointer returned from rte_graph_lookup().
+ * @param node
+ *   Current node pointer.
+ * @param next
+ *   Relative next node index to enqueue objs.
+ * @param objs
+ *   Objs to enqueue.
+ * @param nb_objs
+ *   Number of objs to enqueue.
+ */
+__rte_experimental
+static inline void
+rte_node_enqueue(struct rte_graph *graph, struct rte_node *node,
+		 rte_edge_t next, void **objs, uint16_t nb_objs)
+{
+	node = __rte_node_next_node_get(node, next);
+	const uint16_t idx = node->idx;
+
+	__rte_node_enqueue_prologue(graph, node, idx, nb_objs);
+
+	rte_memcpy(&node->objs[idx], objs, nb_objs * sizeof(void *));
+	node->idx = idx + nb_objs;
+}
+
+/**
+ * Enqueue only one obj to next node for further processing and
+ * set the next node to pending state in the circular buffer.
+ *
+ * @param graph
+ *   Graph pointer returned from rte_graph_lookup().
+ * @param node
+ *   Current node pointer.
+ * @param next
+ *   Relative next node index to enqueue objs.
+ * @param obj
+ *   Obj to enqueue.
+ */
+__rte_experimental
+static inline void
+rte_node_enqueue_x1(struct rte_graph *graph, struct rte_node *node,
+		    rte_edge_t next, void *obj)
+{
+	node = __rte_node_next_node_get(node, next);
+	uint16_t idx = node->idx;
+
+	__rte_node_enqueue_prologue(graph, node, idx, 1);
+
+	node->objs[idx++] = obj;
+	node->idx = idx;
+}
+
+/**
+ * Enqueue only two objs to next node for further processing and
+ * set the next node to pending state in the circular buffer.
+ * Same as rte_node_enqueue_x1 but enqueue two objs.
+ *
+ * @param graph
+ *   Graph pointer returned from rte_graph_lookup().
+ * @param node
+ *   Current node pointer.
+ * @param next
+ *   Relative next node index to enqueue objs.
+ * @param obj0
+ *   Obj to enqueue.
+ * @param obj1
+ *   Obj to enqueue.
+ */
+__rte_experimental
+static inline void
+rte_node_enqueue_x2(struct rte_graph *graph, struct rte_node *node,
+		    rte_edge_t next, void *obj0, void *obj1)
+{
+	node = __rte_node_next_node_get(node, next);
+	uint16_t idx = node->idx;
+
+	__rte_node_enqueue_prologue(graph, node, idx, 2);
+
+	node->objs[idx++] = obj0;
+	node->objs[idx++] = obj1;
+	node->idx = idx;
+}
+
+/**
+ * Enqueue only four objs to next node for further processing and
+ * set the next node to pending state in the circular buffer.
+ * Same as rte_node_enqueue_x1 but enqueue four objs.
+ *
+ * @param graph
+ *   Graph pointer returned from rte_graph_lookup().
+ * @param node
+ *   Current node pointer.
+ * @param next
+ *   Relative next node index to enqueue objs.
+ * @param obj0
+ *   1st obj to enqueue.
+ * @param obj1
+ *   2nd obj to enqueue.
+ * @param obj2
+ *   3rd obj to enqueue.
+ * @param obj3
+ *   4th obj to enqueue.
+ */
+__rte_experimental
+static inline void
+rte_node_enqueue_x4(struct rte_graph *graph, struct rte_node *node,
+		    rte_edge_t next, void *obj0, void *obj1, void *obj2,
+		    void *obj3)
+{
+	node = __rte_node_next_node_get(node, next);
+	uint16_t idx = node->idx;
+
+	__rte_node_enqueue_prologue(graph, node, idx, 4);
+
+	node->objs[idx++] = obj0;
+	node->objs[idx++] = obj1;
+	node->objs[idx++] = obj2;
+	node->objs[idx++] = obj3;
+	node->idx = idx;
+}
+
+/**
+ * Enqueue objs to multiple next nodes for further processing and
+ * set the next nodes to pending state in the circular buffer.
+ * objs[i] will be enqueued to nexts[i].
+ *
+ * @param graph
+ *   Graph pointer returned from rte_graph_lookup().
+ * @param node
+ *   Current node pointer.
+ * @param nexts
+ *   List of relative next node indices to enqueue objs.
+ * @param objs
+ *   List of objs to enqueue.
+ * @param nb_objs
+ *   Number of objs to enqueue.
+ */
+__rte_experimental
+static inline void
+rte_node_enqueue_next(struct rte_graph *graph, struct rte_node *node,
+		      rte_edge_t *nexts, void **objs, uint16_t nb_objs)
+{
+	uint16_t i;
+
+	for (i = 0; i < nb_objs; i++)
+		rte_node_enqueue_x1(graph, node, nexts[i], objs[i]);
+}
+
+/**
+ * Get the stream of next node to enqueue the objs.
+ * Once done with the updating the objs, needs to call
+ * rte_node_next_stream_put to put the next node to pending state.
+ *
+ * @param graph
+ *   Graph pointer returned from rte_graph_lookup().
+ * @param node
+ *   Current node pointer.
+ * @param next
+ *   Relative next node index to get stream.
+ * @param nb_objs
+ *   Requested free size of the next stream.
+ *
+ * @return
+ *   Valid next stream on success.
+ *
+ * @see rte_node_next_stream_put().
+ */
+__rte_experimental
+static inline void **
+rte_node_next_stream_get(struct rte_graph *graph, struct rte_node *node,
+			 rte_edge_t next, uint16_t nb_objs)
+{
+	node = __rte_node_next_node_get(node, next);
+	const uint16_t idx = node->idx;
+	uint16_t free_space = node->size - idx;
+
+	if (unlikely(free_space < nb_objs))
+		__rte_node_stream_alloc_size(graph, node, nb_objs);
+
+	return &node->objs[idx];
+}
+
+/**
+ * Put the next stream to pending state in the circular buffer
+ * for further processing. Should be invoked after rte_node_next_stream_get().
+ *
+ * @param graph
+ *   Graph pointer returned from rte_graph_lookup().
+ * @param node
+ *   Current node pointer.
+ * @param next
+ *   Relative next node index..
+ * @param idx
+ *   Number of objs updated in the stream after getting the stream using
+ *   rte_node_next_stream_get.
+ *
+ * @see rte_node_next_stream_get().
+ */
+__rte_experimental
+static inline void
+rte_node_next_stream_put(struct rte_graph *graph, struct rte_node *node,
+			 rte_edge_t next, uint16_t idx)
+{
+	if (unlikely(!idx))
+		return;
+
+	node = __rte_node_next_node_get(node, next);
+	if (node->idx == 0)
+		__rte_node_enqueue_tail_update(graph, node);
+
+	node->idx += idx;
+}
+
+/**
+ * Home run scenario, Enqueue all the objs of current node to next
+ * node in optimized way by swapping the streams of both nodes.
+ * Performs good when next node is already not in pending state.
+ * If next node is already in pending state then normal enqueue
+ * will be used.
+ *
+ * @param graph
+ *   Graph pointer returned from rte_graph_lookup().
+ * @param src
+ *   Current node pointer.
+ * @param next
+ *   Relative next node index.
+ */
+__rte_experimental
+static inline void
+rte_node_next_stream_move(struct rte_graph *graph, struct rte_node *src,
+			  rte_edge_t next)
+{
+	struct rte_node *dst = __rte_node_next_node_get(src, next);
+
+	/* Let swap the pointers if dst don't have valid objs */
+	if (likely(dst->idx == 0)) {
+		void **dobjs = dst->objs;
+		uint16_t dsz = dst->size;
+		dst->objs = src->objs;
+		dst->size = src->size;
+		src->objs = dobjs;
+		src->size = dsz;
+		dst->idx = src->idx;
+		__rte_node_enqueue_tail_update(graph, dst);
+	} else { /* Move the objects from src node to dst node */
+		rte_node_enqueue(graph, src, next, src->objs, src->idx);
+	}
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_GRAPH_WORKER_H_ */