Adding upstream version 18.2.2.upstream/18.2.2

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

6 files changed, 1137 insertions, 0 deletions

diff --git a/src/spdk/dpdk/drivers/crypto/snow3g/Makefile b/src/spdk/dpdk/drivers/crypto/snow3g/Makefile
new file mode 100644
index 000000000..469aefaa5
--- /dev/null
+++ b/src/spdk/dpdk/drivers/crypto/snow3g/Makefile
@@ -0,0 +1,43 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2016-2019 Intel Corporation
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+# library name
+LIB = librte_pmd_snow3g.a
+
+# build flags
+CFLAGS += -O3
+CFLAGS += $(WERROR_FLAGS)
+
+# versioning export map
+EXPORT_MAP := rte_pmd_snow3g_version.map
+
+# external library dependencies
+LDLIBS += -lIPSec_MB
+LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
+LDLIBS += -lrte_cryptodev
+LDLIBS += -lrte_bus_vdev
+
+H := \#
+IMB_HDR = $(shell echo '$Hinclude <intel-ipsec-mb.h>' | \
+	$(CC) -E $(EXTRA_CFLAGS) - | grep 'intel-ipsec-mb.h' | \
+	head -n1 | cut -d'"' -f2)
+
+# Detect library version
+IMB_VERSION = $(shell grep -e "IMB_VERSION_STR" $(IMB_HDR) | cut -d'"' -f2)
+IMB_VERSION_NUM = $(shell grep -e "IMB_VERSION_NUM" $(IMB_HDR) | cut -d' ' -f3)
+
+ifeq ($(IMB_VERSION),)
+$(error "IPSec_MB version >= 0.53 is required")
+endif
+
+ifeq ($(shell expr $(IMB_VERSION_NUM) \< 0x3400), 1)
+$(error "IPSec_MB version >= 0.53 is required")
+endif
+
+# library source files
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_SNOW3G) += rte_snow3g_pmd.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_SNOW3G) += rte_snow3g_pmd_ops.c
+
+include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/src/spdk/dpdk/drivers/crypto/snow3g/meson.build b/src/spdk/dpdk/drivers/crypto/snow3g/meson.build
new file mode 100644
index 000000000..c1c5fd37f
--- /dev/null
+++ b/src/spdk/dpdk/drivers/crypto/snow3g/meson.build
@@ -0,0 +1,24 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2019-2020 Intel Corporation
+
+IMB_required_ver = '0.53.0'
+lib = cc.find_library('IPSec_MB', required: false)
+if not lib.found()
+	build = false
+	reason = 'missing dependency, "libIPSec_MB"'
+else
+	# version comes with quotes, so we split based on " and take the middle
+	imb_ver = cc.get_define('IMB_VERSION_STR',
+		prefix : '#include<intel-ipsec-mb.h>').split('"')[1]
+
+	if (imb_ver == '') or (imb_ver.version_compare('<' + IMB_required_ver))
+                reason = 'IPSec_MB version >= @0@ is required, found version @1@'.format(
+				IMB_required_ver, imb_ver)
+		build = false
+	endif
+
+endif
+
+ext_deps += lib
+sources = files('rte_snow3g_pmd.c', 'rte_snow3g_pmd_ops.c')
+deps += ['bus_vdev', 'cryptodev']
diff --git a/src/spdk/dpdk/drivers/crypto/snow3g/rte_pmd_snow3g_version.map b/src/spdk/dpdk/drivers/crypto/snow3g/rte_pmd_snow3g_version.map
new file mode 100644
index 000000000..f9f17e4f6
--- /dev/null
+++ b/src/spdk/dpdk/drivers/crypto/snow3g/rte_pmd_snow3g_version.map
@@ -0,0 +1,3 @@
+DPDK_20.0 {
+	local: *;
+};
diff --git a/src/spdk/dpdk/drivers/crypto/snow3g/rte_snow3g_pmd.c b/src/spdk/dpdk/drivers/crypto/snow3g/rte_snow3g_pmd.c
new file mode 100644
index 000000000..c939064d5
--- /dev/null
+++ b/src/spdk/dpdk/drivers/crypto/snow3g/rte_snow3g_pmd.c
@@ -0,0 +1,660 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2018 Intel Corporation
+ */
+
+#include <rte_common.h>
+#include <rte_hexdump.h>
+#include <rte_cryptodev.h>
+#include <rte_cryptodev_pmd.h>
+#include <rte_bus_vdev.h>
+#include <rte_malloc.h>
+#include <rte_cpuflags.h>
+
+#include "snow3g_pmd_private.h"
+
+#define SNOW3G_IV_LENGTH 16
+#define SNOW3G_MAX_BURST 8
+#define BYTE_LEN 8
+
+int snow3g_logtype_driver;
+static uint8_t cryptodev_driver_id;
+
+/** Get xform chain order. */
+static enum snow3g_operation
+snow3g_get_mode(const struct rte_crypto_sym_xform *xform)
+{
+	if (xform == NULL)
+		return SNOW3G_OP_NOT_SUPPORTED;
+
+	if (xform->next)
+		if (xform->next->next != NULL)
+			return SNOW3G_OP_NOT_SUPPORTED;
+
+	if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
+		if (xform->next == NULL)
+			return SNOW3G_OP_ONLY_AUTH;
+		else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
+			return SNOW3G_OP_AUTH_CIPHER;
+		else
+			return SNOW3G_OP_NOT_SUPPORTED;
+	}
+
+	if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
+		if (xform->next == NULL)
+			return SNOW3G_OP_ONLY_CIPHER;
+		else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
+			return SNOW3G_OP_CIPHER_AUTH;
+		else
+			return SNOW3G_OP_NOT_SUPPORTED;
+	}
+
+	return SNOW3G_OP_NOT_SUPPORTED;
+}
+
+
+/** Parse crypto xform chain and set private session parameters. */
+int
+snow3g_set_session_parameters(MB_MGR *mgr, struct snow3g_session *sess,
+		const struct rte_crypto_sym_xform *xform)
+{
+	const struct rte_crypto_sym_xform *auth_xform = NULL;
+	const struct rte_crypto_sym_xform *cipher_xform = NULL;
+	enum snow3g_operation mode;
+
+	/* Select Crypto operation - hash then cipher / cipher then hash */
+	mode = snow3g_get_mode(xform);
+
+	switch (mode) {
+	case SNOW3G_OP_CIPHER_AUTH:
+		auth_xform = xform->next;
+
+		/* Fall-through */
+	case SNOW3G_OP_ONLY_CIPHER:
+		cipher_xform = xform;
+		break;
+	case SNOW3G_OP_AUTH_CIPHER:
+		cipher_xform = xform->next;
+		/* Fall-through */
+	case SNOW3G_OP_ONLY_AUTH:
+		auth_xform = xform;
+		break;
+	case SNOW3G_OP_NOT_SUPPORTED:
+	default:
+		SNOW3G_LOG(ERR, "Unsupported operation chain order parameter");
+		return -ENOTSUP;
+	}
+
+	if (cipher_xform) {
+		/* Only SNOW 3G UEA2 supported */
+		if (cipher_xform->cipher.algo != RTE_CRYPTO_CIPHER_SNOW3G_UEA2)
+			return -ENOTSUP;
+
+		if (cipher_xform->cipher.iv.length != SNOW3G_IV_LENGTH) {
+			SNOW3G_LOG(ERR, "Wrong IV length");
+			return -EINVAL;
+		}
+		if (cipher_xform->cipher.key.length > SNOW3G_MAX_KEY_SIZE) {
+			SNOW3G_LOG(ERR, "Not enough memory to store the key");
+			return -ENOMEM;
+		}
+
+		sess->cipher_iv_offset = cipher_xform->cipher.iv.offset;
+
+		/* Initialize key */
+		IMB_SNOW3G_INIT_KEY_SCHED(mgr, cipher_xform->cipher.key.data,
+					&sess->pKeySched_cipher);
+	}
+
+	if (auth_xform) {
+		/* Only SNOW 3G UIA2 supported */
+		if (auth_xform->auth.algo != RTE_CRYPTO_AUTH_SNOW3G_UIA2)
+			return -ENOTSUP;
+
+		if (auth_xform->auth.digest_length != SNOW3G_DIGEST_LENGTH) {
+			SNOW3G_LOG(ERR, "Wrong digest length");
+			return -EINVAL;
+		}
+		if (auth_xform->auth.key.length > SNOW3G_MAX_KEY_SIZE) {
+			SNOW3G_LOG(ERR, "Not enough memory to store the key");
+			return -ENOMEM;
+		}
+
+		sess->auth_op = auth_xform->auth.op;
+
+		if (auth_xform->auth.iv.length != SNOW3G_IV_LENGTH) {
+			SNOW3G_LOG(ERR, "Wrong IV length");
+			return -EINVAL;
+		}
+		sess->auth_iv_offset = auth_xform->auth.iv.offset;
+
+		/* Initialize key */
+		IMB_SNOW3G_INIT_KEY_SCHED(mgr, auth_xform->auth.key.data,
+					&sess->pKeySched_hash);
+	}
+
+	sess->op = mode;
+
+	return 0;
+}
+
+/** Get SNOW 3G session. */
+static struct snow3g_session *
+snow3g_get_session(struct snow3g_qp *qp, struct rte_crypto_op *op)
+{
+	struct snow3g_session *sess = NULL;
+
+	if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
+		if (likely(op->sym->session != NULL))
+			sess = (struct snow3g_session *)
+					get_sym_session_private_data(
+					op->sym->session,
+					cryptodev_driver_id);
+	} else {
+		void *_sess = NULL;
+		void *_sess_private_data = NULL;
+
+		if (rte_mempool_get(qp->sess_mp, (void **)&_sess))
+			return NULL;
+
+		if (rte_mempool_get(qp->sess_mp_priv,
+				(void **)&_sess_private_data))
+			return NULL;
+
+		sess = (struct snow3g_session *)_sess_private_data;
+
+		if (unlikely(snow3g_set_session_parameters(qp->mgr, sess,
+				op->sym->xform) != 0)) {
+			rte_mempool_put(qp->sess_mp, _sess);
+			rte_mempool_put(qp->sess_mp_priv, _sess_private_data);
+			sess = NULL;
+		}
+		op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
+		set_sym_session_private_data(op->sym->session,
+				cryptodev_driver_id, _sess_private_data);
+	}
+
+	if (unlikely(sess == NULL))
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
+
+
+	return sess;
+}
+
+/** Encrypt/decrypt mbufs with same cipher key. */
+static uint8_t
+process_snow3g_cipher_op(struct snow3g_qp *qp, struct rte_crypto_op **ops,
+		struct snow3g_session *session,
+		uint8_t num_ops)
+{
+	unsigned i;
+	uint8_t processed_ops = 0;
+	const void *src[SNOW3G_MAX_BURST];
+	void *dst[SNOW3G_MAX_BURST];
+	const void *iv[SNOW3G_MAX_BURST];
+	uint32_t num_bytes[SNOW3G_MAX_BURST];
+
+	for (i = 0; i < num_ops; i++) {
+		src[i] = rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) +
+				(ops[i]->sym->cipher.data.offset >> 3);
+		dst[i] = ops[i]->sym->m_dst ?
+			rte_pktmbuf_mtod(ops[i]->sym->m_dst, uint8_t *) +
+				(ops[i]->sym->cipher.data.offset >> 3) :
+			rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) +
+				(ops[i]->sym->cipher.data.offset >> 3);
+		iv[i] = rte_crypto_op_ctod_offset(ops[i], uint8_t *,
+				session->cipher_iv_offset);
+		num_bytes[i] = ops[i]->sym->cipher.data.length >> 3;
+
+		processed_ops++;
+	}
+
+	IMB_SNOW3G_F8_N_BUFFER(qp->mgr, &session->pKeySched_cipher, iv,
+			src, dst, num_bytes, processed_ops);
+
+	return processed_ops;
+}
+
+/** Encrypt/decrypt mbuf (bit level function). */
+static uint8_t
+process_snow3g_cipher_op_bit(struct snow3g_qp *qp,
+		struct rte_crypto_op *op,
+		struct snow3g_session *session)
+{
+	uint8_t *src, *dst;
+	uint8_t *iv;
+	uint32_t length_in_bits, offset_in_bits;
+
+	offset_in_bits = op->sym->cipher.data.offset;
+	src = rte_pktmbuf_mtod(op->sym->m_src, uint8_t *);
+	if (op->sym->m_dst == NULL) {
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		SNOW3G_LOG(ERR, "bit-level in-place not supported\n");
+		return 0;
+	}
+	dst = rte_pktmbuf_mtod(op->sym->m_dst, uint8_t *);
+	iv = rte_crypto_op_ctod_offset(op, uint8_t *,
+				session->cipher_iv_offset);
+	length_in_bits = op->sym->cipher.data.length;
+
+	IMB_SNOW3G_F8_1_BUFFER_BIT(qp->mgr, &session->pKeySched_cipher, iv,
+			src, dst, length_in_bits, offset_in_bits);
+
+	return 1;
+}
+
+/** Generate/verify hash from mbufs with same hash key. */
+static int
+process_snow3g_hash_op(struct snow3g_qp *qp, struct rte_crypto_op **ops,
+		struct snow3g_session *session,
+		uint8_t num_ops)
+{
+	unsigned i;
+	uint8_t processed_ops = 0;
+	uint8_t *src, *dst;
+	uint32_t length_in_bits;
+	uint8_t *iv;
+
+	for (i = 0; i < num_ops; i++) {
+		/* Data must be byte aligned */
+		if ((ops[i]->sym->auth.data.offset % BYTE_LEN) != 0) {
+			ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+			SNOW3G_LOG(ERR, "Offset");
+			break;
+		}
+
+		length_in_bits = ops[i]->sym->auth.data.length;
+
+		src = rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) +
+				(ops[i]->sym->auth.data.offset >> 3);
+		iv = rte_crypto_op_ctod_offset(ops[i], uint8_t *,
+				session->auth_iv_offset);
+
+		if (session->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY) {
+			dst = qp->temp_digest;
+
+			IMB_SNOW3G_F9_1_BUFFER(qp->mgr,
+					&session->pKeySched_hash,
+					iv, src, length_in_bits, dst);
+			/* Verify digest. */
+			if (memcmp(dst, ops[i]->sym->auth.digest.data,
+					SNOW3G_DIGEST_LENGTH) != 0)
+				ops[i]->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
+		} else  {
+			dst = ops[i]->sym->auth.digest.data;
+
+			IMB_SNOW3G_F9_1_BUFFER(qp->mgr,
+					&session->pKeySched_hash,
+					iv, src, length_in_bits, dst);
+		}
+		processed_ops++;
+	}
+
+	return processed_ops;
+}
+
+/** Process a batch of crypto ops which shares the same session. */
+static int
+process_ops(struct rte_crypto_op **ops, struct snow3g_session *session,
+		struct snow3g_qp *qp, uint8_t num_ops,
+		uint16_t *accumulated_enqueued_ops)
+{
+	unsigned i;
+	unsigned enqueued_ops, processed_ops;
+
+#ifdef RTE_LIBRTE_PMD_SNOW3G_DEBUG
+	for (i = 0; i < num_ops; i++) {
+		if (!rte_pktmbuf_is_contiguous(ops[i]->sym->m_src) ||
+				(ops[i]->sym->m_dst != NULL &&
+				!rte_pktmbuf_is_contiguous(
+						ops[i]->sym->m_dst))) {
+			SNOW3G_LOG(ERR, "PMD supports only contiguous mbufs, "
+				"op (%p) provides noncontiguous mbuf as "
+				"source/destination buffer.\n", ops[i]);
+			ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+			return 0;
+		}
+	}
+#endif
+
+	switch (session->op) {
+	case SNOW3G_OP_ONLY_CIPHER:
+		processed_ops = process_snow3g_cipher_op(qp, ops,
+				session, num_ops);
+		break;
+	case SNOW3G_OP_ONLY_AUTH:
+		processed_ops = process_snow3g_hash_op(qp, ops, session,
+				num_ops);
+		break;
+	case SNOW3G_OP_CIPHER_AUTH:
+		processed_ops = process_snow3g_cipher_op(qp, ops, session,
+				num_ops);
+		process_snow3g_hash_op(qp, ops, session, processed_ops);
+		break;
+	case SNOW3G_OP_AUTH_CIPHER:
+		processed_ops = process_snow3g_hash_op(qp, ops, session,
+				num_ops);
+		process_snow3g_cipher_op(qp, ops, session, processed_ops);
+		break;
+	default:
+		/* Operation not supported. */
+		processed_ops = 0;
+	}
+
+	for (i = 0; i < num_ops; i++) {
+		/*
+		 * If there was no error/authentication failure,
+		 * change status to successful.
+		 */
+		if (ops[i]->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
+			ops[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+		/* Free session if a session-less crypto op. */
+		if (ops[i]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
+			memset(session, 0, sizeof(struct snow3g_session));
+			memset(ops[i]->sym->session, 0,
+			rte_cryptodev_sym_get_existing_header_session_size(
+					ops[i]->sym->session));
+			rte_mempool_put(qp->sess_mp_priv, session);
+			rte_mempool_put(qp->sess_mp, ops[i]->sym->session);
+			ops[i]->sym->session = NULL;
+		}
+	}
+
+	enqueued_ops = rte_ring_enqueue_burst(qp->processed_ops,
+			(void **)ops, processed_ops, NULL);
+	qp->qp_stats.enqueued_count += enqueued_ops;
+	*accumulated_enqueued_ops += enqueued_ops;
+
+	return enqueued_ops;
+}
+
+/** Process a crypto op with length/offset in bits. */
+static int
+process_op_bit(struct rte_crypto_op *op, struct snow3g_session *session,
+		struct snow3g_qp *qp, uint16_t *accumulated_enqueued_ops)
+{
+	unsigned enqueued_op, processed_op;
+
+	switch (session->op) {
+	case SNOW3G_OP_ONLY_CIPHER:
+		processed_op = process_snow3g_cipher_op_bit(qp, op,
+				session);
+		break;
+	case SNOW3G_OP_ONLY_AUTH:
+		processed_op = process_snow3g_hash_op(qp, &op, session, 1);
+		break;
+	case SNOW3G_OP_CIPHER_AUTH:
+		processed_op = process_snow3g_cipher_op_bit(qp, op, session);
+		if (processed_op == 1)
+			process_snow3g_hash_op(qp, &op, session, 1);
+		break;
+	case SNOW3G_OP_AUTH_CIPHER:
+		processed_op = process_snow3g_hash_op(qp, &op, session, 1);
+		if (processed_op == 1)
+			process_snow3g_cipher_op_bit(qp, op, session);
+		break;
+	default:
+		/* Operation not supported. */
+		processed_op = 0;
+	}
+
+	/*
+	 * If there was no error/authentication failure,
+	 * change status to successful.
+	 */
+	if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
+		op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+
+	/* Free session if a session-less crypto op. */
+	if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
+		memset(op->sym->session, 0, sizeof(struct snow3g_session));
+		rte_cryptodev_sym_session_free(op->sym->session);
+		op->sym->session = NULL;
+	}
+
+	enqueued_op = rte_ring_enqueue_burst(qp->processed_ops,
+			(void **)&op, processed_op, NULL);
+	qp->qp_stats.enqueued_count += enqueued_op;
+	*accumulated_enqueued_ops += enqueued_op;
+
+	return enqueued_op;
+}
+
+static uint16_t
+snow3g_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
+		uint16_t nb_ops)
+{
+	struct rte_crypto_op *c_ops[SNOW3G_MAX_BURST];
+	struct rte_crypto_op *curr_c_op;
+
+	struct snow3g_session *prev_sess = NULL, *curr_sess = NULL;
+	struct snow3g_qp *qp = queue_pair;
+	unsigned i;
+	uint8_t burst_size = 0;
+	uint16_t enqueued_ops = 0;
+	uint8_t processed_ops;
+
+	for (i = 0; i < nb_ops; i++) {
+		curr_c_op = ops[i];
+
+		/* Set status as enqueued (not processed yet) by default. */
+		curr_c_op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
+
+		curr_sess = snow3g_get_session(qp, curr_c_op);
+		if (unlikely(curr_sess == NULL ||
+				curr_sess->op == SNOW3G_OP_NOT_SUPPORTED)) {
+			curr_c_op->status =
+					RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
+			break;
+		}
+
+		/* If length/offset is at bit-level, process this buffer alone. */
+		if (((curr_c_op->sym->cipher.data.length % BYTE_LEN) != 0)
+				|| ((curr_c_op->sym->cipher.data.offset
+					% BYTE_LEN) != 0)) {
+			/* Process the ops of the previous session. */
+			if (prev_sess != NULL) {
+				processed_ops = process_ops(c_ops, prev_sess,
+				qp, burst_size, &enqueued_ops);
+				if (processed_ops < burst_size) {
+					burst_size = 0;
+					break;
+				}
+
+				burst_size = 0;
+				prev_sess = NULL;
+			}
+
+			processed_ops = process_op_bit(curr_c_op, curr_sess,
+							qp, &enqueued_ops);
+			if (processed_ops != 1)
+				break;
+
+			continue;
+		}
+
+		/* Batch ops that share the same session. */
+		if (prev_sess == NULL) {
+			prev_sess = curr_sess;
+			c_ops[burst_size++] = curr_c_op;
+		} else if (curr_sess == prev_sess) {
+			c_ops[burst_size++] = curr_c_op;
+			/*
+			 * When there are enough ops to process in a batch,
+			 * process them, and start a new batch.
+			 */
+			if (burst_size == SNOW3G_MAX_BURST) {
+				processed_ops = process_ops(c_ops, prev_sess,
+						qp, burst_size, &enqueued_ops);
+				if (processed_ops < burst_size) {
+					burst_size = 0;
+					break;
+				}
+
+				burst_size = 0;
+				prev_sess = NULL;
+			}
+		} else {
+			/*
+			 * Different session, process the ops
+			 * of the previous session.
+			 */
+			processed_ops = process_ops(c_ops, prev_sess,
+					qp, burst_size, &enqueued_ops);
+			if (processed_ops < burst_size) {
+				burst_size = 0;
+				break;
+			}
+
+			burst_size = 0;
+			prev_sess = curr_sess;
+
+			c_ops[burst_size++] = curr_c_op;
+		}
+	}
+
+	if (burst_size != 0) {
+		/* Process the crypto ops of the last session. */
+		processed_ops = process_ops(c_ops, prev_sess,
+				qp, burst_size, &enqueued_ops);
+	}
+
+	qp->qp_stats.enqueue_err_count += nb_ops - enqueued_ops;
+	return enqueued_ops;
+}
+
+static uint16_t
+snow3g_pmd_dequeue_burst(void *queue_pair,
+		struct rte_crypto_op **c_ops, uint16_t nb_ops)
+{
+	struct snow3g_qp *qp = queue_pair;
+
+	unsigned nb_dequeued;
+
+	nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops,
+			(void **)c_ops, nb_ops, NULL);
+	qp->qp_stats.dequeued_count += nb_dequeued;
+
+	return nb_dequeued;
+}
+
+static int cryptodev_snow3g_remove(struct rte_vdev_device *vdev);
+
+static int
+cryptodev_snow3g_create(const char *name,
+			struct rte_vdev_device *vdev,
+			struct rte_cryptodev_pmd_init_params *init_params)
+{
+	struct rte_cryptodev *dev;
+	struct snow3g_private *internals;
+	MB_MGR *mgr;
+
+	dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
+	if (dev == NULL) {
+		SNOW3G_LOG(ERR, "failed to create cryptodev vdev");
+		goto init_error;
+	}
+
+	dev->driver_id = cryptodev_driver_id;
+	dev->dev_ops = rte_snow3g_pmd_ops;
+
+	/* Register RX/TX burst functions for data path. */
+	dev->dequeue_burst = snow3g_pmd_dequeue_burst;
+	dev->enqueue_burst = snow3g_pmd_enqueue_burst;
+
+	dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
+			RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
+			RTE_CRYPTODEV_FF_NON_BYTE_ALIGNED_DATA |
+			RTE_CRYPTODEV_FF_SYM_SESSIONLESS;
+
+	mgr = alloc_mb_mgr(0);
+	if (mgr == NULL)
+		return -ENOMEM;
+
+	if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2)) {
+		dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2;
+		init_mb_mgr_avx2(mgr);
+	} else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX)) {
+		dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX;
+		init_mb_mgr_avx(mgr);
+	} else {
+		dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_SSE;
+		init_mb_mgr_sse(mgr);
+	}
+
+	internals = dev->data->dev_private;
+	internals->mgr = mgr;
+
+	internals->max_nb_queue_pairs = init_params->max_nb_queue_pairs;
+
+	return 0;
+init_error:
+	SNOW3G_LOG(ERR, "driver %s: cryptodev_snow3g_create failed",
+			init_params->name);
+
+	cryptodev_snow3g_remove(vdev);
+	return -EFAULT;
+}
+
+static int
+cryptodev_snow3g_probe(struct rte_vdev_device *vdev)
+{
+	struct rte_cryptodev_pmd_init_params init_params = {
+		"",
+		sizeof(struct snow3g_private),
+		rte_socket_id(),
+		RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
+	};
+	const char *name;
+	const char *input_args;
+
+	name = rte_vdev_device_name(vdev);
+	if (name == NULL)
+		return -EINVAL;
+	input_args = rte_vdev_device_args(vdev);
+
+	rte_cryptodev_pmd_parse_input_args(&init_params, input_args);
+
+	return cryptodev_snow3g_create(name, vdev, &init_params);
+}
+
+static int
+cryptodev_snow3g_remove(struct rte_vdev_device *vdev)
+{
+	struct rte_cryptodev *cryptodev;
+	const char *name;
+	struct snow3g_private *internals;
+
+	name = rte_vdev_device_name(vdev);
+	if (name == NULL)
+		return -EINVAL;
+
+	cryptodev = rte_cryptodev_pmd_get_named_dev(name);
+	if (cryptodev == NULL)
+		return -ENODEV;
+
+	internals = cryptodev->data->dev_private;
+
+	free_mb_mgr(internals->mgr);
+
+	return rte_cryptodev_pmd_destroy(cryptodev);
+}
+
+static struct rte_vdev_driver cryptodev_snow3g_pmd_drv = {
+	.probe = cryptodev_snow3g_probe,
+	.remove = cryptodev_snow3g_remove
+};
+
+static struct cryptodev_driver snow3g_crypto_drv;
+
+RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_SNOW3G_PMD, cryptodev_snow3g_pmd_drv);
+RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_SNOW3G_PMD, cryptodev_snow3g_pmd);
+RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SNOW3G_PMD,
+	"max_nb_queue_pairs=<int> "
+	"socket_id=<int>");
+RTE_PMD_REGISTER_CRYPTO_DRIVER(snow3g_crypto_drv,
+		cryptodev_snow3g_pmd_drv.driver, cryptodev_driver_id);
+
+RTE_INIT(snow3g_init_log)
+{
+	snow3g_logtype_driver = rte_log_register("pmd.crypto.snow3g");
+}
diff --git a/src/spdk/dpdk/drivers/crypto/snow3g/rte_snow3g_pmd_ops.c b/src/spdk/dpdk/drivers/crypto/snow3g/rte_snow3g_pmd_ops.c
new file mode 100644
index 000000000..906a0fe60
--- /dev/null
+++ b/src/spdk/dpdk/drivers/crypto/snow3g/rte_snow3g_pmd_ops.c
@@ -0,0 +1,323 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2018 Intel Corporation
+ */
+
+#include <string.h>
+
+#include <rte_common.h>
+#include <rte_malloc.h>
+#include <rte_cryptodev_pmd.h>
+
+#include "snow3g_pmd_private.h"
+
+static const struct rte_cryptodev_capabilities snow3g_pmd_capabilities[] = {
+	{	/* SNOW 3G (UIA2) */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_SNOW3G_UIA2,
+				.block_size = 16,
+				.key_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				},
+				.digest_size = {
+					.min = 4,
+					.max = 4,
+					.increment = 0
+				},
+				.iv_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				}
+			}, }
+		}, }
+	},
+	{	/* SNOW 3G (UEA2) */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+			{.cipher = {
+				.algo = RTE_CRYPTO_CIPHER_SNOW3G_UEA2,
+				.block_size = 16,
+				.key_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				},
+				.iv_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				}
+			}, }
+		}, }
+	},
+	RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
+};
+
+/** Configure device */
+static int
+snow3g_pmd_config(__rte_unused struct rte_cryptodev *dev,
+		__rte_unused struct rte_cryptodev_config *config)
+{
+	return 0;
+}
+
+/** Start device */
+static int
+snow3g_pmd_start(__rte_unused struct rte_cryptodev *dev)
+{
+	return 0;
+}
+
+/** Stop device */
+static void
+snow3g_pmd_stop(__rte_unused struct rte_cryptodev *dev)
+{
+}
+
+/** Close device */
+static int
+snow3g_pmd_close(__rte_unused struct rte_cryptodev *dev)
+{
+	return 0;
+}
+
+
+/** Get device statistics */
+static void
+snow3g_pmd_stats_get(struct rte_cryptodev *dev,
+		struct rte_cryptodev_stats *stats)
+{
+	int qp_id;
+
+	for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
+		struct snow3g_qp *qp = dev->data->queue_pairs[qp_id];
+
+		stats->enqueued_count += qp->qp_stats.enqueued_count;
+		stats->dequeued_count += qp->qp_stats.dequeued_count;
+
+		stats->enqueue_err_count += qp->qp_stats.enqueue_err_count;
+		stats->dequeue_err_count += qp->qp_stats.dequeue_err_count;
+	}
+}
+
+/** Reset device statistics */
+static void
+snow3g_pmd_stats_reset(struct rte_cryptodev *dev)
+{
+	int qp_id;
+
+	for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
+		struct snow3g_qp *qp = dev->data->queue_pairs[qp_id];
+
+		memset(&qp->qp_stats, 0, sizeof(qp->qp_stats));
+	}
+}
+
+
+/** Get device info */
+static void
+snow3g_pmd_info_get(struct rte_cryptodev *dev,
+		struct rte_cryptodev_info *dev_info)
+{
+	struct snow3g_private *internals = dev->data->dev_private;
+
+	if (dev_info != NULL) {
+		dev_info->driver_id = dev->driver_id;
+		dev_info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
+		/* No limit of number of sessions */
+		dev_info->sym.max_nb_sessions = 0;
+		dev_info->feature_flags = dev->feature_flags;
+		dev_info->capabilities = snow3g_pmd_capabilities;
+	}
+}
+
+/** Release queue pair */
+static int
+snow3g_pmd_qp_release(struct rte_cryptodev *dev, uint16_t qp_id)
+{
+	if (dev->data->queue_pairs[qp_id] != NULL) {
+		struct snow3g_qp *qp = dev->data->queue_pairs[qp_id];
+
+		if (qp->processed_ops)
+			rte_ring_free(qp->processed_ops);
+
+		rte_free(dev->data->queue_pairs[qp_id]);
+		dev->data->queue_pairs[qp_id] = NULL;
+	}
+	return 0;
+}
+
+/** set a unique name for the queue pair based on its name, dev_id and qp_id */
+static int
+snow3g_pmd_qp_set_unique_name(struct rte_cryptodev *dev,
+		struct snow3g_qp *qp)
+{
+	unsigned n = snprintf(qp->name, sizeof(qp->name),
+			"snow3g_pmd_%u_qp_%u",
+			dev->data->dev_id, qp->id);
+
+	if (n >= sizeof(qp->name))
+		return -1;
+
+	return 0;
+}
+
+/** Create a ring to place processed ops on */
+static struct rte_ring *
+snow3g_pmd_qp_create_processed_ops_ring(struct snow3g_qp *qp,
+		unsigned ring_size, int socket_id)
+{
+	struct rte_ring *r;
+
+	r = rte_ring_lookup(qp->name);
+	if (r) {
+		if (rte_ring_get_size(r) >= ring_size) {
+			SNOW3G_LOG(INFO, "Reusing existing ring %s"
+					" for processed packets",
+					 qp->name);
+			return r;
+		}
+
+		SNOW3G_LOG(ERR, "Unable to reuse existing ring %s"
+				" for processed packets",
+				 qp->name);
+		return NULL;
+	}
+
+	return rte_ring_create(qp->name, ring_size, socket_id,
+			RING_F_SP_ENQ | RING_F_SC_DEQ);
+}
+
+/** Setup a queue pair */
+static int
+snow3g_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
+		const struct rte_cryptodev_qp_conf *qp_conf,
+		int socket_id)
+{
+	struct snow3g_qp *qp = NULL;
+	struct snow3g_private *internals = dev->data->dev_private;
+
+	/* Free memory prior to re-allocation if needed. */
+	if (dev->data->queue_pairs[qp_id] != NULL)
+		snow3g_pmd_qp_release(dev, qp_id);
+
+	/* Allocate the queue pair data structure. */
+	qp = rte_zmalloc_socket("SNOW 3G PMD Queue Pair", sizeof(*qp),
+					RTE_CACHE_LINE_SIZE, socket_id);
+	if (qp == NULL)
+		return (-ENOMEM);
+
+	qp->id = qp_id;
+	dev->data->queue_pairs[qp_id] = qp;
+
+	if (snow3g_pmd_qp_set_unique_name(dev, qp))
+		goto qp_setup_cleanup;
+
+	qp->processed_ops = snow3g_pmd_qp_create_processed_ops_ring(qp,
+			qp_conf->nb_descriptors, socket_id);
+	if (qp->processed_ops == NULL)
+		goto qp_setup_cleanup;
+
+	qp->mgr = internals->mgr;
+	qp->sess_mp = qp_conf->mp_session;
+	qp->sess_mp_priv = qp_conf->mp_session_private;
+
+	memset(&qp->qp_stats, 0, sizeof(qp->qp_stats));
+
+	return 0;
+
+qp_setup_cleanup:
+	if (qp)
+		rte_free(qp);
+
+	return -1;
+}
+
+/** Returns the size of the SNOW 3G session structure */
+static unsigned
+snow3g_pmd_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
+{
+	return sizeof(struct snow3g_session);
+}
+
+/** Configure a SNOW 3G session from a crypto xform chain */
+static int
+snow3g_pmd_sym_session_configure(struct rte_cryptodev *dev,
+		struct rte_crypto_sym_xform *xform,
+		struct rte_cryptodev_sym_session *sess,
+		struct rte_mempool *mempool)
+{
+	void *sess_private_data;
+	int ret;
+	struct snow3g_private *internals = dev->data->dev_private;
+
+	if (unlikely(sess == NULL)) {
+		SNOW3G_LOG(ERR, "invalid session struct");
+		return -EINVAL;
+	}
+
+	if (rte_mempool_get(mempool, &sess_private_data)) {
+		SNOW3G_LOG(ERR,
+			"Couldn't get object from session mempool");
+		return -ENOMEM;
+	}
+
+	ret = snow3g_set_session_parameters(internals->mgr,
+					sess_private_data, xform);
+	if (ret != 0) {
+		SNOW3G_LOG(ERR, "failed configure session parameters");
+
+		/* Return session to mempool */
+		rte_mempool_put(mempool, sess_private_data);
+		return ret;
+	}
+
+	set_sym_session_private_data(sess, dev->driver_id,
+		sess_private_data);
+
+	return 0;
+}
+
+/** Clear the memory of session so it doesn't leave key material behind */
+static void
+snow3g_pmd_sym_session_clear(struct rte_cryptodev *dev,
+		struct rte_cryptodev_sym_session *sess)
+{
+	uint8_t index = dev->driver_id;
+	void *sess_priv = get_sym_session_private_data(sess, index);
+
+	/* Zero out the whole structure */
+	if (sess_priv) {
+		memset(sess_priv, 0, sizeof(struct snow3g_session));
+		struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
+		set_sym_session_private_data(sess, index, NULL);
+		rte_mempool_put(sess_mp, sess_priv);
+	}
+}
+
+struct rte_cryptodev_ops snow3g_pmd_ops = {
+		.dev_configure      = snow3g_pmd_config,
+		.dev_start          = snow3g_pmd_start,
+		.dev_stop           = snow3g_pmd_stop,
+		.dev_close          = snow3g_pmd_close,
+
+		.stats_get          = snow3g_pmd_stats_get,
+		.stats_reset        = snow3g_pmd_stats_reset,
+
+		.dev_infos_get      = snow3g_pmd_info_get,
+
+		.queue_pair_setup   = snow3g_pmd_qp_setup,
+		.queue_pair_release = snow3g_pmd_qp_release,
+
+		.sym_session_get_size   = snow3g_pmd_sym_session_get_size,
+		.sym_session_configure  = snow3g_pmd_sym_session_configure,
+		.sym_session_clear      = snow3g_pmd_sym_session_clear
+};
+
+struct rte_cryptodev_ops *rte_snow3g_pmd_ops = &snow3g_pmd_ops;
diff --git a/src/spdk/dpdk/drivers/crypto/snow3g/snow3g_pmd_private.h b/src/spdk/dpdk/drivers/crypto/snow3g/snow3g_pmd_private.h
new file mode 100644
index 000000000..23cf078a9
--- /dev/null
+++ b/src/spdk/dpdk/drivers/crypto/snow3g/snow3g_pmd_private.h
@@ -0,0 +1,84 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2019 Intel Corporation
+ */
+
+#ifndef _SNOW3G_PMD_PRIVATE_H_
+#define _SNOW3G_PMD_PRIVATE_H_
+
+#include <intel-ipsec-mb.h>
+
+#define CRYPTODEV_NAME_SNOW3G_PMD	crypto_snow3g
+/**< SNOW 3G PMD device name */
+
+/** SNOW 3G PMD LOGTYPE DRIVER */
+extern int snow3g_logtype_driver;
+
+#define SNOW3G_LOG(level, fmt, ...)  \
+	rte_log(RTE_LOG_ ## level, snow3g_logtype_driver,  \
+			"%s() line %u: " fmt "\n", __func__, __LINE__,  \
+					## __VA_ARGS__)
+
+#define SNOW3G_DIGEST_LENGTH 4
+#define SNOW3G_MAX_KEY_SIZE  128
+
+/** private data structure for each virtual SNOW 3G device */
+struct snow3g_private {
+	unsigned max_nb_queue_pairs;
+	/**< Max number of queue pairs supported by device */
+	MB_MGR *mgr;
+	/**< Multi-buffer instance */
+};
+
+/** SNOW 3G buffer queue pair */
+struct snow3g_qp {
+	uint16_t id;
+	/**< Queue Pair Identifier */
+	char name[RTE_CRYPTODEV_NAME_MAX_LEN];
+	/**< Unique Queue Pair Name */
+	struct rte_ring *processed_ops;
+	/**< Ring for placing processed ops */
+	struct rte_mempool *sess_mp;
+	/**< Session Mempool */
+	struct rte_mempool *sess_mp_priv;
+	/**< Session Private Data Mempool */
+	struct rte_cryptodev_stats qp_stats;
+	/**< Queue pair statistics */
+	uint8_t temp_digest[SNOW3G_DIGEST_LENGTH];
+	/**< Buffer used to store the digest generated
+	 * by the driver when verifying a digest provided
+	 * by the user (using authentication verify operation)
+	 */
+	MB_MGR *mgr;
+	/**< Multi-buffer instance */
+} __rte_cache_aligned;
+
+enum snow3g_operation {
+	SNOW3G_OP_ONLY_CIPHER,
+	SNOW3G_OP_ONLY_AUTH,
+	SNOW3G_OP_CIPHER_AUTH,
+	SNOW3G_OP_AUTH_CIPHER,
+	SNOW3G_OP_NOT_SUPPORTED
+};
+
+/** SNOW 3G private session structure */
+struct snow3g_session {
+	enum snow3g_operation op;
+	enum rte_crypto_auth_operation auth_op;
+	snow3g_key_schedule_t pKeySched_cipher;
+	snow3g_key_schedule_t pKeySched_hash;
+	uint16_t cipher_iv_offset;
+	uint16_t auth_iv_offset;
+} __rte_cache_aligned;
+
+
+extern int
+snow3g_set_session_parameters(MB_MGR *mgr, struct snow3g_session *sess,
+		const struct rte_crypto_sym_xform *xform);
+
+
+/** device specific operations function pointer structure */
+extern struct rte_cryptodev_ops *rte_snow3g_pmd_ops;
+
+
+
+#endif /* _SNOW3G_PMD_PRIVATE_H_ */