summaryrefslogtreecommitdiffstats
path: root/drivers/crypto/cavium/nitrox
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/crypto/cavium/nitrox')
-rw-r--r--drivers/crypto/cavium/nitrox/Kconfig21
-rw-r--r--drivers/crypto/cavium/nitrox/Makefile15
-rw-r--r--drivers/crypto/cavium/nitrox/nitrox_aead.c567
-rw-r--r--drivers/crypto/cavium/nitrox/nitrox_algs.c24
-rw-r--r--drivers/crypto/cavium/nitrox/nitrox_common.h30
-rw-r--r--drivers/crypto/cavium/nitrox/nitrox_csr.h1439
-rw-r--r--drivers/crypto/cavium/nitrox/nitrox_debugfs.c70
-rw-r--r--drivers/crypto/cavium/nitrox/nitrox_debugfs.h20
-rw-r--r--drivers/crypto/cavium/nitrox/nitrox_dev.h302
-rw-r--r--drivers/crypto/cavium/nitrox/nitrox_hal.c679
-rw-r--r--drivers/crypto/cavium/nitrox/nitrox_hal.h29
-rw-r--r--drivers/crypto/cavium/nitrox/nitrox_isr.c458
-rw-r--r--drivers/crypto/cavium/nitrox/nitrox_isr.h21
-rw-r--r--drivers/crypto/cavium/nitrox/nitrox_lib.c300
-rw-r--r--drivers/crypto/cavium/nitrox/nitrox_main.c583
-rw-r--r--drivers/crypto/cavium/nitrox/nitrox_mbx.c218
-rw-r--r--drivers/crypto/cavium/nitrox/nitrox_mbx.h9
-rw-r--r--drivers/crypto/cavium/nitrox/nitrox_req.h728
-rw-r--r--drivers/crypto/cavium/nitrox/nitrox_reqmgr.c607
-rw-r--r--drivers/crypto/cavium/nitrox/nitrox_skcipher.c552
-rw-r--r--drivers/crypto/cavium/nitrox/nitrox_sriov.c234
21 files changed, 6906 insertions, 0 deletions
diff --git a/drivers/crypto/cavium/nitrox/Kconfig b/drivers/crypto/cavium/nitrox/Kconfig
new file mode 100644
index 000000000..7dc008332
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/Kconfig
@@ -0,0 +1,21 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Cavium NITROX Crypto Device configuration
+#
+config CRYPTO_DEV_NITROX
+ tristate
+ select CRYPTO_SKCIPHER
+ select CRYPTO_AES
+ select CRYPTO_LIB_DES
+ select FW_LOADER
+
+config CRYPTO_DEV_NITROX_CNN55XX
+ tristate "Support for Cavium CNN55XX driver"
+ depends on PCI_MSI && 64BIT
+ select CRYPTO_DEV_NITROX
+ help
+ Support for Cavium NITROX family CNN55XX driver
+ for accelerating crypto workloads.
+
+ To compile this as a module, choose M here: the module
+ will be called n5pf.
diff --git a/drivers/crypto/cavium/nitrox/Makefile b/drivers/crypto/cavium/nitrox/Makefile
new file mode 100644
index 000000000..f83991aaf
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/Makefile
@@ -0,0 +1,15 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_CRYPTO_DEV_NITROX_CNN55XX) += n5pf.o
+
+n5pf-objs := nitrox_main.o \
+ nitrox_isr.o \
+ nitrox_lib.o \
+ nitrox_hal.o \
+ nitrox_reqmgr.o \
+ nitrox_algs.o \
+ nitrox_mbx.o \
+ nitrox_skcipher.o \
+ nitrox_aead.o
+
+n5pf-$(CONFIG_PCI_IOV) += nitrox_sriov.o
+n5pf-$(CONFIG_DEBUG_FS) += nitrox_debugfs.o
diff --git a/drivers/crypto/cavium/nitrox/nitrox_aead.c b/drivers/crypto/cavium/nitrox/nitrox_aead.c
new file mode 100644
index 000000000..c93c4e41d
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/nitrox_aead.c
@@ -0,0 +1,567 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/kernel.h>
+#include <linux/printk.h>
+#include <linux/crypto.h>
+#include <linux/rtnetlink.h>
+
+#include <crypto/aead.h>
+#include <crypto/authenc.h>
+#include <crypto/des.h>
+#include <crypto/internal/aead.h>
+#include <crypto/scatterwalk.h>
+#include <crypto/gcm.h>
+
+#include "nitrox_dev.h"
+#include "nitrox_common.h"
+#include "nitrox_req.h"
+
+#define GCM_AES_SALT_SIZE 4
+
+union gph_p3 {
+ struct {
+#ifdef __BIG_ENDIAN_BITFIELD
+ u16 iv_offset : 8;
+ u16 auth_offset : 8;
+#else
+ u16 auth_offset : 8;
+ u16 iv_offset : 8;
+#endif
+ };
+ u16 param;
+};
+
+static int nitrox_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key,
+ unsigned int keylen)
+{
+ int aes_keylen;
+ struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
+ struct flexi_crypto_context *fctx;
+ union fc_ctx_flags flags;
+
+ aes_keylen = flexi_aes_keylen(keylen);
+ if (aes_keylen < 0)
+ return -EINVAL;
+
+ /* fill crypto context */
+ fctx = nctx->u.fctx;
+ flags.fu = be64_to_cpu(fctx->flags.f);
+ flags.w0.aes_keylen = aes_keylen;
+ fctx->flags.f = cpu_to_be64(flags.fu);
+
+ /* copy enc key to context */
+ memset(&fctx->crypto, 0, sizeof(fctx->crypto));
+ memcpy(fctx->crypto.u.key, key, keylen);
+
+ return 0;
+}
+
+static int nitrox_aead_setauthsize(struct crypto_aead *aead,
+ unsigned int authsize)
+{
+ struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
+ struct flexi_crypto_context *fctx = nctx->u.fctx;
+ union fc_ctx_flags flags;
+
+ flags.fu = be64_to_cpu(fctx->flags.f);
+ flags.w0.mac_len = authsize;
+ fctx->flags.f = cpu_to_be64(flags.fu);
+
+ aead->authsize = authsize;
+
+ return 0;
+}
+
+static int nitrox_aes_gcm_setauthsize(struct crypto_aead *aead,
+ unsigned int authsize)
+{
+ switch (authsize) {
+ case 4:
+ case 8:
+ case 12:
+ case 13:
+ case 14:
+ case 15:
+ case 16:
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ return nitrox_aead_setauthsize(aead, authsize);
+}
+
+static int alloc_src_sglist(struct nitrox_kcrypt_request *nkreq,
+ struct scatterlist *src, char *iv, int ivsize,
+ int buflen)
+{
+ int nents = sg_nents_for_len(src, buflen);
+ int ret;
+
+ if (nents < 0)
+ return nents;
+
+ /* IV entry */
+ nents += 1;
+ /* Allocate buffer to hold IV and input scatterlist array */
+ ret = alloc_src_req_buf(nkreq, nents, ivsize);
+ if (ret)
+ return ret;
+
+ nitrox_creq_copy_iv(nkreq->src, iv, ivsize);
+ nitrox_creq_set_src_sg(nkreq, nents, ivsize, src, buflen);
+
+ return 0;
+}
+
+static int alloc_dst_sglist(struct nitrox_kcrypt_request *nkreq,
+ struct scatterlist *dst, int ivsize, int buflen)
+{
+ int nents = sg_nents_for_len(dst, buflen);
+ int ret;
+
+ if (nents < 0)
+ return nents;
+
+ /* IV, ORH, COMPLETION entries */
+ nents += 3;
+ /* Allocate buffer to hold ORH, COMPLETION and output scatterlist
+ * array
+ */
+ ret = alloc_dst_req_buf(nkreq, nents);
+ if (ret)
+ return ret;
+
+ nitrox_creq_set_orh(nkreq);
+ nitrox_creq_set_comp(nkreq);
+ nitrox_creq_set_dst_sg(nkreq, nents, ivsize, dst, buflen);
+
+ return 0;
+}
+
+static void free_src_sglist(struct nitrox_kcrypt_request *nkreq)
+{
+ kfree(nkreq->src);
+}
+
+static void free_dst_sglist(struct nitrox_kcrypt_request *nkreq)
+{
+ kfree(nkreq->dst);
+}
+
+static int nitrox_set_creq(struct nitrox_aead_rctx *rctx)
+{
+ struct se_crypto_request *creq = &rctx->nkreq.creq;
+ union gph_p3 param3;
+ int ret;
+
+ creq->flags = rctx->flags;
+ creq->gfp = (rctx->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL :
+ GFP_ATOMIC;
+
+ creq->ctrl.value = 0;
+ creq->opcode = FLEXI_CRYPTO_ENCRYPT_HMAC;
+ creq->ctrl.s.arg = rctx->ctrl_arg;
+
+ creq->gph.param0 = cpu_to_be16(rctx->cryptlen);
+ creq->gph.param1 = cpu_to_be16(rctx->cryptlen + rctx->assoclen);
+ creq->gph.param2 = cpu_to_be16(rctx->ivsize + rctx->assoclen);
+ param3.iv_offset = 0;
+ param3.auth_offset = rctx->ivsize;
+ creq->gph.param3 = cpu_to_be16(param3.param);
+
+ creq->ctx_handle = rctx->ctx_handle;
+ creq->ctrl.s.ctxl = sizeof(struct flexi_crypto_context);
+
+ ret = alloc_src_sglist(&rctx->nkreq, rctx->src, rctx->iv, rctx->ivsize,
+ rctx->srclen);
+ if (ret)
+ return ret;
+
+ ret = alloc_dst_sglist(&rctx->nkreq, rctx->dst, rctx->ivsize,
+ rctx->dstlen);
+ if (ret) {
+ free_src_sglist(&rctx->nkreq);
+ return ret;
+ }
+
+ return 0;
+}
+
+static void nitrox_aead_callback(void *arg, int err)
+{
+ struct aead_request *areq = arg;
+ struct nitrox_aead_rctx *rctx = aead_request_ctx(areq);
+
+ free_src_sglist(&rctx->nkreq);
+ free_dst_sglist(&rctx->nkreq);
+ if (err) {
+ pr_err_ratelimited("request failed status 0x%0x\n", err);
+ err = -EINVAL;
+ }
+
+ areq->base.complete(&areq->base, err);
+}
+
+static inline bool nitrox_aes_gcm_assoclen_supported(unsigned int assoclen)
+{
+ if (assoclen <= 512)
+ return true;
+
+ return false;
+}
+
+static int nitrox_aes_gcm_enc(struct aead_request *areq)
+{
+ struct crypto_aead *aead = crypto_aead_reqtfm(areq);
+ struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
+ struct nitrox_aead_rctx *rctx = aead_request_ctx(areq);
+ struct se_crypto_request *creq = &rctx->nkreq.creq;
+ struct flexi_crypto_context *fctx = nctx->u.fctx;
+ int ret;
+
+ if (!nitrox_aes_gcm_assoclen_supported(areq->assoclen))
+ return -EINVAL;
+
+ memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE);
+
+ rctx->cryptlen = areq->cryptlen;
+ rctx->assoclen = areq->assoclen;
+ rctx->srclen = areq->assoclen + areq->cryptlen;
+ rctx->dstlen = rctx->srclen + aead->authsize;
+ rctx->iv = &areq->iv[GCM_AES_SALT_SIZE];
+ rctx->ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE;
+ rctx->flags = areq->base.flags;
+ rctx->ctx_handle = nctx->u.ctx_handle;
+ rctx->src = areq->src;
+ rctx->dst = areq->dst;
+ rctx->ctrl_arg = ENCRYPT;
+ ret = nitrox_set_creq(rctx);
+ if (ret)
+ return ret;
+
+ /* send the crypto request */
+ return nitrox_process_se_request(nctx->ndev, creq, nitrox_aead_callback,
+ areq);
+}
+
+static int nitrox_aes_gcm_dec(struct aead_request *areq)
+{
+ struct crypto_aead *aead = crypto_aead_reqtfm(areq);
+ struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
+ struct nitrox_aead_rctx *rctx = aead_request_ctx(areq);
+ struct se_crypto_request *creq = &rctx->nkreq.creq;
+ struct flexi_crypto_context *fctx = nctx->u.fctx;
+ int ret;
+
+ if (!nitrox_aes_gcm_assoclen_supported(areq->assoclen))
+ return -EINVAL;
+
+ memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE);
+
+ rctx->cryptlen = areq->cryptlen - aead->authsize;
+ rctx->assoclen = areq->assoclen;
+ rctx->srclen = areq->cryptlen + areq->assoclen;
+ rctx->dstlen = rctx->srclen - aead->authsize;
+ rctx->iv = &areq->iv[GCM_AES_SALT_SIZE];
+ rctx->ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE;
+ rctx->flags = areq->base.flags;
+ rctx->ctx_handle = nctx->u.ctx_handle;
+ rctx->src = areq->src;
+ rctx->dst = areq->dst;
+ rctx->ctrl_arg = DECRYPT;
+ ret = nitrox_set_creq(rctx);
+ if (ret)
+ return ret;
+
+ /* send the crypto request */
+ return nitrox_process_se_request(nctx->ndev, creq, nitrox_aead_callback,
+ areq);
+}
+
+static int nitrox_aead_init(struct crypto_aead *aead)
+{
+ struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
+ struct crypto_ctx_hdr *chdr;
+
+ /* get the first device */
+ nctx->ndev = nitrox_get_first_device();
+ if (!nctx->ndev)
+ return -ENODEV;
+
+ /* allocate nitrox crypto context */
+ chdr = crypto_alloc_context(nctx->ndev);
+ if (!chdr) {
+ nitrox_put_device(nctx->ndev);
+ return -ENOMEM;
+ }
+ nctx->chdr = chdr;
+ nctx->u.ctx_handle = (uintptr_t)((u8 *)chdr->vaddr +
+ sizeof(struct ctx_hdr));
+ nctx->u.fctx->flags.f = 0;
+
+ return 0;
+}
+
+static int nitrox_gcm_common_init(struct crypto_aead *aead)
+{
+ int ret;
+ struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
+ union fc_ctx_flags *flags;
+
+ ret = nitrox_aead_init(aead);
+ if (ret)
+ return ret;
+
+ flags = &nctx->u.fctx->flags;
+ flags->w0.cipher_type = CIPHER_AES_GCM;
+ flags->w0.hash_type = AUTH_NULL;
+ flags->w0.iv_source = IV_FROM_DPTR;
+ /* ask microcode to calculate ipad/opad */
+ flags->w0.auth_input_type = 1;
+ flags->f = cpu_to_be64(flags->fu);
+
+ return 0;
+}
+
+static int nitrox_aes_gcm_init(struct crypto_aead *aead)
+{
+ int ret;
+
+ ret = nitrox_gcm_common_init(aead);
+ if (ret)
+ return ret;
+
+ crypto_aead_set_reqsize(aead,
+ sizeof(struct aead_request) +
+ sizeof(struct nitrox_aead_rctx));
+
+ return 0;
+}
+
+static void nitrox_aead_exit(struct crypto_aead *aead)
+{
+ struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
+
+ /* free the nitrox crypto context */
+ if (nctx->u.ctx_handle) {
+ struct flexi_crypto_context *fctx = nctx->u.fctx;
+
+ memzero_explicit(&fctx->crypto, sizeof(struct crypto_keys));
+ memzero_explicit(&fctx->auth, sizeof(struct auth_keys));
+ crypto_free_context((void *)nctx->chdr);
+ }
+ nitrox_put_device(nctx->ndev);
+
+ nctx->u.ctx_handle = 0;
+ nctx->ndev = NULL;
+}
+
+static int nitrox_rfc4106_setkey(struct crypto_aead *aead, const u8 *key,
+ unsigned int keylen)
+{
+ struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
+ struct flexi_crypto_context *fctx = nctx->u.fctx;
+ int ret;
+
+ if (keylen < GCM_AES_SALT_SIZE)
+ return -EINVAL;
+
+ keylen -= GCM_AES_SALT_SIZE;
+ ret = nitrox_aes_gcm_setkey(aead, key, keylen);
+ if (ret)
+ return ret;
+
+ memcpy(fctx->crypto.iv, key + keylen, GCM_AES_SALT_SIZE);
+ return 0;
+}
+
+static int nitrox_rfc4106_setauthsize(struct crypto_aead *aead,
+ unsigned int authsize)
+{
+ switch (authsize) {
+ case 8:
+ case 12:
+ case 16:
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ return nitrox_aead_setauthsize(aead, authsize);
+}
+
+static int nitrox_rfc4106_set_aead_rctx_sglist(struct aead_request *areq)
+{
+ struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq);
+ struct nitrox_aead_rctx *aead_rctx = &rctx->base;
+ unsigned int assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE;
+ struct scatterlist *sg;
+
+ if (areq->assoclen != 16 && areq->assoclen != 20)
+ return -EINVAL;
+
+ scatterwalk_map_and_copy(rctx->assoc, areq->src, 0, assoclen, 0);
+ sg_init_table(rctx->src, 3);
+ sg_set_buf(rctx->src, rctx->assoc, assoclen);
+ sg = scatterwalk_ffwd(rctx->src + 1, areq->src, areq->assoclen);
+ if (sg != rctx->src + 1)
+ sg_chain(rctx->src, 2, sg);
+
+ if (areq->src != areq->dst) {
+ sg_init_table(rctx->dst, 3);
+ sg_set_buf(rctx->dst, rctx->assoc, assoclen);
+ sg = scatterwalk_ffwd(rctx->dst + 1, areq->dst, areq->assoclen);
+ if (sg != rctx->dst + 1)
+ sg_chain(rctx->dst, 2, sg);
+ }
+
+ aead_rctx->src = rctx->src;
+ aead_rctx->dst = (areq->src == areq->dst) ? rctx->src : rctx->dst;
+
+ return 0;
+}
+
+static void nitrox_rfc4106_callback(void *arg, int err)
+{
+ struct aead_request *areq = arg;
+ struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq);
+ struct nitrox_kcrypt_request *nkreq = &rctx->base.nkreq;
+
+ free_src_sglist(nkreq);
+ free_dst_sglist(nkreq);
+ if (err) {
+ pr_err_ratelimited("request failed status 0x%0x\n", err);
+ err = -EINVAL;
+ }
+
+ areq->base.complete(&areq->base, err);
+}
+
+static int nitrox_rfc4106_enc(struct aead_request *areq)
+{
+ struct crypto_aead *aead = crypto_aead_reqtfm(areq);
+ struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
+ struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq);
+ struct nitrox_aead_rctx *aead_rctx = &rctx->base;
+ struct se_crypto_request *creq = &aead_rctx->nkreq.creq;
+ int ret;
+
+ aead_rctx->cryptlen = areq->cryptlen;
+ aead_rctx->assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE;
+ aead_rctx->srclen = aead_rctx->assoclen + aead_rctx->cryptlen;
+ aead_rctx->dstlen = aead_rctx->srclen + aead->authsize;
+ aead_rctx->iv = areq->iv;
+ aead_rctx->ivsize = GCM_RFC4106_IV_SIZE;
+ aead_rctx->flags = areq->base.flags;
+ aead_rctx->ctx_handle = nctx->u.ctx_handle;
+ aead_rctx->ctrl_arg = ENCRYPT;
+
+ ret = nitrox_rfc4106_set_aead_rctx_sglist(areq);
+ if (ret)
+ return ret;
+
+ ret = nitrox_set_creq(aead_rctx);
+ if (ret)
+ return ret;
+
+ /* send the crypto request */
+ return nitrox_process_se_request(nctx->ndev, creq,
+ nitrox_rfc4106_callback, areq);
+}
+
+static int nitrox_rfc4106_dec(struct aead_request *areq)
+{
+ struct crypto_aead *aead = crypto_aead_reqtfm(areq);
+ struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
+ struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq);
+ struct nitrox_aead_rctx *aead_rctx = &rctx->base;
+ struct se_crypto_request *creq = &aead_rctx->nkreq.creq;
+ int ret;
+
+ aead_rctx->cryptlen = areq->cryptlen - aead->authsize;
+ aead_rctx->assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE;
+ aead_rctx->srclen =
+ areq->cryptlen - GCM_RFC4106_IV_SIZE + areq->assoclen;
+ aead_rctx->dstlen = aead_rctx->srclen - aead->authsize;
+ aead_rctx->iv = areq->iv;
+ aead_rctx->ivsize = GCM_RFC4106_IV_SIZE;
+ aead_rctx->flags = areq->base.flags;
+ aead_rctx->ctx_handle = nctx->u.ctx_handle;
+ aead_rctx->ctrl_arg = DECRYPT;
+
+ ret = nitrox_rfc4106_set_aead_rctx_sglist(areq);
+ if (ret)
+ return ret;
+
+ ret = nitrox_set_creq(aead_rctx);
+ if (ret)
+ return ret;
+
+ /* send the crypto request */
+ return nitrox_process_se_request(nctx->ndev, creq,
+ nitrox_rfc4106_callback, areq);
+}
+
+static int nitrox_rfc4106_init(struct crypto_aead *aead)
+{
+ int ret;
+
+ ret = nitrox_gcm_common_init(aead);
+ if (ret)
+ return ret;
+
+ crypto_aead_set_reqsize(aead, sizeof(struct aead_request) +
+ sizeof(struct nitrox_rfc4106_rctx));
+
+ return 0;
+}
+
+static struct aead_alg nitrox_aeads[] = { {
+ .base = {
+ .cra_name = "gcm(aes)",
+ .cra_driver_name = "n5_aes_gcm",
+ .cra_priority = PRIO,
+ .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ },
+ .setkey = nitrox_aes_gcm_setkey,
+ .setauthsize = nitrox_aes_gcm_setauthsize,
+ .encrypt = nitrox_aes_gcm_enc,
+ .decrypt = nitrox_aes_gcm_dec,
+ .init = nitrox_aes_gcm_init,
+ .exit = nitrox_aead_exit,
+ .ivsize = GCM_AES_IV_SIZE,
+ .maxauthsize = AES_BLOCK_SIZE,
+}, {
+ .base = {
+ .cra_name = "rfc4106(gcm(aes))",
+ .cra_driver_name = "n5_rfc4106",
+ .cra_priority = PRIO,
+ .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ },
+ .setkey = nitrox_rfc4106_setkey,
+ .setauthsize = nitrox_rfc4106_setauthsize,
+ .encrypt = nitrox_rfc4106_enc,
+ .decrypt = nitrox_rfc4106_dec,
+ .init = nitrox_rfc4106_init,
+ .exit = nitrox_aead_exit,
+ .ivsize = GCM_RFC4106_IV_SIZE,
+ .maxauthsize = AES_BLOCK_SIZE,
+} };
+
+int nitrox_register_aeads(void)
+{
+ return crypto_register_aeads(nitrox_aeads, ARRAY_SIZE(nitrox_aeads));
+}
+
+void nitrox_unregister_aeads(void)
+{
+ crypto_unregister_aeads(nitrox_aeads, ARRAY_SIZE(nitrox_aeads));
+}
diff --git a/drivers/crypto/cavium/nitrox/nitrox_algs.c b/drivers/crypto/cavium/nitrox/nitrox_algs.c
new file mode 100644
index 000000000..d646ae5f2
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/nitrox_algs.c
@@ -0,0 +1,24 @@
+#include "nitrox_common.h"
+
+int nitrox_crypto_register(void)
+{
+ int err;
+
+ err = nitrox_register_skciphers();
+ if (err)
+ return err;
+
+ err = nitrox_register_aeads();
+ if (err) {
+ nitrox_unregister_skciphers();
+ return err;
+ }
+
+ return 0;
+}
+
+void nitrox_crypto_unregister(void)
+{
+ nitrox_unregister_aeads();
+ nitrox_unregister_skciphers();
+}
diff --git a/drivers/crypto/cavium/nitrox/nitrox_common.h b/drivers/crypto/cavium/nitrox/nitrox_common.h
new file mode 100644
index 000000000..e4be69d7e
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/nitrox_common.h
@@ -0,0 +1,30 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __NITROX_COMMON_H
+#define __NITROX_COMMON_H
+
+#include "nitrox_dev.h"
+#include "nitrox_req.h"
+
+int nitrox_crypto_register(void);
+void nitrox_crypto_unregister(void);
+int nitrox_register_aeads(void);
+void nitrox_unregister_aeads(void);
+int nitrox_register_skciphers(void);
+void nitrox_unregister_skciphers(void);
+void *crypto_alloc_context(struct nitrox_device *ndev);
+void crypto_free_context(void *ctx);
+struct nitrox_device *nitrox_get_first_device(void);
+void nitrox_put_device(struct nitrox_device *ndev);
+
+int nitrox_common_sw_init(struct nitrox_device *ndev);
+void nitrox_common_sw_cleanup(struct nitrox_device *ndev);
+
+void pkt_slc_resp_tasklet(unsigned long data);
+int nitrox_process_se_request(struct nitrox_device *ndev,
+ struct se_crypto_request *req,
+ completion_t cb,
+ void *cb_arg);
+void backlog_qflush_work(struct work_struct *work);
+
+
+#endif /* __NITROX_COMMON_H */
diff --git a/drivers/crypto/cavium/nitrox/nitrox_csr.h b/drivers/crypto/cavium/nitrox/nitrox_csr.h
new file mode 100644
index 000000000..1c8715ae0
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/nitrox_csr.h
@@ -0,0 +1,1439 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __NITROX_CSR_H
+#define __NITROX_CSR_H
+
+#include <asm/byteorder.h>
+#include <linux/types.h>
+
+/* EMU clusters */
+#define NR_CLUSTERS 4
+/* Maximum cores per cluster,
+ * varies based on partname
+ */
+#define AE_CORES_PER_CLUSTER 20
+#define SE_CORES_PER_CLUSTER 16
+
+#define AE_MAX_CORES (AE_CORES_PER_CLUSTER * NR_CLUSTERS)
+#define SE_MAX_CORES (SE_CORES_PER_CLUSTER * NR_CLUSTERS)
+#define ZIP_MAX_CORES 5
+
+/* BIST registers */
+#define EMU_BIST_STATUSX(_i) (0x1402700 + ((_i) * 0x40000))
+#define UCD_BIST_STATUS 0x12C0070
+#define NPS_CORE_BIST_REG 0x10000E8
+#define NPS_CORE_NPC_BIST_REG 0x1000128
+#define NPS_PKT_SLC_BIST_REG 0x1040088
+#define NPS_PKT_IN_BIST_REG 0x1040100
+#define POM_BIST_REG 0x11C0100
+#define BMI_BIST_REG 0x1140080
+#define EFL_CORE_BIST_REGX(_i) (0x1240100 + ((_i) * 0x400))
+#define EFL_TOP_BIST_STAT 0x1241090
+#define BMO_BIST_REG 0x1180080
+#define LBC_BIST_STATUS 0x1200020
+#define PEM_BIST_STATUSX(_i) (0x1080468 | ((_i) << 18))
+
+/* EMU registers */
+#define EMU_SE_ENABLEX(_i) (0x1400000 + ((_i) * 0x40000))
+#define EMU_AE_ENABLEX(_i) (0x1400008 + ((_i) * 0x40000))
+#define EMU_WD_INT_ENA_W1SX(_i) (0x1402318 + ((_i) * 0x40000))
+#define EMU_GE_INT_ENA_W1SX(_i) (0x1402518 + ((_i) * 0x40000))
+#define EMU_FUSE_MAPX(_i) (0x1402708 + ((_i) * 0x40000))
+
+/* UCD registers */
+#define UCD_SE_EID_UCODE_BLOCK_NUMX(_i) (0x12C0000 + ((_i) * 0x1000))
+#define UCD_AE_EID_UCODE_BLOCK_NUMX(_i) (0x12C0008 + ((_i) * 0x800))
+#define UCD_UCODE_LOAD_BLOCK_NUM 0x12C0010
+#define UCD_UCODE_LOAD_IDX_DATAX(_i) (0x12C0018 + ((_i) * 0x20))
+#define UCD_SE_CNTX(_i) (0x12C0040 + ((_i) * 0x1000))
+#define UCD_AE_CNTX(_i) (0x12C0048 + ((_i) * 0x800))
+
+/* AQM registers */
+#define AQM_CTL 0x1300000
+#define AQM_INT 0x1300008
+#define AQM_DBELL_OVF_LO 0x1300010
+#define AQM_DBELL_OVF_HI 0x1300018
+#define AQM_DBELL_OVF_LO_W1S 0x1300020
+#define AQM_DBELL_OVF_LO_ENA_W1C 0x1300028
+#define AQM_DBELL_OVF_LO_ENA_W1S 0x1300030
+#define AQM_DBELL_OVF_HI_W1S 0x1300038
+#define AQM_DBELL_OVF_HI_ENA_W1C 0x1300040
+#define AQM_DBELL_OVF_HI_ENA_W1S 0x1300048
+#define AQM_DMA_RD_ERR_LO 0x1300050
+#define AQM_DMA_RD_ERR_HI 0x1300058
+#define AQM_DMA_RD_ERR_LO_W1S 0x1300060
+#define AQM_DMA_RD_ERR_LO_ENA_W1C 0x1300068
+#define AQM_DMA_RD_ERR_LO_ENA_W1S 0x1300070
+#define AQM_DMA_RD_ERR_HI_W1S 0x1300078
+#define AQM_DMA_RD_ERR_HI_ENA_W1C 0x1300080
+#define AQM_DMA_RD_ERR_HI_ENA_W1S 0x1300088
+#define AQM_EXEC_NA_LO 0x1300090
+#define AQM_EXEC_NA_HI 0x1300098
+#define AQM_EXEC_NA_LO_W1S 0x13000A0
+#define AQM_EXEC_NA_LO_ENA_W1C 0x13000A8
+#define AQM_EXEC_NA_LO_ENA_W1S 0x13000B0
+#define AQM_EXEC_NA_HI_W1S 0x13000B8
+#define AQM_EXEC_NA_HI_ENA_W1C 0x13000C0
+#define AQM_EXEC_NA_HI_ENA_W1S 0x13000C8
+#define AQM_EXEC_ERR_LO 0x13000D0
+#define AQM_EXEC_ERR_HI 0x13000D8
+#define AQM_EXEC_ERR_LO_W1S 0x13000E0
+#define AQM_EXEC_ERR_LO_ENA_W1C 0x13000E8
+#define AQM_EXEC_ERR_LO_ENA_W1S 0x13000F0
+#define AQM_EXEC_ERR_HI_W1S 0x13000F8
+#define AQM_EXEC_ERR_HI_ENA_W1C 0x1300100
+#define AQM_EXEC_ERR_HI_ENA_W1S 0x1300108
+#define AQM_ECC_INT 0x1300110
+#define AQM_ECC_INT_W1S 0x1300118
+#define AQM_ECC_INT_ENA_W1C 0x1300120
+#define AQM_ECC_INT_ENA_W1S 0x1300128
+#define AQM_ECC_CTL 0x1300130
+#define AQM_BIST_STATUS 0x1300138
+#define AQM_CMD_INF_THRX(x) (0x1300400 + ((x) * 0x8))
+#define AQM_CMD_INFX(x) (0x1300800 + ((x) * 0x8))
+#define AQM_GRP_EXECMSK_LOX(x) (0x1300C00 + ((x) * 0x10))
+#define AQM_GRP_EXECMSK_HIX(x) (0x1300C08 + ((x) * 0x10))
+#define AQM_ACTIVITY_STAT_LO 0x1300C80
+#define AQM_ACTIVITY_STAT_HI 0x1300C88
+#define AQM_Q_CMD_PROCX(x) (0x1301000 + ((x) * 0x8))
+#define AQM_PERF_CTL_LO 0x1301400
+#define AQM_PERF_CTL_HI 0x1301408
+#define AQM_PERF_CNT 0x1301410
+
+#define AQMQ_DRBLX(x) (0x20000 + ((x) * 0x40000))
+#define AQMQ_QSZX(x) (0x20008 + ((x) * 0x40000))
+#define AQMQ_BADRX(x) (0x20010 + ((x) * 0x40000))
+#define AQMQ_NXT_CMDX(x) (0x20018 + ((x) * 0x40000))
+#define AQMQ_CMD_CNTX(x) (0x20020 + ((x) * 0x40000))
+#define AQMQ_CMP_THRX(x) (0x20028 + ((x) * 0x40000))
+#define AQMQ_CMP_CNTX(x) (0x20030 + ((x) * 0x40000))
+#define AQMQ_TIM_LDX(x) (0x20038 + ((x) * 0x40000))
+#define AQMQ_TIMERX(x) (0x20040 + ((x) * 0x40000))
+#define AQMQ_ENX(x) (0x20048 + ((x) * 0x40000))
+#define AQMQ_ACTIVITY_STATX(x) (0x20050 + ((x) * 0x40000))
+#define AQM_VF_CMP_STATX(x) (0x28000 + ((x) * 0x40000))
+
+/* NPS core registers */
+#define NPS_CORE_GBL_VFCFG 0x1000000
+#define NPS_CORE_CONTROL 0x1000008
+#define NPS_CORE_INT_ACTIVE 0x1000080
+#define NPS_CORE_INT 0x10000A0
+#define NPS_CORE_INT_ENA_W1S 0x10000B8
+#define NPS_STATS_PKT_DMA_RD_CNT 0x1000180
+#define NPS_STATS_PKT_DMA_WR_CNT 0x1000190
+
+/* NPS packet registers */
+#define NPS_PKT_INT 0x1040018
+#define NPS_PKT_MBOX_INT_LO 0x1040020
+#define NPS_PKT_MBOX_INT_LO_ENA_W1C 0x1040030
+#define NPS_PKT_MBOX_INT_LO_ENA_W1S 0x1040038
+#define NPS_PKT_MBOX_INT_HI 0x1040040
+#define NPS_PKT_MBOX_INT_HI_ENA_W1C 0x1040050
+#define NPS_PKT_MBOX_INT_HI_ENA_W1S 0x1040058
+#define NPS_PKT_IN_RERR_HI 0x1040108
+#define NPS_PKT_IN_RERR_HI_ENA_W1S 0x1040120
+#define NPS_PKT_IN_RERR_LO 0x1040128
+#define NPS_PKT_IN_RERR_LO_ENA_W1S 0x1040140
+#define NPS_PKT_IN_ERR_TYPE 0x1040148
+#define NPS_PKT_IN_ERR_TYPE_ENA_W1S 0x1040160
+#define NPS_PKT_IN_INSTR_CTLX(_i) (0x10060 + ((_i) * 0x40000))
+#define NPS_PKT_IN_INSTR_BADDRX(_i) (0x10068 + ((_i) * 0x40000))
+#define NPS_PKT_IN_INSTR_RSIZEX(_i) (0x10070 + ((_i) * 0x40000))
+#define NPS_PKT_IN_DONE_CNTSX(_i) (0x10080 + ((_i) * 0x40000))
+#define NPS_PKT_IN_INSTR_BAOFF_DBELLX(_i) (0x10078 + ((_i) * 0x40000))
+#define NPS_PKT_IN_INT_LEVELSX(_i) (0x10088 + ((_i) * 0x40000))
+
+#define NPS_PKT_SLC_RERR_HI 0x1040208
+#define NPS_PKT_SLC_RERR_HI_ENA_W1S 0x1040220
+#define NPS_PKT_SLC_RERR_LO 0x1040228
+#define NPS_PKT_SLC_RERR_LO_ENA_W1S 0x1040240
+#define NPS_PKT_SLC_ERR_TYPE 0x1040248
+#define NPS_PKT_SLC_ERR_TYPE_ENA_W1S 0x1040260
+/* Mailbox PF->VF PF Accessible Data registers */
+#define NPS_PKT_MBOX_PF_VF_PFDATAX(_i) (0x1040800 + ((_i) * 0x8))
+#define NPS_PKT_MBOX_VF_PF_PFDATAX(_i) (0x1040C00 + ((_i) * 0x8))
+
+#define NPS_PKT_SLC_CTLX(_i) (0x10000 + ((_i) * 0x40000))
+#define NPS_PKT_SLC_CNTSX(_i) (0x10008 + ((_i) * 0x40000))
+#define NPS_PKT_SLC_INT_LEVELSX(_i) (0x10010 + ((_i) * 0x40000))
+
+/* POM registers */
+#define POM_INT_ENA_W1S 0x11C0018
+#define POM_GRP_EXECMASKX(_i) (0x11C1100 | ((_i) * 8))
+#define POM_INT 0x11C0000
+#define POM_PERF_CTL 0x11CC400
+
+/* BMI registers */
+#define BMI_INT 0x1140000
+#define BMI_CTL 0x1140020
+#define BMI_INT_ENA_W1S 0x1140018
+#define BMI_NPS_PKT_CNT 0x1140070
+
+/* EFL registers */
+#define EFL_CORE_INT_ENA_W1SX(_i) (0x1240018 + ((_i) * 0x400))
+#define EFL_CORE_VF_ERR_INT0X(_i) (0x1240050 + ((_i) * 0x400))
+#define EFL_CORE_VF_ERR_INT0_ENA_W1SX(_i) (0x1240068 + ((_i) * 0x400))
+#define EFL_CORE_VF_ERR_INT1X(_i) (0x1240070 + ((_i) * 0x400))
+#define EFL_CORE_VF_ERR_INT1_ENA_W1SX(_i) (0x1240088 + ((_i) * 0x400))
+#define EFL_CORE_SE_ERR_INTX(_i) (0x12400A0 + ((_i) * 0x400))
+#define EFL_RNM_CTL_STATUS 0x1241800
+#define EFL_CORE_INTX(_i) (0x1240000 + ((_i) * 0x400))
+
+/* BMO registers */
+#define BMO_CTL2 0x1180028
+#define BMO_NPS_SLC_PKT_CNT 0x1180078
+
+/* LBC registers */
+#define LBC_INT 0x1200000
+#define LBC_INVAL_CTL 0x1201010
+#define LBC_PLM_VF1_64_INT 0x1202008
+#define LBC_INVAL_STATUS 0x1202010
+#define LBC_INT_ENA_W1S 0x1203000
+#define LBC_PLM_VF1_64_INT_ENA_W1S 0x1205008
+#define LBC_PLM_VF65_128_INT 0x1206008
+#define LBC_ELM_VF1_64_INT 0x1208000
+#define LBC_PLM_VF65_128_INT_ENA_W1S 0x1209008
+#define LBC_ELM_VF1_64_INT_ENA_W1S 0x120B000
+#define LBC_ELM_VF65_128_INT 0x120C000
+#define LBC_ELM_VF65_128_INT_ENA_W1S 0x120F000
+
+#define RST_BOOT 0x10C1600
+#define FUS_DAT1 0x10C1408
+
+/* PEM registers */
+#define PEM0_INT 0x1080428
+
+/**
+ * struct ucd_core_eid_ucode_block_num - Core Eid to Ucode Blk Mapping Registers
+ * @ucode_len: Ucode length identifier 32KB or 64KB
+ * @ucode_blk: Ucode Block Number
+ */
+union ucd_core_eid_ucode_block_num {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz_4_63 : 60;
+ u64 ucode_len : 1;
+ u64 ucode_blk : 3;
+#else
+ u64 ucode_blk : 3;
+ u64 ucode_len : 1;
+ u64 raz_4_63 : 60;
+#endif
+ };
+};
+
+/**
+ * struct aqm_grp_execmsk_lo - Available AE engines for the group
+ * @exec_0_to_39: AE engines 0 to 39 status
+ */
+union aqm_grp_execmsk_lo {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz_40_63 : 24;
+ u64 exec_0_to_39 : 40;
+#else
+ u64 exec_0_to_39 : 40;
+ u64 raz_40_63 : 24;
+#endif
+ };
+};
+
+/**
+ * struct aqm_grp_execmsk_hi - Available AE engines for the group
+ * @exec_40_to_79: AE engines 40 to 79 status
+ */
+union aqm_grp_execmsk_hi {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz_40_63 : 24;
+ u64 exec_40_to_79 : 40;
+#else
+ u64 exec_40_to_79 : 40;
+ u64 raz_40_63 : 24;
+#endif
+ };
+};
+
+/**
+ * struct aqmq_drbl - AQM Queue Doorbell Counter Registers
+ * @dbell_count: Doorbell Counter
+ */
+union aqmq_drbl {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz_32_63 : 32;
+ u64 dbell_count : 32;
+#else
+ u64 dbell_count : 32;
+ u64 raz_32_63 : 32;
+#endif
+ };
+};
+
+/**
+ * struct aqmq_qsz - AQM Queue Host Queue Size Registers
+ * @host_queue_size: Size, in numbers of 'aqmq_command_s' command
+ * of the Host Ring.
+ */
+union aqmq_qsz {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz_32_63 : 32;
+ u64 host_queue_size : 32;
+#else
+ u64 host_queue_size : 32;
+ u64 raz_32_63 : 32;
+#endif
+ };
+};
+
+/**
+ * struct aqmq_cmp_thr - AQM Queue Commands Completed Threshold Registers
+ * @commands_completed_threshold: Count of 'aqmq_command_s' commands executed
+ * by AE engines for which completion interrupt is asserted.
+ */
+union aqmq_cmp_thr {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz_32_63 : 32;
+ u64 commands_completed_threshold : 32;
+#else
+ u64 commands_completed_threshold : 32;
+ u64 raz_32_63 : 32;
+#endif
+ };
+};
+
+/**
+ * struct aqmq_cmp_cnt - AQM Queue Commands Completed Count Registers
+ * @resend: Bit to request completion interrupt Resend.
+ * @completion_status: Command completion status of the ring.
+ * @commands_completed_count: Count of 'aqmq_command_s' commands executed by
+ * AE engines.
+ */
+union aqmq_cmp_cnt {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz_34_63 : 30;
+ u64 resend : 1;
+ u64 completion_status : 1;
+ u64 commands_completed_count : 32;
+#else
+ u64 commands_completed_count : 32;
+ u64 completion_status : 1;
+ u64 resend : 1;
+ u64 raz_34_63 : 30;
+#endif
+ };
+};
+
+/**
+ * struct aqmq_en - AQM Queue Enable Registers
+ * @queue_status: 1 = AQMQ is enabled, 0 = AQMQ is disabled
+ */
+union aqmq_en {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz_1_63 : 63;
+ u64 queue_enable : 1;
+#else
+ u64 queue_enable : 1;
+ u64 raz_1_63 : 63;
+#endif
+ };
+};
+
+/**
+ * struct aqmq_activity_stat - AQM Queue Activity Status Registers
+ * @queue_active: 1 = AQMQ is active, 0 = AQMQ is quiescent
+ */
+union aqmq_activity_stat {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz_1_63 : 63;
+ u64 queue_active : 1;
+#else
+ u64 queue_active : 1;
+ u64 raz_1_63 : 63;
+#endif
+ };
+};
+
+/**
+ * struct emu_fuse_map - EMU Fuse Map Registers
+ * @ae_fuse: Fuse settings for AE 19..0
+ * @se_fuse: Fuse settings for SE 15..0
+ *
+ * A set bit indicates the unit is fuse disabled.
+ */
+union emu_fuse_map {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 valid : 1;
+ u64 raz_52_62 : 11;
+ u64 ae_fuse : 20;
+ u64 raz_16_31 : 16;
+ u64 se_fuse : 16;
+#else
+ u64 se_fuse : 16;
+ u64 raz_16_31 : 16;
+ u64 ae_fuse : 20;
+ u64 raz_52_62 : 11;
+ u64 valid : 1;
+#endif
+ } s;
+};
+
+/**
+ * struct emu_se_enable - Symmetric Engine Enable Registers
+ * @enable: Individual enables for each of the clusters
+ * 16 symmetric engines.
+ */
+union emu_se_enable {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz : 48;
+ u64 enable : 16;
+#else
+ u64 enable : 16;
+ u64 raz : 48;
+#endif
+ } s;
+};
+
+/**
+ * struct emu_ae_enable - EMU Asymmetric engines.
+ * @enable: Individual enables for each of the cluster's
+ * 20 Asymmetric Engines.
+ */
+union emu_ae_enable {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz : 44;
+ u64 enable : 20;
+#else
+ u64 enable : 20;
+ u64 raz : 44;
+#endif
+ } s;
+};
+
+/**
+ * struct emu_wd_int_ena_w1s - EMU Interrupt Enable Registers
+ * @ae_wd: Reads or sets enable for EMU(0..3)_WD_INT[AE_WD]
+ * @se_wd: Reads or sets enable for EMU(0..3)_WD_INT[SE_WD]
+ */
+union emu_wd_int_ena_w1s {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz2 : 12;
+ u64 ae_wd : 20;
+ u64 raz1 : 16;
+ u64 se_wd : 16;
+#else
+ u64 se_wd : 16;
+ u64 raz1 : 16;
+ u64 ae_wd : 20;
+ u64 raz2 : 12;
+#endif
+ } s;
+};
+
+/**
+ * struct emu_ge_int_ena_w1s - EMU Interrupt Enable set registers
+ * @ae_ge: Reads or sets enable for EMU(0..3)_GE_INT[AE_GE]
+ * @se_ge: Reads or sets enable for EMU(0..3)_GE_INT[SE_GE]
+ */
+union emu_ge_int_ena_w1s {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz_52_63 : 12;
+ u64 ae_ge : 20;
+ u64 raz_16_31: 16;
+ u64 se_ge : 16;
+#else
+ u64 se_ge : 16;
+ u64 raz_16_31: 16;
+ u64 ae_ge : 20;
+ u64 raz_52_63 : 12;
+#endif
+ } s;
+};
+
+/**
+ * struct nps_pkt_slc_ctl - Solicited Packet Out Control Registers
+ * @rh: Indicates whether to remove or include the response header
+ * 1 = Include, 0 = Remove
+ * @z: If set, 8 trailing 0x00 bytes will be added to the end of the
+ * outgoing packet.
+ * @enb: Enable for this port.
+ */
+union nps_pkt_slc_ctl {
+ u64 value;
+ struct {
+#if defined(__BIG_ENDIAN_BITFIELD)
+ u64 raz : 61;
+ u64 rh : 1;
+ u64 z : 1;
+ u64 enb : 1;
+#else
+ u64 enb : 1;
+ u64 z : 1;
+ u64 rh : 1;
+ u64 raz : 61;
+#endif
+ } s;
+};
+
+/**
+ * struct nps_pkt_slc_cnts - Solicited Packet Out Count Registers
+ * @slc_int: Returns a 1 when:
+ * NPS_PKT_SLC(i)_CNTS[CNT] > NPS_PKT_SLC(i)_INT_LEVELS[CNT], or
+ * NPS_PKT_SLC(i)_CNTS[TIMER] > NPS_PKT_SLC(i)_INT_LEVELS[TIMET].
+ * To clear the bit, the CNTS register must be written to clear.
+ * @in_int: Returns a 1 when:
+ * NPS_PKT_IN(i)_DONE_CNTS[CNT] > NPS_PKT_IN(i)_INT_LEVELS[CNT].
+ * To clear the bit, the DONE_CNTS register must be written to clear.
+ * @mbox_int: Returns a 1 when:
+ * NPS_PKT_MBOX_PF_VF(i)_INT[INTR] is set. To clear the bit,
+ * write NPS_PKT_MBOX_PF_VF(i)_INT[INTR] with 1.
+ * @timer: Timer, incremented every 2048 coprocessor clock cycles
+ * when [CNT] is not zero. The hardware clears both [TIMER] and
+ * [INT] when [CNT] goes to 0.
+ * @cnt: Packet counter. Hardware adds to [CNT] as it sends packets out.
+ * On a write to this CSR, hardware subtracts the amount written to the
+ * [CNT] field from [CNT].
+ */
+union nps_pkt_slc_cnts {
+ u64 value;
+ struct {
+#if defined(__BIG_ENDIAN_BITFIELD)
+ u64 slc_int : 1;
+ u64 uns_int : 1;
+ u64 in_int : 1;
+ u64 mbox_int : 1;
+ u64 resend : 1;
+ u64 raz : 5;
+ u64 timer : 22;
+ u64 cnt : 32;
+#else
+ u64 cnt : 32;
+ u64 timer : 22;
+ u64 raz : 5;
+ u64 resend : 1;
+ u64 mbox_int : 1;
+ u64 in_int : 1;
+ u64 uns_int : 1;
+ u64 slc_int : 1;
+#endif
+ } s;
+};
+
+/**
+ * struct nps_pkt_slc_int_levels - Solicited Packet Out Interrupt Levels
+ * Registers.
+ * @bmode: Determines whether NPS_PKT_SLC_CNTS[CNT] is a byte or
+ * packet counter.
+ * @timet: Output port counter time interrupt threshold.
+ * @cnt: Output port counter interrupt threshold.
+ */
+union nps_pkt_slc_int_levels {
+ u64 value;
+ struct {
+#if defined(__BIG_ENDIAN_BITFIELD)
+ u64 bmode : 1;
+ u64 raz : 9;
+ u64 timet : 22;
+ u64 cnt : 32;
+#else
+ u64 cnt : 32;
+ u64 timet : 22;
+ u64 raz : 9;
+ u64 bmode : 1;
+#endif
+ } s;
+};
+
+/**
+ * struct nps_pkt_inst - NPS Packet Interrupt Register
+ * @in_err: Set when any NPS_PKT_IN_RERR_HI/LO bit and
+ * corresponding NPS_PKT_IN_RERR_*_ENA_* bit are bot set.
+ * @uns_err: Set when any NSP_PKT_UNS_RERR_HI/LO bit and
+ * corresponding NPS_PKT_UNS_RERR_*_ENA_* bit are both set.
+ * @slc_er: Set when any NSP_PKT_SLC_RERR_HI/LO bit and
+ * corresponding NPS_PKT_SLC_RERR_*_ENA_* bit are both set.
+ */
+union nps_pkt_int {
+ u64 value;
+ struct {
+#if defined(__BIG_ENDIAN_BITFIELD)
+ u64 raz : 54;
+ u64 uns_wto : 1;
+ u64 in_err : 1;
+ u64 uns_err : 1;
+ u64 slc_err : 1;
+ u64 in_dbe : 1;
+ u64 in_sbe : 1;
+ u64 uns_dbe : 1;
+ u64 uns_sbe : 1;
+ u64 slc_dbe : 1;
+ u64 slc_sbe : 1;
+#else
+ u64 slc_sbe : 1;
+ u64 slc_dbe : 1;
+ u64 uns_sbe : 1;
+ u64 uns_dbe : 1;
+ u64 in_sbe : 1;
+ u64 in_dbe : 1;
+ u64 slc_err : 1;
+ u64 uns_err : 1;
+ u64 in_err : 1;
+ u64 uns_wto : 1;
+ u64 raz : 54;
+#endif
+ } s;
+};
+
+/**
+ * struct nps_pkt_in_done_cnts - Input instruction ring counts registers
+ * @slc_cnt: Returns a 1 when:
+ * NPS_PKT_SLC(i)_CNTS[CNT] > NPS_PKT_SLC(i)_INT_LEVELS[CNT], or
+ * NPS_PKT_SLC(i)_CNTS[TIMER] > NPS_PKT_SCL(i)_INT_LEVELS[TIMET]
+ * To clear the bit, the CNTS register must be
+ * written to clear the underlying condition
+ * @uns_int: Return a 1 when:
+ * NPS_PKT_UNS(i)_CNTS[CNT] > NPS_PKT_UNS(i)_INT_LEVELS[CNT], or
+ * NPS_PKT_UNS(i)_CNTS[TIMER] > NPS_PKT_UNS(i)_INT_LEVELS[TIMET]
+ * To clear the bit, the CNTS register must be
+ * written to clear the underlying condition
+ * @in_int: Returns a 1 when:
+ * NPS_PKT_IN(i)_DONE_CNTS[CNT] > NPS_PKT_IN(i)_INT_LEVELS[CNT]
+ * To clear the bit, the DONE_CNTS register
+ * must be written to clear the underlying condition
+ * @mbox_int: Returns a 1 when:
+ * NPS_PKT_MBOX_PF_VF(i)_INT[INTR] is set.
+ * To clear the bit, write NPS_PKT_MBOX_PF_VF(i)_INT[INTR]
+ * with 1.
+ * @resend: A write of 1 will resend an MSI-X interrupt message if any
+ * of the following conditions are true for this ring "i".
+ * NPS_PKT_SLC(i)_CNTS[CNT] > NPS_PKT_SLC(i)_INT_LEVELS[CNT]
+ * NPS_PKT_SLC(i)_CNTS[TIMER] > NPS_PKT_SLC(i)_INT_LEVELS[TIMET]
+ * NPS_PKT_UNS(i)_CNTS[CNT] > NPS_PKT_UNS(i)_INT_LEVELS[CNT]
+ * NPS_PKT_UNS(i)_CNTS[TIMER] > NPS_PKT_UNS(i)_INT_LEVELS[TIMET]
+ * NPS_PKT_IN(i)_DONE_CNTS[CNT] > NPS_PKT_IN(i)_INT_LEVELS[CNT]
+ * NPS_PKT_MBOX_PF_VF(i)_INT[INTR] is set
+ * @cnt: Packet counter. Hardware adds to [CNT] as it reads
+ * packets. On a write to this CSR, hardware substracts the
+ * amount written to the [CNT] field from [CNT], which will
+ * clear PKT_IN(i)_INT_STATUS[INTR] if [CNT] becomes <=
+ * NPS_PKT_IN(i)_INT_LEVELS[CNT]. This register should be
+ * cleared before enabling a ring by reading the current
+ * value and writing it back.
+ */
+union nps_pkt_in_done_cnts {
+ u64 value;
+ struct {
+#if defined(__BIG_ENDIAN_BITFIELD)
+ u64 slc_int : 1;
+ u64 uns_int : 1;
+ u64 in_int : 1;
+ u64 mbox_int : 1;
+ u64 resend : 1;
+ u64 raz : 27;
+ u64 cnt : 32;
+#else
+ u64 cnt : 32;
+ u64 raz : 27;
+ u64 resend : 1;
+ u64 mbox_int : 1;
+ u64 in_int : 1;
+ u64 uns_int : 1;
+ u64 slc_int : 1;
+#endif
+ } s;
+};
+
+/**
+ * struct nps_pkt_in_instr_ctl - Input Instruction Ring Control Registers.
+ * @is64b: If 1, the ring uses 64-byte instructions. If 0, the
+ * ring uses 32-byte instructions.
+ * @enb: Enable for the input ring.
+ */
+union nps_pkt_in_instr_ctl {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz : 62;
+ u64 is64b : 1;
+ u64 enb : 1;
+#else
+ u64 enb : 1;
+ u64 is64b : 1;
+ u64 raz : 62;
+#endif
+ } s;
+};
+
+/**
+ * struct nps_pkt_in_instr_rsize - Input instruction ring size registers
+ * @rsize: Ring size (number of instructions)
+ */
+union nps_pkt_in_instr_rsize {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz : 32;
+ u64 rsize : 32;
+#else
+ u64 rsize : 32;
+ u64 raz : 32;
+#endif
+ } s;
+};
+
+/**
+ * struct nps_pkt_in_instr_baoff_dbell - Input instruction ring
+ * base address offset and doorbell registers
+ * @aoff: Address offset. The offset from the NPS_PKT_IN_INSTR_BADDR
+ * where the next pointer is read.
+ * @dbell: Pointer list doorbell count. Write operations to this field
+ * increments the present value here. Read operations return the
+ * present value.
+ */
+union nps_pkt_in_instr_baoff_dbell {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 aoff : 32;
+ u64 dbell : 32;
+#else
+ u64 dbell : 32;
+ u64 aoff : 32;
+#endif
+ } s;
+};
+
+/**
+ * struct nps_core_int_ena_w1s - NPS core interrupt enable set register
+ * @host_nps_wr_err: Reads or sets enable for
+ * NPS_CORE_INT[HOST_NPS_WR_ERR].
+ * @npco_dma_malform: Reads or sets enable for
+ * NPS_CORE_INT[NPCO_DMA_MALFORM].
+ * @exec_wr_timeout: Reads or sets enable for
+ * NPS_CORE_INT[EXEC_WR_TIMEOUT].
+ * @host_wr_timeout: Reads or sets enable for
+ * NPS_CORE_INT[HOST_WR_TIMEOUT].
+ * @host_wr_err: Reads or sets enable for
+ * NPS_CORE_INT[HOST_WR_ERR]
+ */
+union nps_core_int_ena_w1s {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz4 : 55;
+ u64 host_nps_wr_err : 1;
+ u64 npco_dma_malform : 1;
+ u64 exec_wr_timeout : 1;
+ u64 host_wr_timeout : 1;
+ u64 host_wr_err : 1;
+ u64 raz3 : 1;
+ u64 raz2 : 1;
+ u64 raz1 : 1;
+ u64 raz0 : 1;
+#else
+ u64 raz0 : 1;
+ u64 raz1 : 1;
+ u64 raz2 : 1;
+ u64 raz3 : 1;
+ u64 host_wr_err : 1;
+ u64 host_wr_timeout : 1;
+ u64 exec_wr_timeout : 1;
+ u64 npco_dma_malform : 1;
+ u64 host_nps_wr_err : 1;
+ u64 raz4 : 55;
+#endif
+ } s;
+};
+
+/**
+ * struct nps_core_gbl_vfcfg - Global VF Configuration Register.
+ * @ilk_disable: When set, this bit indicates that the ILK interface has
+ * been disabled.
+ * @obaf: BMO allocation control
+ * 0 = allocate per queue
+ * 1 = allocate per VF
+ * @ibaf: BMI allocation control
+ * 0 = allocate per queue
+ * 1 = allocate per VF
+ * @zaf: ZIP allocation control
+ * 0 = allocate per queue
+ * 1 = allocate per VF
+ * @aeaf: AE allocation control
+ * 0 = allocate per queue
+ * 1 = allocate per VF
+ * @seaf: SE allocation control
+ * 0 = allocation per queue
+ * 1 = allocate per VF
+ * @cfg: VF/PF mode.
+ */
+union nps_core_gbl_vfcfg {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz :55;
+ u64 ilk_disable :1;
+ u64 obaf :1;
+ u64 ibaf :1;
+ u64 zaf :1;
+ u64 aeaf :1;
+ u64 seaf :1;
+ u64 cfg :3;
+#else
+ u64 cfg :3;
+ u64 seaf :1;
+ u64 aeaf :1;
+ u64 zaf :1;
+ u64 ibaf :1;
+ u64 obaf :1;
+ u64 ilk_disable :1;
+ u64 raz :55;
+#endif
+ } s;
+};
+
+/**
+ * struct nps_core_int_active - NPS Core Interrupt Active Register
+ * @resend: Resend MSI-X interrupt if needs to handle interrupts
+ * Sofware can set this bit and then exit the ISR.
+ * @ocla: Set when any OCLA(0)_INT and corresponding OCLA(0_INT_ENA_W1C
+ * bit are set
+ * @mbox: Set when any NPS_PKT_MBOX_INT_LO/HI and corresponding
+ * NPS_PKT_MBOX_INT_LO_ENA_W1C/HI_ENA_W1C bits are set
+ * @emu: bit i is set in [EMU] when any EMU(i)_INT bit is set
+ * @bmo: Set when any BMO_INT bit is set
+ * @bmi: Set when any BMI_INT bit is set or when any non-RO
+ * BMI_INT and corresponding BMI_INT_ENA_W1C bits are both set
+ * @aqm: Set when any AQM_INT bit is set
+ * @zqm: Set when any ZQM_INT bit is set
+ * @efl: Set when any EFL_INT RO bit is set or when any non-RO EFL_INT
+ * and corresponding EFL_INT_ENA_W1C bits are both set
+ * @ilk: Set when any ILK_INT bit is set
+ * @lbc: Set when any LBC_INT RO bit is set or when any non-RO LBC_INT
+ * and corresponding LBC_INT_ENA_W1C bits are bot set
+ * @pem: Set when any PEM(0)_INT RO bit is set or when any non-RO
+ * PEM(0)_INT and corresponding PEM(0)_INT_ENA_W1C bit are both set
+ * @ucd: Set when any UCD_INT bit is set
+ * @zctl: Set when any ZIP_INT RO bit is set or when any non-RO ZIP_INT
+ * and corresponding ZIP_INT_ENA_W1C bits are both set
+ * @lbm: Set when any LBM_INT bit is set
+ * @nps_pkt: Set when any NPS_PKT_INT bit is set
+ * @nps_core: Set when any NPS_CORE_INT RO bit is set or when non-RO
+ * NPS_CORE_INT and corresponding NSP_CORE_INT_ENA_W1C bits are both set
+ */
+union nps_core_int_active {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 resend : 1;
+ u64 raz : 43;
+ u64 ocla : 1;
+ u64 mbox : 1;
+ u64 emu : 4;
+ u64 bmo : 1;
+ u64 bmi : 1;
+ u64 aqm : 1;
+ u64 zqm : 1;
+ u64 efl : 1;
+ u64 ilk : 1;
+ u64 lbc : 1;
+ u64 pem : 1;
+ u64 pom : 1;
+ u64 ucd : 1;
+ u64 zctl : 1;
+ u64 lbm : 1;
+ u64 nps_pkt : 1;
+ u64 nps_core : 1;
+#else
+ u64 nps_core : 1;
+ u64 nps_pkt : 1;
+ u64 lbm : 1;
+ u64 zctl: 1;
+ u64 ucd : 1;
+ u64 pom : 1;
+ u64 pem : 1;
+ u64 lbc : 1;
+ u64 ilk : 1;
+ u64 efl : 1;
+ u64 zqm : 1;
+ u64 aqm : 1;
+ u64 bmi : 1;
+ u64 bmo : 1;
+ u64 emu : 4;
+ u64 mbox : 1;
+ u64 ocla : 1;
+ u64 raz : 43;
+ u64 resend : 1;
+#endif
+ } s;
+};
+
+/**
+ * struct efl_core_int - EFL Interrupt Registers
+ * @epci_decode_err: EPCI decoded a transacation that was unknown
+ * This error should only occurred when there is a micrcode/SE error
+ * and should be considered fatal
+ * @ae_err: An AE uncorrectable error occurred.
+ * See EFL_CORE(0..3)_AE_ERR_INT
+ * @se_err: An SE uncorrectable error occurred.
+ * See EFL_CORE(0..3)_SE_ERR_INT
+ * @dbe: Double-bit error occurred in EFL
+ * @sbe: Single-bit error occurred in EFL
+ * @d_left: Asserted when new POM-Header-BMI-data is
+ * being sent to an Exec, and that Exec has Not read all BMI
+ * data associated with the previous POM header
+ * @len_ovr: Asserted when an Exec-Read is issued that is more than
+ * 14 greater in length that the BMI data left to be read
+ */
+union efl_core_int {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz : 57;
+ u64 epci_decode_err : 1;
+ u64 ae_err : 1;
+ u64 se_err : 1;
+ u64 dbe : 1;
+ u64 sbe : 1;
+ u64 d_left : 1;
+ u64 len_ovr : 1;
+#else
+ u64 len_ovr : 1;
+ u64 d_left : 1;
+ u64 sbe : 1;
+ u64 dbe : 1;
+ u64 se_err : 1;
+ u64 ae_err : 1;
+ u64 epci_decode_err : 1;
+ u64 raz : 57;
+#endif
+ } s;
+};
+
+/**
+ * struct efl_core_int_ena_w1s - EFL core interrupt enable set register
+ * @epci_decode_err: Reads or sets enable for
+ * EFL_CORE(0..3)_INT[EPCI_DECODE_ERR].
+ * @d_left: Reads or sets enable for
+ * EFL_CORE(0..3)_INT[D_LEFT].
+ * @len_ovr: Reads or sets enable for
+ * EFL_CORE(0..3)_INT[LEN_OVR].
+ */
+union efl_core_int_ena_w1s {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz_7_63 : 57;
+ u64 epci_decode_err : 1;
+ u64 raz_2_5 : 4;
+ u64 d_left : 1;
+ u64 len_ovr : 1;
+#else
+ u64 len_ovr : 1;
+ u64 d_left : 1;
+ u64 raz_2_5 : 4;
+ u64 epci_decode_err : 1;
+ u64 raz_7_63 : 57;
+#endif
+ } s;
+};
+
+/**
+ * struct efl_rnm_ctl_status - RNM Control and Status Register
+ * @ent_sel: Select input to RNM FIFO
+ * @exp_ent: Exported entropy enable for random number generator
+ * @rng_rst: Reset to RNG. Setting this bit to 1 cancels the generation
+ * of the current random number.
+ * @rnm_rst: Reset the RNM. Setting this bit to 1 clears all sorted numbers
+ * in the random number memory.
+ * @rng_en: Enabled the output of the RNG.
+ * @ent_en: Entropy enable for random number generator.
+ */
+union efl_rnm_ctl_status {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz_9_63 : 55;
+ u64 ent_sel : 4;
+ u64 exp_ent : 1;
+ u64 rng_rst : 1;
+ u64 rnm_rst : 1;
+ u64 rng_en : 1;
+ u64 ent_en : 1;
+#else
+ u64 ent_en : 1;
+ u64 rng_en : 1;
+ u64 rnm_rst : 1;
+ u64 rng_rst : 1;
+ u64 exp_ent : 1;
+ u64 ent_sel : 4;
+ u64 raz_9_63 : 55;
+#endif
+ } s;
+};
+
+/**
+ * struct bmi_ctl - BMI control register
+ * @ilk_hdrq_thrsh: Maximum number of header queue locations
+ * that ILK packets may consume. When the threshold is
+ * exceeded ILK_XOFF is sent to the BMI_X2P_ARB.
+ * @nps_hdrq_thrsh: Maximum number of header queue locations
+ * that NPS packets may consume. When the threshold is
+ * exceeded NPS_XOFF is sent to the BMI_X2P_ARB.
+ * @totl_hdrq_thrsh: Maximum number of header queue locations
+ * that the sum of ILK and NPS packets may consume.
+ * @ilk_free_thrsh: Maximum number of buffers that ILK packet
+ * flows may consume before ILK_XOFF is sent to the BMI_X2P_ARB.
+ * @nps_free_thrsh: Maximum number of buffers that NPS packet
+ * flows may consume before NPS XOFF is sent to the BMI_X2p_ARB.
+ * @totl_free_thrsh: Maximum number of buffers that bot ILK and NPS
+ * packet flows may consume before both NPS_XOFF and ILK_XOFF
+ * are asserted to the BMI_X2P_ARB.
+ * @max_pkt_len: Maximum packet length, integral number of 256B
+ * buffers.
+ */
+union bmi_ctl {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz_56_63 : 8;
+ u64 ilk_hdrq_thrsh : 8;
+ u64 nps_hdrq_thrsh : 8;
+ u64 totl_hdrq_thrsh : 8;
+ u64 ilk_free_thrsh : 8;
+ u64 nps_free_thrsh : 8;
+ u64 totl_free_thrsh : 8;
+ u64 max_pkt_len : 8;
+#else
+ u64 max_pkt_len : 8;
+ u64 totl_free_thrsh : 8;
+ u64 nps_free_thrsh : 8;
+ u64 ilk_free_thrsh : 8;
+ u64 totl_hdrq_thrsh : 8;
+ u64 nps_hdrq_thrsh : 8;
+ u64 ilk_hdrq_thrsh : 8;
+ u64 raz_56_63 : 8;
+#endif
+ } s;
+};
+
+/**
+ * struct bmi_int_ena_w1s - BMI interrupt enable set register
+ * @ilk_req_oflw: Reads or sets enable for
+ * BMI_INT[ILK_REQ_OFLW].
+ * @nps_req_oflw: Reads or sets enable for
+ * BMI_INT[NPS_REQ_OFLW].
+ * @fpf_undrrn: Reads or sets enable for
+ * BMI_INT[FPF_UNDRRN].
+ * @eop_err_ilk: Reads or sets enable for
+ * BMI_INT[EOP_ERR_ILK].
+ * @eop_err_nps: Reads or sets enable for
+ * BMI_INT[EOP_ERR_NPS].
+ * @sop_err_ilk: Reads or sets enable for
+ * BMI_INT[SOP_ERR_ILK].
+ * @sop_err_nps: Reads or sets enable for
+ * BMI_INT[SOP_ERR_NPS].
+ * @pkt_rcv_err_ilk: Reads or sets enable for
+ * BMI_INT[PKT_RCV_ERR_ILK].
+ * @pkt_rcv_err_nps: Reads or sets enable for
+ * BMI_INT[PKT_RCV_ERR_NPS].
+ * @max_len_err_ilk: Reads or sets enable for
+ * BMI_INT[MAX_LEN_ERR_ILK].
+ * @max_len_err_nps: Reads or sets enable for
+ * BMI_INT[MAX_LEN_ERR_NPS].
+ */
+union bmi_int_ena_w1s {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz_13_63 : 51;
+ u64 ilk_req_oflw : 1;
+ u64 nps_req_oflw : 1;
+ u64 raz_10 : 1;
+ u64 raz_9 : 1;
+ u64 fpf_undrrn : 1;
+ u64 eop_err_ilk : 1;
+ u64 eop_err_nps : 1;
+ u64 sop_err_ilk : 1;
+ u64 sop_err_nps : 1;
+ u64 pkt_rcv_err_ilk : 1;
+ u64 pkt_rcv_err_nps : 1;
+ u64 max_len_err_ilk : 1;
+ u64 max_len_err_nps : 1;
+#else
+ u64 max_len_err_nps : 1;
+ u64 max_len_err_ilk : 1;
+ u64 pkt_rcv_err_nps : 1;
+ u64 pkt_rcv_err_ilk : 1;
+ u64 sop_err_nps : 1;
+ u64 sop_err_ilk : 1;
+ u64 eop_err_nps : 1;
+ u64 eop_err_ilk : 1;
+ u64 fpf_undrrn : 1;
+ u64 raz_9 : 1;
+ u64 raz_10 : 1;
+ u64 nps_req_oflw : 1;
+ u64 ilk_req_oflw : 1;
+ u64 raz_13_63 : 51;
+#endif
+ } s;
+};
+
+/**
+ * struct bmo_ctl2 - BMO Control2 Register
+ * @arb_sel: Determines P2X Arbitration
+ * @ilk_buf_thrsh: Maximum number of buffers that the
+ * ILK packet flows may consume before ILK XOFF is
+ * asserted to the POM.
+ * @nps_slc_buf_thrsh: Maximum number of buffers that the
+ * NPS_SLC packet flow may consume before NPS_SLC XOFF is
+ * asserted to the POM.
+ * @nps_uns_buf_thrsh: Maximum number of buffers that the
+ * NPS_UNS packet flow may consume before NPS_UNS XOFF is
+ * asserted to the POM.
+ * @totl_buf_thrsh: Maximum number of buffers that ILK, NPS_UNS and
+ * NPS_SLC packet flows may consume before NPS_UNS XOFF, NSP_SLC and
+ * ILK_XOFF are all asserted POM.
+ */
+union bmo_ctl2 {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 arb_sel : 1;
+ u64 raz_32_62 : 31;
+ u64 ilk_buf_thrsh : 8;
+ u64 nps_slc_buf_thrsh : 8;
+ u64 nps_uns_buf_thrsh : 8;
+ u64 totl_buf_thrsh : 8;
+#else
+ u64 totl_buf_thrsh : 8;
+ u64 nps_uns_buf_thrsh : 8;
+ u64 nps_slc_buf_thrsh : 8;
+ u64 ilk_buf_thrsh : 8;
+ u64 raz_32_62 : 31;
+ u64 arb_sel : 1;
+#endif
+ } s;
+};
+
+/**
+ * struct pom_int_ena_w1s - POM interrupt enable set register
+ * @illegal_intf: Reads or sets enable for POM_INT[ILLEGAL_INTF].
+ * @illegal_dport: Reads or sets enable for POM_INT[ILLEGAL_DPORT].
+ */
+union pom_int_ena_w1s {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz2 : 60;
+ u64 illegal_intf : 1;
+ u64 illegal_dport : 1;
+ u64 raz1 : 1;
+ u64 raz0 : 1;
+#else
+ u64 raz0 : 1;
+ u64 raz1 : 1;
+ u64 illegal_dport : 1;
+ u64 illegal_intf : 1;
+ u64 raz2 : 60;
+#endif
+ } s;
+};
+
+/**
+ * struct lbc_inval_ctl - LBC invalidation control register
+ * @wait_timer: Wait timer for wait state. [WAIT_TIMER] must
+ * always be written with its reset value.
+ * @cam_inval_start: Software should write [CAM_INVAL_START]=1
+ * to initiate an LBC cache invalidation. After this, software
+ * should read LBC_INVAL_STATUS until LBC_INVAL_STATUS[DONE] is set.
+ * LBC hardware clears [CAVM_INVAL_START] before software can
+ * observed LBC_INVAL_STATUS[DONE] to be set
+ */
+union lbc_inval_ctl {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz2 : 48;
+ u64 wait_timer : 8;
+ u64 raz1 : 6;
+ u64 cam_inval_start : 1;
+ u64 raz0 : 1;
+#else
+ u64 raz0 : 1;
+ u64 cam_inval_start : 1;
+ u64 raz1 : 6;
+ u64 wait_timer : 8;
+ u64 raz2 : 48;
+#endif
+ } s;
+};
+
+/**
+ * struct lbc_int_ena_w1s - LBC interrupt enable set register
+ * @cam_hard_err: Reads or sets enable for LBC_INT[CAM_HARD_ERR].
+ * @cam_inval_abort: Reads or sets enable for LBC_INT[CAM_INVAL_ABORT].
+ * @over_fetch_err: Reads or sets enable for LBC_INT[OVER_FETCH_ERR].
+ * @cache_line_to_err: Reads or sets enable for
+ * LBC_INT[CACHE_LINE_TO_ERR].
+ * @cam_soft_err: Reads or sets enable for
+ * LBC_INT[CAM_SOFT_ERR].
+ * @dma_rd_err: Reads or sets enable for
+ * LBC_INT[DMA_RD_ERR].
+ */
+union lbc_int_ena_w1s {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz_10_63 : 54;
+ u64 cam_hard_err : 1;
+ u64 cam_inval_abort : 1;
+ u64 over_fetch_err : 1;
+ u64 cache_line_to_err : 1;
+ u64 raz_2_5 : 4;
+ u64 cam_soft_err : 1;
+ u64 dma_rd_err : 1;
+#else
+ u64 dma_rd_err : 1;
+ u64 cam_soft_err : 1;
+ u64 raz_2_5 : 4;
+ u64 cache_line_to_err : 1;
+ u64 over_fetch_err : 1;
+ u64 cam_inval_abort : 1;
+ u64 cam_hard_err : 1;
+ u64 raz_10_63 : 54;
+#endif
+ } s;
+};
+
+/**
+ * struct lbc_int - LBC interrupt summary register
+ * @cam_hard_err: indicates a fatal hardware error.
+ * It requires system reset.
+ * When [CAM_HARD_ERR] is set, LBC stops logging any new information in
+ * LBC_POM_MISS_INFO_LOG,
+ * LBC_POM_MISS_ADDR_LOG,
+ * LBC_EFL_MISS_INFO_LOG, and
+ * LBC_EFL_MISS_ADDR_LOG.
+ * Software should sample them.
+ * @cam_inval_abort: indicates a fatal hardware error.
+ * System reset is required.
+ * @over_fetch_err: indicates a fatal hardware error
+ * System reset is required
+ * @cache_line_to_err: is a debug feature.
+ * This timeout interrupt bit tells the software that
+ * a cacheline in LBC has non-zero usage and the context
+ * has not been used for greater than the
+ * LBC_TO_CNT[TO_CNT] time interval.
+ * @sbe: Memory SBE error. This is recoverable via ECC.
+ * See LBC_ECC_INT for more details.
+ * @dbe: Memory DBE error. This is a fatal and requires a
+ * system reset.
+ * @pref_dat_len_mismatch_err: Summary bit for context length
+ * mismatch errors.
+ * @rd_dat_len_mismatch_err: Summary bit for SE read data length
+ * greater than data prefect length errors.
+ * @cam_soft_err: is recoverable. Software must complete a
+ * LBC_INVAL_CTL[CAM_INVAL_START] invalidation sequence and
+ * then clear [CAM_SOFT_ERR].
+ * @dma_rd_err: A context prefect read of host memory returned with
+ * a read error.
+ */
+union lbc_int {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz_10_63 : 54;
+ u64 cam_hard_err : 1;
+ u64 cam_inval_abort : 1;
+ u64 over_fetch_err : 1;
+ u64 cache_line_to_err : 1;
+ u64 sbe : 1;
+ u64 dbe : 1;
+ u64 pref_dat_len_mismatch_err : 1;
+ u64 rd_dat_len_mismatch_err : 1;
+ u64 cam_soft_err : 1;
+ u64 dma_rd_err : 1;
+#else
+ u64 dma_rd_err : 1;
+ u64 cam_soft_err : 1;
+ u64 rd_dat_len_mismatch_err : 1;
+ u64 pref_dat_len_mismatch_err : 1;
+ u64 dbe : 1;
+ u64 sbe : 1;
+ u64 cache_line_to_err : 1;
+ u64 over_fetch_err : 1;
+ u64 cam_inval_abort : 1;
+ u64 cam_hard_err : 1;
+ u64 raz_10_63 : 54;
+#endif
+ } s;
+};
+
+/**
+ * struct lbc_inval_status: LBC Invalidation status register
+ * @cam_clean_entry_complete_cnt: The number of entries that are
+ * cleaned up successfully.
+ * @cam_clean_entry_cnt: The number of entries that have the CAM
+ * inval command issued.
+ * @cam_inval_state: cam invalidation FSM state
+ * @cam_inval_abort: cam invalidation abort
+ * @cam_rst_rdy: lbc_cam reset ready
+ * @done: LBC clears [DONE] when
+ * LBC_INVAL_CTL[CAM_INVAL_START] is written with a one,
+ * and sets [DONE] when it completes the invalidation
+ * sequence.
+ */
+union lbc_inval_status {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz3 : 23;
+ u64 cam_clean_entry_complete_cnt : 9;
+ u64 raz2 : 7;
+ u64 cam_clean_entry_cnt : 9;
+ u64 raz1 : 5;
+ u64 cam_inval_state : 3;
+ u64 raz0 : 5;
+ u64 cam_inval_abort : 1;
+ u64 cam_rst_rdy : 1;
+ u64 done : 1;
+#else
+ u64 done : 1;
+ u64 cam_rst_rdy : 1;
+ u64 cam_inval_abort : 1;
+ u64 raz0 : 5;
+ u64 cam_inval_state : 3;
+ u64 raz1 : 5;
+ u64 cam_clean_entry_cnt : 9;
+ u64 raz2 : 7;
+ u64 cam_clean_entry_complete_cnt : 9;
+ u64 raz3 : 23;
+#endif
+ } s;
+};
+
+/**
+ * struct rst_boot: RST Boot Register
+ * @jtcsrdis: when set, internal CSR access via JTAG TAP controller
+ * is disabled
+ * @jt_tst_mode: JTAG test mode
+ * @io_supply: I/O power supply setting based on IO_VDD_SELECT pin:
+ * 0x1 = 1.8V
+ * 0x2 = 2.5V
+ * 0x4 = 3.3V
+ * All other values are reserved
+ * @pnr_mul: clock multiplier
+ * @lboot: last boot cause mask, resets only with PLL_DC_OK
+ * @rboot: determines whether core 0 remains in reset after
+ * chip cold or warm or soft reset
+ * @rboot_pin: read only access to REMOTE_BOOT pin
+ */
+union rst_boot {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz_63 : 1;
+ u64 jtcsrdis : 1;
+ u64 raz_59_61 : 3;
+ u64 jt_tst_mode : 1;
+ u64 raz_40_57 : 18;
+ u64 io_supply : 3;
+ u64 raz_30_36 : 7;
+ u64 pnr_mul : 6;
+ u64 raz_12_23 : 12;
+ u64 lboot : 10;
+ u64 rboot : 1;
+ u64 rboot_pin : 1;
+#else
+ u64 rboot_pin : 1;
+ u64 rboot : 1;
+ u64 lboot : 10;
+ u64 raz_12_23 : 12;
+ u64 pnr_mul : 6;
+ u64 raz_30_36 : 7;
+ u64 io_supply : 3;
+ u64 raz_40_57 : 18;
+ u64 jt_tst_mode : 1;
+ u64 raz_59_61 : 3;
+ u64 jtcsrdis : 1;
+ u64 raz_63 : 1;
+#endif
+ };
+};
+
+/**
+ * struct fus_dat1: Fuse Data 1 Register
+ * @pll_mul: main clock PLL multiplier hardware limit
+ * @pll_half_dis: main clock PLL control
+ * @efus_lck: efuse lockdown
+ * @zip_info: ZIP information
+ * @bar2_sz_conf: when zero, BAR2 size conforms to
+ * PCIe specification
+ * @efus_ign: efuse ignore
+ * @nozip: ZIP disable
+ * @pll_alt_matrix: select alternate PLL matrix
+ * @pll_bwadj_denom: select CLKF denominator for
+ * BWADJ value
+ * @chip_id: chip ID
+ */
+union fus_dat1 {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz_57_63 : 7;
+ u64 pll_mul : 3;
+ u64 pll_half_dis : 1;
+ u64 raz_43_52 : 10;
+ u64 efus_lck : 3;
+ u64 raz_26_39 : 14;
+ u64 zip_info : 5;
+ u64 bar2_sz_conf : 1;
+ u64 efus_ign : 1;
+ u64 nozip : 1;
+ u64 raz_11_17 : 7;
+ u64 pll_alt_matrix : 1;
+ u64 pll_bwadj_denom : 2;
+ u64 chip_id : 8;
+#else
+ u64 chip_id : 8;
+ u64 pll_bwadj_denom : 2;
+ u64 pll_alt_matrix : 1;
+ u64 raz_11_17 : 7;
+ u64 nozip : 1;
+ u64 efus_ign : 1;
+ u64 bar2_sz_conf : 1;
+ u64 zip_info : 5;
+ u64 raz_26_39 : 14;
+ u64 efus_lck : 3;
+ u64 raz_43_52 : 10;
+ u64 pll_half_dis : 1;
+ u64 pll_mul : 3;
+ u64 raz_57_63 : 7;
+#endif
+ };
+};
+
+#endif /* __NITROX_CSR_H */
diff --git a/drivers/crypto/cavium/nitrox/nitrox_debugfs.c b/drivers/crypto/cavium/nitrox/nitrox_debugfs.c
new file mode 100644
index 000000000..741572a01
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/nitrox_debugfs.c
@@ -0,0 +1,70 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/seq_file.h>
+#include <linux/debugfs.h>
+
+#include "nitrox_csr.h"
+#include "nitrox_debugfs.h"
+#include "nitrox_dev.h"
+
+static int firmware_show(struct seq_file *s, void *v)
+{
+ struct nitrox_device *ndev = s->private;
+
+ seq_printf(s, "Version: %s\n", ndev->hw.fw_name[0]);
+ seq_printf(s, "Version: %s\n", ndev->hw.fw_name[1]);
+ return 0;
+}
+
+DEFINE_SHOW_ATTRIBUTE(firmware);
+
+static int device_show(struct seq_file *s, void *v)
+{
+ struct nitrox_device *ndev = s->private;
+
+ seq_printf(s, "NITROX [%d]\n", ndev->idx);
+ seq_printf(s, " Part Name: %s\n", ndev->hw.partname);
+ seq_printf(s, " Frequency: %d MHz\n", ndev->hw.freq);
+ seq_printf(s, " Device ID: 0x%0x\n", ndev->hw.device_id);
+ seq_printf(s, " Revision ID: 0x%0x\n", ndev->hw.revision_id);
+ seq_printf(s, " Cores: [AE=%u SE=%u ZIP=%u]\n",
+ ndev->hw.ae_cores, ndev->hw.se_cores, ndev->hw.zip_cores);
+
+ return 0;
+}
+
+DEFINE_SHOW_ATTRIBUTE(device);
+
+static int stats_show(struct seq_file *s, void *v)
+{
+ struct nitrox_device *ndev = s->private;
+
+ seq_printf(s, "NITROX [%d] Request Statistics\n", ndev->idx);
+ seq_printf(s, " Posted: %llu\n",
+ (u64)atomic64_read(&ndev->stats.posted));
+ seq_printf(s, " Completed: %llu\n",
+ (u64)atomic64_read(&ndev->stats.completed));
+ seq_printf(s, " Dropped: %llu\n",
+ (u64)atomic64_read(&ndev->stats.dropped));
+
+ return 0;
+}
+
+DEFINE_SHOW_ATTRIBUTE(stats);
+
+void nitrox_debugfs_exit(struct nitrox_device *ndev)
+{
+ debugfs_remove_recursive(ndev->debugfs_dir);
+ ndev->debugfs_dir = NULL;
+}
+
+void nitrox_debugfs_init(struct nitrox_device *ndev)
+{
+ struct dentry *dir;
+
+ dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
+
+ ndev->debugfs_dir = dir;
+ debugfs_create_file("firmware", 0400, dir, ndev, &firmware_fops);
+ debugfs_create_file("device", 0400, dir, ndev, &device_fops);
+ debugfs_create_file("stats", 0400, dir, ndev, &stats_fops);
+}
diff --git a/drivers/crypto/cavium/nitrox/nitrox_debugfs.h b/drivers/crypto/cavium/nitrox/nitrox_debugfs.h
new file mode 100644
index 000000000..09c4cf251
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/nitrox_debugfs.h
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __NITROX_DEBUGFS_H
+#define __NITROX_DEBUGFS_H
+
+#include "nitrox_dev.h"
+
+#ifdef CONFIG_DEBUG_FS
+void nitrox_debugfs_init(struct nitrox_device *ndev);
+void nitrox_debugfs_exit(struct nitrox_device *ndev);
+#else
+static inline void nitrox_debugfs_init(struct nitrox_device *ndev)
+{
+}
+
+static inline void nitrox_debugfs_exit(struct nitrox_device *ndev)
+{
+}
+#endif /* !CONFIG_DEBUG_FS */
+
+#endif /* __NITROX_DEBUGFS_H */
diff --git a/drivers/crypto/cavium/nitrox/nitrox_dev.h b/drivers/crypto/cavium/nitrox/nitrox_dev.h
new file mode 100644
index 000000000..c2d0c23fb
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/nitrox_dev.h
@@ -0,0 +1,302 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __NITROX_DEV_H
+#define __NITROX_DEV_H
+
+#include <linux/dma-mapping.h>
+#include <linux/interrupt.h>
+#include <linux/pci.h>
+#include <linux/if.h>
+
+#define VERSION_LEN 32
+/* Maximum queues in PF mode */
+#define MAX_PF_QUEUES 64
+/* Maximum device queues */
+#define MAX_DEV_QUEUES (MAX_PF_QUEUES)
+/* Maximum UCD Blocks */
+#define CNN55XX_MAX_UCD_BLOCKS 8
+
+/**
+ * struct nitrox_cmdq - NITROX command queue
+ * @cmd_qlock: command queue lock
+ * @resp_qlock: response queue lock
+ * @backlog_qlock: backlog queue lock
+ * @ndev: NITROX device
+ * @response_head: submitted request list
+ * @backlog_head: backlog queue
+ * @dbell_csr_addr: doorbell register address for this queue
+ * @compl_cnt_csr_addr: completion count register address of the slc port
+ * @base: command queue base address
+ * @dma: dma address of the base
+ * @pending_count: request pending at device
+ * @backlog_count: backlog request count
+ * @write_idx: next write index for the command
+ * @instr_size: command size
+ * @qno: command queue number
+ * @qsize: command queue size
+ * @unalign_base: unaligned base address
+ * @unalign_dma: unaligned dma address
+ */
+struct nitrox_cmdq {
+ spinlock_t cmd_qlock;
+ spinlock_t resp_qlock;
+ spinlock_t backlog_qlock;
+
+ struct nitrox_device *ndev;
+ struct list_head response_head;
+ struct list_head backlog_head;
+
+ u8 __iomem *dbell_csr_addr;
+ u8 __iomem *compl_cnt_csr_addr;
+ u8 *base;
+ dma_addr_t dma;
+
+ struct work_struct backlog_qflush;
+
+ atomic_t pending_count;
+ atomic_t backlog_count;
+
+ int write_idx;
+ u8 instr_size;
+ u8 qno;
+ u32 qsize;
+
+ u8 *unalign_base;
+ dma_addr_t unalign_dma;
+};
+
+/**
+ * struct nitrox_hw - NITROX hardware information
+ * @partname: partname ex: CNN55xxx-xxx
+ * @fw_name: firmware version
+ * @freq: NITROX frequency
+ * @vendor_id: vendor ID
+ * @device_id: device ID
+ * @revision_id: revision ID
+ * @se_cores: number of symmetric cores
+ * @ae_cores: number of asymmetric cores
+ * @zip_cores: number of zip cores
+ */
+struct nitrox_hw {
+ char partname[IFNAMSIZ * 2];
+ char fw_name[CNN55XX_MAX_UCD_BLOCKS][VERSION_LEN];
+
+ int freq;
+ u16 vendor_id;
+ u16 device_id;
+ u8 revision_id;
+
+ u8 se_cores;
+ u8 ae_cores;
+ u8 zip_cores;
+};
+
+struct nitrox_stats {
+ atomic64_t posted;
+ atomic64_t completed;
+ atomic64_t dropped;
+};
+
+#define IRQ_NAMESZ 32
+
+struct nitrox_q_vector {
+ char name[IRQ_NAMESZ];
+ bool valid;
+ int ring;
+ struct tasklet_struct resp_tasklet;
+ union {
+ struct nitrox_cmdq *cmdq;
+ struct nitrox_device *ndev;
+ };
+};
+
+enum mcode_type {
+ MCODE_TYPE_INVALID,
+ MCODE_TYPE_AE,
+ MCODE_TYPE_SE_SSL,
+ MCODE_TYPE_SE_IPSEC,
+};
+
+/**
+ * mbox_msg - Mailbox message data
+ * @type: message type
+ * @opcode: message opcode
+ * @data: message data
+ */
+union mbox_msg {
+ u64 value;
+ struct {
+ u64 type: 2;
+ u64 opcode: 6;
+ u64 data: 58;
+ };
+ struct {
+ u64 type: 2;
+ u64 opcode: 6;
+ u64 chipid: 8;
+ u64 vfid: 8;
+ } id;
+ struct {
+ u64 type: 2;
+ u64 opcode: 6;
+ u64 count: 4;
+ u64 info: 40;
+ u64 next_se_grp: 3;
+ u64 next_ae_grp: 3;
+ } mcode_info;
+};
+
+/**
+ * nitrox_vfdev - NITROX VF device instance in PF
+ * @state: VF device state
+ * @vfno: VF number
+ * @nr_queues: number of queues enabled in VF
+ * @ring: ring to communicate with VF
+ * @msg: Mailbox message data from VF
+ * @mbx_resp: Mailbox counters
+ */
+struct nitrox_vfdev {
+ atomic_t state;
+ int vfno;
+ int nr_queues;
+ int ring;
+ union mbox_msg msg;
+ atomic64_t mbx_resp;
+};
+
+/**
+ * struct nitrox_iov - SR-IOV information
+ * @num_vfs: number of VF(s) enabled
+ * @max_vf_queues: Maximum number of queues allowed for VF
+ * @vfdev: VF(s) devices
+ * @pf2vf_wq: workqueue for PF2VF communication
+ * @msix: MSI-X entry for PF in SR-IOV case
+ */
+struct nitrox_iov {
+ int num_vfs;
+ int max_vf_queues;
+ struct nitrox_vfdev *vfdev;
+ struct workqueue_struct *pf2vf_wq;
+ struct msix_entry msix;
+};
+
+/*
+ * NITROX Device states
+ */
+enum ndev_state {
+ __NDEV_NOT_READY,
+ __NDEV_READY,
+ __NDEV_IN_RESET,
+};
+
+/* NITROX support modes for VF(s) */
+enum vf_mode {
+ __NDEV_MODE_PF,
+ __NDEV_MODE_VF16,
+ __NDEV_MODE_VF32,
+ __NDEV_MODE_VF64,
+ __NDEV_MODE_VF128,
+};
+
+#define __NDEV_SRIOV_BIT 0
+
+/* command queue size */
+#define DEFAULT_CMD_QLEN 2048
+/* command timeout in milliseconds */
+#define CMD_TIMEOUT 2000
+
+#define DEV(ndev) ((struct device *)(&(ndev)->pdev->dev))
+
+#define NITROX_CSR_ADDR(ndev, offset) \
+ ((ndev)->bar_addr + (offset))
+
+/**
+ * struct nitrox_device - NITROX Device Information.
+ * @list: pointer to linked list of devices
+ * @bar_addr: iomap address
+ * @pdev: PCI device information
+ * @state: NITROX device state
+ * @flags: flags to indicate device the features
+ * @timeout: Request timeout in jiffies
+ * @refcnt: Device usage count
+ * @idx: device index (0..N)
+ * @node: NUMA node id attached
+ * @qlen: Command queue length
+ * @nr_queues: Number of command queues
+ * @mode: Device mode PF/VF
+ * @ctx_pool: DMA pool for crypto context
+ * @pkt_inq: Packet input rings
+ * @aqmq: AQM command queues
+ * @qvec: MSI-X queue vectors information
+ * @iov: SR-IOV informatin
+ * @num_vecs: number of MSI-X vectors
+ * @stats: request statistics
+ * @hw: hardware information
+ * @debugfs_dir: debugfs directory
+ */
+struct nitrox_device {
+ struct list_head list;
+
+ u8 __iomem *bar_addr;
+ struct pci_dev *pdev;
+
+ atomic_t state;
+ unsigned long flags;
+ unsigned long timeout;
+ refcount_t refcnt;
+
+ u8 idx;
+ int node;
+ u16 qlen;
+ u16 nr_queues;
+ enum vf_mode mode;
+
+ struct dma_pool *ctx_pool;
+ struct nitrox_cmdq *pkt_inq;
+ struct nitrox_cmdq *aqmq[MAX_DEV_QUEUES] ____cacheline_aligned_in_smp;
+
+ struct nitrox_q_vector *qvec;
+ struct nitrox_iov iov;
+ int num_vecs;
+
+ struct nitrox_stats stats;
+ struct nitrox_hw hw;
+#if IS_ENABLED(CONFIG_DEBUG_FS)
+ struct dentry *debugfs_dir;
+#endif
+};
+
+/**
+ * nitrox_read_csr - Read from device register
+ * @ndev: NITROX device
+ * @offset: offset of the register to read
+ *
+ * Returns: value read
+ */
+static inline u64 nitrox_read_csr(struct nitrox_device *ndev, u64 offset)
+{
+ return readq(ndev->bar_addr + offset);
+}
+
+/**
+ * nitrox_write_csr - Write to device register
+ * @ndev: NITROX device
+ * @offset: offset of the register to write
+ * @value: value to write
+ */
+static inline void nitrox_write_csr(struct nitrox_device *ndev, u64 offset,
+ u64 value)
+{
+ writeq(value, (ndev->bar_addr + offset));
+}
+
+static inline bool nitrox_ready(struct nitrox_device *ndev)
+{
+ return atomic_read(&ndev->state) == __NDEV_READY;
+}
+
+static inline bool nitrox_vfdev_ready(struct nitrox_vfdev *vfdev)
+{
+ return atomic_read(&vfdev->state) == __NDEV_READY;
+}
+
+#endif /* __NITROX_DEV_H */
diff --git a/drivers/crypto/cavium/nitrox/nitrox_hal.c b/drivers/crypto/cavium/nitrox/nitrox_hal.c
new file mode 100644
index 000000000..13b137410
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/nitrox_hal.c
@@ -0,0 +1,679 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/delay.h>
+
+#include "nitrox_dev.h"
+#include "nitrox_csr.h"
+#include "nitrox_hal.h"
+
+#define PLL_REF_CLK 50
+#define MAX_CSR_RETRIES 10
+
+/**
+ * emu_enable_cores - Enable EMU cluster cores.
+ * @ndev: NITROX device
+ */
+static void emu_enable_cores(struct nitrox_device *ndev)
+{
+ union emu_se_enable emu_se;
+ union emu_ae_enable emu_ae;
+ int i;
+
+ /* AE cores 20 per cluster */
+ emu_ae.value = 0;
+ emu_ae.s.enable = 0xfffff;
+
+ /* SE cores 16 per cluster */
+ emu_se.value = 0;
+ emu_se.s.enable = 0xffff;
+
+ /* enable per cluster cores */
+ for (i = 0; i < NR_CLUSTERS; i++) {
+ nitrox_write_csr(ndev, EMU_AE_ENABLEX(i), emu_ae.value);
+ nitrox_write_csr(ndev, EMU_SE_ENABLEX(i), emu_se.value);
+ }
+}
+
+/**
+ * nitrox_config_emu_unit - configure EMU unit.
+ * @ndev: NITROX device
+ */
+void nitrox_config_emu_unit(struct nitrox_device *ndev)
+{
+ union emu_wd_int_ena_w1s emu_wd_int;
+ union emu_ge_int_ena_w1s emu_ge_int;
+ u64 offset;
+ int i;
+
+ /* enable cores */
+ emu_enable_cores(ndev);
+
+ /* enable general error and watch dog interrupts */
+ emu_ge_int.value = 0;
+ emu_ge_int.s.se_ge = 0xffff;
+ emu_ge_int.s.ae_ge = 0xfffff;
+ emu_wd_int.value = 0;
+ emu_wd_int.s.se_wd = 1;
+
+ for (i = 0; i < NR_CLUSTERS; i++) {
+ offset = EMU_WD_INT_ENA_W1SX(i);
+ nitrox_write_csr(ndev, offset, emu_wd_int.value);
+ offset = EMU_GE_INT_ENA_W1SX(i);
+ nitrox_write_csr(ndev, offset, emu_ge_int.value);
+ }
+}
+
+static void reset_pkt_input_ring(struct nitrox_device *ndev, int ring)
+{
+ union nps_pkt_in_instr_ctl pkt_in_ctl;
+ union nps_pkt_in_done_cnts pkt_in_cnts;
+ int max_retries = MAX_CSR_RETRIES;
+ u64 offset;
+
+ /* step 1: disable the ring, clear enable bit */
+ offset = NPS_PKT_IN_INSTR_CTLX(ring);
+ pkt_in_ctl.value = nitrox_read_csr(ndev, offset);
+ pkt_in_ctl.s.enb = 0;
+ nitrox_write_csr(ndev, offset, pkt_in_ctl.value);
+
+ /* step 2: wait to clear [ENB] */
+ usleep_range(100, 150);
+ do {
+ pkt_in_ctl.value = nitrox_read_csr(ndev, offset);
+ if (!pkt_in_ctl.s.enb)
+ break;
+ udelay(50);
+ } while (max_retries--);
+
+ /* step 3: clear done counts */
+ offset = NPS_PKT_IN_DONE_CNTSX(ring);
+ pkt_in_cnts.value = nitrox_read_csr(ndev, offset);
+ nitrox_write_csr(ndev, offset, pkt_in_cnts.value);
+ usleep_range(50, 100);
+}
+
+void enable_pkt_input_ring(struct nitrox_device *ndev, int ring)
+{
+ union nps_pkt_in_instr_ctl pkt_in_ctl;
+ int max_retries = MAX_CSR_RETRIES;
+ u64 offset;
+
+ /* 64-byte instruction size */
+ offset = NPS_PKT_IN_INSTR_CTLX(ring);
+ pkt_in_ctl.value = nitrox_read_csr(ndev, offset);
+ pkt_in_ctl.s.is64b = 1;
+ pkt_in_ctl.s.enb = 1;
+ nitrox_write_csr(ndev, offset, pkt_in_ctl.value);
+
+ /* wait for set [ENB] */
+ do {
+ pkt_in_ctl.value = nitrox_read_csr(ndev, offset);
+ if (pkt_in_ctl.s.enb)
+ break;
+ udelay(50);
+ } while (max_retries--);
+}
+
+/**
+ * nitrox_config_pkt_input_rings - configure Packet Input Rings
+ * @ndev: NITROX device
+ */
+void nitrox_config_pkt_input_rings(struct nitrox_device *ndev)
+{
+ int i;
+
+ for (i = 0; i < ndev->nr_queues; i++) {
+ struct nitrox_cmdq *cmdq = &ndev->pkt_inq[i];
+ union nps_pkt_in_instr_rsize pkt_in_rsize;
+ union nps_pkt_in_instr_baoff_dbell pkt_in_dbell;
+ u64 offset;
+
+ reset_pkt_input_ring(ndev, i);
+
+ /**
+ * step 4:
+ * configure ring base address 16-byte aligned,
+ * size and interrupt threshold.
+ */
+ offset = NPS_PKT_IN_INSTR_BADDRX(i);
+ nitrox_write_csr(ndev, offset, cmdq->dma);
+
+ /* configure ring size */
+ offset = NPS_PKT_IN_INSTR_RSIZEX(i);
+ pkt_in_rsize.value = 0;
+ pkt_in_rsize.s.rsize = ndev->qlen;
+ nitrox_write_csr(ndev, offset, pkt_in_rsize.value);
+
+ /* set high threshold for pkt input ring interrupts */
+ offset = NPS_PKT_IN_INT_LEVELSX(i);
+ nitrox_write_csr(ndev, offset, 0xffffffff);
+
+ /* step 5: clear off door bell counts */
+ offset = NPS_PKT_IN_INSTR_BAOFF_DBELLX(i);
+ pkt_in_dbell.value = 0;
+ pkt_in_dbell.s.dbell = 0xffffffff;
+ nitrox_write_csr(ndev, offset, pkt_in_dbell.value);
+
+ /* enable the ring */
+ enable_pkt_input_ring(ndev, i);
+ }
+}
+
+static void reset_pkt_solicit_port(struct nitrox_device *ndev, int port)
+{
+ union nps_pkt_slc_ctl pkt_slc_ctl;
+ union nps_pkt_slc_cnts pkt_slc_cnts;
+ int max_retries = MAX_CSR_RETRIES;
+ u64 offset;
+
+ /* step 1: disable slc port */
+ offset = NPS_PKT_SLC_CTLX(port);
+ pkt_slc_ctl.value = nitrox_read_csr(ndev, offset);
+ pkt_slc_ctl.s.enb = 0;
+ nitrox_write_csr(ndev, offset, pkt_slc_ctl.value);
+
+ /* step 2 */
+ usleep_range(100, 150);
+ /* wait to clear [ENB] */
+ do {
+ pkt_slc_ctl.value = nitrox_read_csr(ndev, offset);
+ if (!pkt_slc_ctl.s.enb)
+ break;
+ udelay(50);
+ } while (max_retries--);
+
+ /* step 3: clear slc counters */
+ offset = NPS_PKT_SLC_CNTSX(port);
+ pkt_slc_cnts.value = nitrox_read_csr(ndev, offset);
+ nitrox_write_csr(ndev, offset, pkt_slc_cnts.value);
+ usleep_range(50, 100);
+}
+
+void enable_pkt_solicit_port(struct nitrox_device *ndev, int port)
+{
+ union nps_pkt_slc_ctl pkt_slc_ctl;
+ int max_retries = MAX_CSR_RETRIES;
+ u64 offset;
+
+ offset = NPS_PKT_SLC_CTLX(port);
+ pkt_slc_ctl.value = 0;
+ pkt_slc_ctl.s.enb = 1;
+ /*
+ * 8 trailing 0x00 bytes will be added
+ * to the end of the outgoing packet.
+ */
+ pkt_slc_ctl.s.z = 1;
+ /* enable response header */
+ pkt_slc_ctl.s.rh = 1;
+ nitrox_write_csr(ndev, offset, pkt_slc_ctl.value);
+
+ /* wait to set [ENB] */
+ do {
+ pkt_slc_ctl.value = nitrox_read_csr(ndev, offset);
+ if (pkt_slc_ctl.s.enb)
+ break;
+ udelay(50);
+ } while (max_retries--);
+}
+
+static void config_pkt_solicit_port(struct nitrox_device *ndev, int port)
+{
+ union nps_pkt_slc_int_levels pkt_slc_int;
+ u64 offset;
+
+ reset_pkt_solicit_port(ndev, port);
+
+ /* step 4: configure interrupt levels */
+ offset = NPS_PKT_SLC_INT_LEVELSX(port);
+ pkt_slc_int.value = 0;
+ /* time interrupt threshold */
+ pkt_slc_int.s.timet = 0x3fffff;
+ nitrox_write_csr(ndev, offset, pkt_slc_int.value);
+
+ /* enable the solicit port */
+ enable_pkt_solicit_port(ndev, port);
+}
+
+void nitrox_config_pkt_solicit_ports(struct nitrox_device *ndev)
+{
+ int i;
+
+ for (i = 0; i < ndev->nr_queues; i++)
+ config_pkt_solicit_port(ndev, i);
+}
+
+/**
+ * enable_nps_core_interrupts - enable NPS core interrutps
+ * @ndev: NITROX device.
+ *
+ * This includes NPS core interrupts.
+ */
+static void enable_nps_core_interrupts(struct nitrox_device *ndev)
+{
+ union nps_core_int_ena_w1s core_int;
+
+ /* NPS core interrutps */
+ core_int.value = 0;
+ core_int.s.host_wr_err = 1;
+ core_int.s.host_wr_timeout = 1;
+ core_int.s.exec_wr_timeout = 1;
+ core_int.s.npco_dma_malform = 1;
+ core_int.s.host_nps_wr_err = 1;
+ nitrox_write_csr(ndev, NPS_CORE_INT_ENA_W1S, core_int.value);
+}
+
+void nitrox_config_nps_core_unit(struct nitrox_device *ndev)
+{
+ union nps_core_gbl_vfcfg core_gbl_vfcfg;
+
+ /* endian control information */
+ nitrox_write_csr(ndev, NPS_CORE_CONTROL, 1ULL);
+
+ /* disable ILK interface */
+ core_gbl_vfcfg.value = 0;
+ core_gbl_vfcfg.s.ilk_disable = 1;
+ core_gbl_vfcfg.s.cfg = __NDEV_MODE_PF;
+ nitrox_write_csr(ndev, NPS_CORE_GBL_VFCFG, core_gbl_vfcfg.value);
+
+ /* enable nps core interrupts */
+ enable_nps_core_interrupts(ndev);
+}
+
+/**
+ * enable_nps_pkt_interrupts - enable NPS packet interrutps
+ * @ndev: NITROX device.
+ *
+ * This includes NPS packet in and slc interrupts.
+ */
+static void enable_nps_pkt_interrupts(struct nitrox_device *ndev)
+{
+ /* NPS packet in ring interrupts */
+ nitrox_write_csr(ndev, NPS_PKT_IN_RERR_LO_ENA_W1S, (~0ULL));
+ nitrox_write_csr(ndev, NPS_PKT_IN_RERR_HI_ENA_W1S, (~0ULL));
+ nitrox_write_csr(ndev, NPS_PKT_IN_ERR_TYPE_ENA_W1S, (~0ULL));
+ /* NPS packet slc port interrupts */
+ nitrox_write_csr(ndev, NPS_PKT_SLC_RERR_HI_ENA_W1S, (~0ULL));
+ nitrox_write_csr(ndev, NPS_PKT_SLC_RERR_LO_ENA_W1S, (~0ULL));
+ nitrox_write_csr(ndev, NPS_PKT_SLC_ERR_TYPE_ENA_W1S, (~0uLL));
+}
+
+void nitrox_config_nps_pkt_unit(struct nitrox_device *ndev)
+{
+ /* config input and solicit ports */
+ nitrox_config_pkt_input_rings(ndev);
+ nitrox_config_pkt_solicit_ports(ndev);
+
+ /* enable nps packet interrupts */
+ enable_nps_pkt_interrupts(ndev);
+}
+
+static void reset_aqm_ring(struct nitrox_device *ndev, int ring)
+{
+ union aqmq_en aqmq_en_reg;
+ union aqmq_activity_stat activity_stat;
+ union aqmq_cmp_cnt cmp_cnt;
+ int max_retries = MAX_CSR_RETRIES;
+ u64 offset;
+
+ /* step 1: disable the queue */
+ offset = AQMQ_ENX(ring);
+ aqmq_en_reg.value = 0;
+ aqmq_en_reg.queue_enable = 0;
+ nitrox_write_csr(ndev, offset, aqmq_en_reg.value);
+
+ /* step 2: wait for AQMQ_ACTIVITY_STATX[QUEUE_ACTIVE] to clear */
+ usleep_range(100, 150);
+ offset = AQMQ_ACTIVITY_STATX(ring);
+ do {
+ activity_stat.value = nitrox_read_csr(ndev, offset);
+ if (!activity_stat.queue_active)
+ break;
+ udelay(50);
+ } while (max_retries--);
+
+ /* step 3: clear commands completed count */
+ offset = AQMQ_CMP_CNTX(ring);
+ cmp_cnt.value = nitrox_read_csr(ndev, offset);
+ nitrox_write_csr(ndev, offset, cmp_cnt.value);
+ usleep_range(50, 100);
+}
+
+void enable_aqm_ring(struct nitrox_device *ndev, int ring)
+{
+ union aqmq_en aqmq_en_reg;
+ u64 offset;
+
+ offset = AQMQ_ENX(ring);
+ aqmq_en_reg.value = 0;
+ aqmq_en_reg.queue_enable = 1;
+ nitrox_write_csr(ndev, offset, aqmq_en_reg.value);
+ usleep_range(50, 100);
+}
+
+void nitrox_config_aqm_rings(struct nitrox_device *ndev)
+{
+ int ring;
+
+ for (ring = 0; ring < ndev->nr_queues; ring++) {
+ struct nitrox_cmdq *cmdq = ndev->aqmq[ring];
+ union aqmq_drbl drbl;
+ union aqmq_qsz qsize;
+ union aqmq_cmp_thr cmp_thr;
+ u64 offset;
+
+ /* steps 1 - 3 */
+ reset_aqm_ring(ndev, ring);
+
+ /* step 4: clear doorbell count of ring */
+ offset = AQMQ_DRBLX(ring);
+ drbl.value = 0;
+ drbl.dbell_count = 0xFFFFFFFF;
+ nitrox_write_csr(ndev, offset, drbl.value);
+
+ /* step 5: configure host ring details */
+
+ /* set host address for next command of ring */
+ offset = AQMQ_NXT_CMDX(ring);
+ nitrox_write_csr(ndev, offset, 0ULL);
+
+ /* set host address of ring base */
+ offset = AQMQ_BADRX(ring);
+ nitrox_write_csr(ndev, offset, cmdq->dma);
+
+ /* set ring size */
+ offset = AQMQ_QSZX(ring);
+ qsize.value = 0;
+ qsize.host_queue_size = ndev->qlen;
+ nitrox_write_csr(ndev, offset, qsize.value);
+
+ /* set command completion threshold */
+ offset = AQMQ_CMP_THRX(ring);
+ cmp_thr.value = 0;
+ cmp_thr.commands_completed_threshold = 1;
+ nitrox_write_csr(ndev, offset, cmp_thr.value);
+
+ /* step 6: enable the queue */
+ enable_aqm_ring(ndev, ring);
+ }
+}
+
+static void enable_aqm_interrupts(struct nitrox_device *ndev)
+{
+ /* clear interrupt enable bits */
+ nitrox_write_csr(ndev, AQM_DBELL_OVF_LO_ENA_W1S, (~0ULL));
+ nitrox_write_csr(ndev, AQM_DBELL_OVF_HI_ENA_W1S, (~0ULL));
+ nitrox_write_csr(ndev, AQM_DMA_RD_ERR_LO_ENA_W1S, (~0ULL));
+ nitrox_write_csr(ndev, AQM_DMA_RD_ERR_HI_ENA_W1S, (~0ULL));
+ nitrox_write_csr(ndev, AQM_EXEC_NA_LO_ENA_W1S, (~0ULL));
+ nitrox_write_csr(ndev, AQM_EXEC_NA_HI_ENA_W1S, (~0ULL));
+ nitrox_write_csr(ndev, AQM_EXEC_ERR_LO_ENA_W1S, (~0ULL));
+ nitrox_write_csr(ndev, AQM_EXEC_ERR_HI_ENA_W1S, (~0ULL));
+}
+
+void nitrox_config_aqm_unit(struct nitrox_device *ndev)
+{
+ /* config aqm command queues */
+ nitrox_config_aqm_rings(ndev);
+
+ /* enable aqm interrupts */
+ enable_aqm_interrupts(ndev);
+}
+
+void nitrox_config_pom_unit(struct nitrox_device *ndev)
+{
+ union pom_int_ena_w1s pom_int;
+ int i;
+
+ /* enable pom interrupts */
+ pom_int.value = 0;
+ pom_int.s.illegal_dport = 1;
+ nitrox_write_csr(ndev, POM_INT_ENA_W1S, pom_int.value);
+
+ /* enable perf counters */
+ for (i = 0; i < ndev->hw.se_cores; i++)
+ nitrox_write_csr(ndev, POM_PERF_CTL, BIT_ULL(i));
+}
+
+/**
+ * nitrox_config_rand_unit - enable NITROX random number unit
+ * @ndev: NITROX device
+ */
+void nitrox_config_rand_unit(struct nitrox_device *ndev)
+{
+ union efl_rnm_ctl_status efl_rnm_ctl;
+ u64 offset;
+
+ offset = EFL_RNM_CTL_STATUS;
+ efl_rnm_ctl.value = nitrox_read_csr(ndev, offset);
+ efl_rnm_ctl.s.ent_en = 1;
+ efl_rnm_ctl.s.rng_en = 1;
+ nitrox_write_csr(ndev, offset, efl_rnm_ctl.value);
+}
+
+void nitrox_config_efl_unit(struct nitrox_device *ndev)
+{
+ int i;
+
+ for (i = 0; i < NR_CLUSTERS; i++) {
+ union efl_core_int_ena_w1s efl_core_int;
+ u64 offset;
+
+ /* EFL core interrupts */
+ offset = EFL_CORE_INT_ENA_W1SX(i);
+ efl_core_int.value = 0;
+ efl_core_int.s.len_ovr = 1;
+ efl_core_int.s.d_left = 1;
+ efl_core_int.s.epci_decode_err = 1;
+ nitrox_write_csr(ndev, offset, efl_core_int.value);
+
+ offset = EFL_CORE_VF_ERR_INT0_ENA_W1SX(i);
+ nitrox_write_csr(ndev, offset, (~0ULL));
+ offset = EFL_CORE_VF_ERR_INT1_ENA_W1SX(i);
+ nitrox_write_csr(ndev, offset, (~0ULL));
+ }
+}
+
+void nitrox_config_bmi_unit(struct nitrox_device *ndev)
+{
+ union bmi_ctl bmi_ctl;
+ union bmi_int_ena_w1s bmi_int_ena;
+ u64 offset;
+
+ /* no threshold limits for PCIe */
+ offset = BMI_CTL;
+ bmi_ctl.value = nitrox_read_csr(ndev, offset);
+ bmi_ctl.s.max_pkt_len = 0xff;
+ bmi_ctl.s.nps_free_thrsh = 0xff;
+ bmi_ctl.s.nps_hdrq_thrsh = 0x7a;
+ nitrox_write_csr(ndev, offset, bmi_ctl.value);
+
+ /* enable interrupts */
+ offset = BMI_INT_ENA_W1S;
+ bmi_int_ena.value = 0;
+ bmi_int_ena.s.max_len_err_nps = 1;
+ bmi_int_ena.s.pkt_rcv_err_nps = 1;
+ bmi_int_ena.s.fpf_undrrn = 1;
+ nitrox_write_csr(ndev, offset, bmi_int_ena.value);
+}
+
+void nitrox_config_bmo_unit(struct nitrox_device *ndev)
+{
+ union bmo_ctl2 bmo_ctl2;
+ u64 offset;
+
+ /* no threshold limits for PCIe */
+ offset = BMO_CTL2;
+ bmo_ctl2.value = nitrox_read_csr(ndev, offset);
+ bmo_ctl2.s.nps_slc_buf_thrsh = 0xff;
+ nitrox_write_csr(ndev, offset, bmo_ctl2.value);
+}
+
+void invalidate_lbc(struct nitrox_device *ndev)
+{
+ union lbc_inval_ctl lbc_ctl;
+ union lbc_inval_status lbc_stat;
+ int max_retries = MAX_CSR_RETRIES;
+ u64 offset;
+
+ /* invalidate LBC */
+ offset = LBC_INVAL_CTL;
+ lbc_ctl.value = nitrox_read_csr(ndev, offset);
+ lbc_ctl.s.cam_inval_start = 1;
+ nitrox_write_csr(ndev, offset, lbc_ctl.value);
+
+ offset = LBC_INVAL_STATUS;
+ do {
+ lbc_stat.value = nitrox_read_csr(ndev, offset);
+ if (lbc_stat.s.done)
+ break;
+ udelay(50);
+ } while (max_retries--);
+}
+
+void nitrox_config_lbc_unit(struct nitrox_device *ndev)
+{
+ union lbc_int_ena_w1s lbc_int_ena;
+ u64 offset;
+
+ invalidate_lbc(ndev);
+
+ /* enable interrupts */
+ offset = LBC_INT_ENA_W1S;
+ lbc_int_ena.value = 0;
+ lbc_int_ena.s.dma_rd_err = 1;
+ lbc_int_ena.s.over_fetch_err = 1;
+ lbc_int_ena.s.cam_inval_abort = 1;
+ lbc_int_ena.s.cam_hard_err = 1;
+ nitrox_write_csr(ndev, offset, lbc_int_ena.value);
+
+ offset = LBC_PLM_VF1_64_INT_ENA_W1S;
+ nitrox_write_csr(ndev, offset, (~0ULL));
+ offset = LBC_PLM_VF65_128_INT_ENA_W1S;
+ nitrox_write_csr(ndev, offset, (~0ULL));
+
+ offset = LBC_ELM_VF1_64_INT_ENA_W1S;
+ nitrox_write_csr(ndev, offset, (~0ULL));
+ offset = LBC_ELM_VF65_128_INT_ENA_W1S;
+ nitrox_write_csr(ndev, offset, (~0ULL));
+}
+
+void config_nps_core_vfcfg_mode(struct nitrox_device *ndev, enum vf_mode mode)
+{
+ union nps_core_gbl_vfcfg vfcfg;
+
+ vfcfg.value = nitrox_read_csr(ndev, NPS_CORE_GBL_VFCFG);
+ vfcfg.s.cfg = mode & 0x7;
+
+ nitrox_write_csr(ndev, NPS_CORE_GBL_VFCFG, vfcfg.value);
+}
+
+static const char *get_core_option(u8 se_cores, u8 ae_cores)
+{
+ const char *option = "";
+
+ if (ae_cores == AE_MAX_CORES) {
+ switch (se_cores) {
+ case SE_MAX_CORES:
+ option = "60";
+ break;
+ case 40:
+ option = "60s";
+ break;
+ }
+ } else if (ae_cores == (AE_MAX_CORES / 2)) {
+ option = "30";
+ } else {
+ option = "60i";
+ }
+
+ return option;
+}
+
+static const char *get_feature_option(u8 zip_cores, int core_freq)
+{
+ if (zip_cores == 0)
+ return "";
+ else if (zip_cores < ZIP_MAX_CORES)
+ return "-C15";
+
+ if (core_freq >= 850)
+ return "-C45";
+ else if (core_freq >= 750)
+ return "-C35";
+ else if (core_freq >= 550)
+ return "-C25";
+
+ return "";
+}
+
+void nitrox_get_hwinfo(struct nitrox_device *ndev)
+{
+ union emu_fuse_map emu_fuse;
+ union rst_boot rst_boot;
+ union fus_dat1 fus_dat1;
+ unsigned char name[IFNAMSIZ * 2] = {};
+ int i, dead_cores;
+ u64 offset;
+
+ /* get core frequency */
+ offset = RST_BOOT;
+ rst_boot.value = nitrox_read_csr(ndev, offset);
+ ndev->hw.freq = (rst_boot.pnr_mul + 3) * PLL_REF_CLK;
+
+ for (i = 0; i < NR_CLUSTERS; i++) {
+ offset = EMU_FUSE_MAPX(i);
+ emu_fuse.value = nitrox_read_csr(ndev, offset);
+ if (emu_fuse.s.valid) {
+ dead_cores = hweight32(emu_fuse.s.ae_fuse);
+ ndev->hw.ae_cores += AE_CORES_PER_CLUSTER - dead_cores;
+ dead_cores = hweight16(emu_fuse.s.se_fuse);
+ ndev->hw.se_cores += SE_CORES_PER_CLUSTER - dead_cores;
+ }
+ }
+ /* find zip hardware availability */
+ offset = FUS_DAT1;
+ fus_dat1.value = nitrox_read_csr(ndev, offset);
+ if (!fus_dat1.nozip) {
+ dead_cores = hweight8(fus_dat1.zip_info);
+ ndev->hw.zip_cores = ZIP_MAX_CORES - dead_cores;
+ }
+
+ /* determine the partname
+ * CNN55<core option>-<freq><pincount>-<feature option>-<rev>
+ */
+ snprintf(name, sizeof(name), "CNN55%s-%3dBG676%s-1.%u",
+ get_core_option(ndev->hw.se_cores, ndev->hw.ae_cores),
+ ndev->hw.freq,
+ get_feature_option(ndev->hw.zip_cores, ndev->hw.freq),
+ ndev->hw.revision_id);
+
+ /* copy partname */
+ strncpy(ndev->hw.partname, name, sizeof(ndev->hw.partname));
+}
+
+void enable_pf2vf_mbox_interrupts(struct nitrox_device *ndev)
+{
+ u64 value = ~0ULL;
+ u64 reg_addr;
+
+ /* Mailbox interrupt low enable set register */
+ reg_addr = NPS_PKT_MBOX_INT_LO_ENA_W1S;
+ nitrox_write_csr(ndev, reg_addr, value);
+
+ /* Mailbox interrupt high enable set register */
+ reg_addr = NPS_PKT_MBOX_INT_HI_ENA_W1S;
+ nitrox_write_csr(ndev, reg_addr, value);
+}
+
+void disable_pf2vf_mbox_interrupts(struct nitrox_device *ndev)
+{
+ u64 value = ~0ULL;
+ u64 reg_addr;
+
+ /* Mailbox interrupt low enable clear register */
+ reg_addr = NPS_PKT_MBOX_INT_LO_ENA_W1C;
+ nitrox_write_csr(ndev, reg_addr, value);
+
+ /* Mailbox interrupt high enable clear register */
+ reg_addr = NPS_PKT_MBOX_INT_HI_ENA_W1C;
+ nitrox_write_csr(ndev, reg_addr, value);
+}
diff --git a/drivers/crypto/cavium/nitrox/nitrox_hal.h b/drivers/crypto/cavium/nitrox/nitrox_hal.h
new file mode 100644
index 000000000..48b0af039
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/nitrox_hal.h
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __NITROX_HAL_H
+#define __NITROX_HAL_H
+
+#include "nitrox_dev.h"
+
+void nitrox_config_aqm_rings(struct nitrox_device *ndev);
+void nitrox_config_aqm_unit(struct nitrox_device *ndev);
+void nitrox_config_emu_unit(struct nitrox_device *ndev);
+void nitrox_config_pkt_input_rings(struct nitrox_device *ndev);
+void nitrox_config_pkt_solicit_ports(struct nitrox_device *ndev);
+void nitrox_config_nps_core_unit(struct nitrox_device *ndev);
+void nitrox_config_nps_pkt_unit(struct nitrox_device *ndev);
+void nitrox_config_pom_unit(struct nitrox_device *ndev);
+void nitrox_config_rand_unit(struct nitrox_device *ndev);
+void nitrox_config_efl_unit(struct nitrox_device *ndev);
+void nitrox_config_bmi_unit(struct nitrox_device *ndev);
+void nitrox_config_bmo_unit(struct nitrox_device *ndev);
+void nitrox_config_lbc_unit(struct nitrox_device *ndev);
+void invalidate_lbc(struct nitrox_device *ndev);
+void enable_aqm_ring(struct nitrox_device *ndev, int qno);
+void enable_pkt_input_ring(struct nitrox_device *ndev, int ring);
+void enable_pkt_solicit_port(struct nitrox_device *ndev, int port);
+void config_nps_core_vfcfg_mode(struct nitrox_device *ndev, enum vf_mode mode);
+void nitrox_get_hwinfo(struct nitrox_device *ndev);
+void enable_pf2vf_mbox_interrupts(struct nitrox_device *ndev);
+void disable_pf2vf_mbox_interrupts(struct nitrox_device *ndev);
+
+#endif /* __NITROX_HAL_H */
diff --git a/drivers/crypto/cavium/nitrox/nitrox_isr.c b/drivers/crypto/cavium/nitrox/nitrox_isr.c
new file mode 100644
index 000000000..f19e520da
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/nitrox_isr.c
@@ -0,0 +1,458 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/pci.h>
+#include <linux/printk.h>
+#include <linux/slab.h>
+
+#include "nitrox_dev.h"
+#include "nitrox_csr.h"
+#include "nitrox_common.h"
+#include "nitrox_hal.h"
+#include "nitrox_isr.h"
+#include "nitrox_mbx.h"
+
+/*
+ * One vector for each type of ring
+ * - NPS packet ring, AQMQ ring and ZQMQ ring
+ */
+#define NR_RING_VECTORS 3
+#define NR_NON_RING_VECTORS 1
+/* base entry for packet ring/port */
+#define PKT_RING_MSIX_BASE 0
+#define NON_RING_MSIX_BASE 192
+
+/**
+ * nps_pkt_slc_isr - IRQ handler for NPS solicit port
+ * @irq: irq number
+ * @data: argument
+ */
+static irqreturn_t nps_pkt_slc_isr(int irq, void *data)
+{
+ struct nitrox_q_vector *qvec = data;
+ union nps_pkt_slc_cnts slc_cnts;
+ struct nitrox_cmdq *cmdq = qvec->cmdq;
+
+ slc_cnts.value = readq(cmdq->compl_cnt_csr_addr);
+ /* New packet on SLC output port */
+ if (slc_cnts.s.slc_int)
+ tasklet_hi_schedule(&qvec->resp_tasklet);
+
+ return IRQ_HANDLED;
+}
+
+static void clear_nps_core_err_intr(struct nitrox_device *ndev)
+{
+ u64 value;
+
+ /* Write 1 to clear */
+ value = nitrox_read_csr(ndev, NPS_CORE_INT);
+ nitrox_write_csr(ndev, NPS_CORE_INT, value);
+
+ dev_err_ratelimited(DEV(ndev), "NSP_CORE_INT 0x%016llx\n", value);
+}
+
+static void clear_nps_pkt_err_intr(struct nitrox_device *ndev)
+{
+ union nps_pkt_int pkt_int;
+ unsigned long value, offset;
+ int i;
+
+ pkt_int.value = nitrox_read_csr(ndev, NPS_PKT_INT);
+ dev_err_ratelimited(DEV(ndev), "NPS_PKT_INT 0x%016llx\n",
+ pkt_int.value);
+
+ if (pkt_int.s.slc_err) {
+ offset = NPS_PKT_SLC_ERR_TYPE;
+ value = nitrox_read_csr(ndev, offset);
+ nitrox_write_csr(ndev, offset, value);
+ dev_err_ratelimited(DEV(ndev),
+ "NPS_PKT_SLC_ERR_TYPE 0x%016lx\n", value);
+
+ offset = NPS_PKT_SLC_RERR_LO;
+ value = nitrox_read_csr(ndev, offset);
+ nitrox_write_csr(ndev, offset, value);
+ /* enable the solicit ports */
+ for_each_set_bit(i, &value, BITS_PER_LONG)
+ enable_pkt_solicit_port(ndev, i);
+
+ dev_err_ratelimited(DEV(ndev),
+ "NPS_PKT_SLC_RERR_LO 0x%016lx\n", value);
+
+ offset = NPS_PKT_SLC_RERR_HI;
+ value = nitrox_read_csr(ndev, offset);
+ nitrox_write_csr(ndev, offset, value);
+ dev_err_ratelimited(DEV(ndev),
+ "NPS_PKT_SLC_RERR_HI 0x%016lx\n", value);
+ }
+
+ if (pkt_int.s.in_err) {
+ offset = NPS_PKT_IN_ERR_TYPE;
+ value = nitrox_read_csr(ndev, offset);
+ nitrox_write_csr(ndev, offset, value);
+ dev_err_ratelimited(DEV(ndev),
+ "NPS_PKT_IN_ERR_TYPE 0x%016lx\n", value);
+ offset = NPS_PKT_IN_RERR_LO;
+ value = nitrox_read_csr(ndev, offset);
+ nitrox_write_csr(ndev, offset, value);
+ /* enable the input ring */
+ for_each_set_bit(i, &value, BITS_PER_LONG)
+ enable_pkt_input_ring(ndev, i);
+
+ dev_err_ratelimited(DEV(ndev),
+ "NPS_PKT_IN_RERR_LO 0x%016lx\n", value);
+
+ offset = NPS_PKT_IN_RERR_HI;
+ value = nitrox_read_csr(ndev, offset);
+ nitrox_write_csr(ndev, offset, value);
+ dev_err_ratelimited(DEV(ndev),
+ "NPS_PKT_IN_RERR_HI 0x%016lx\n", value);
+ }
+}
+
+static void clear_pom_err_intr(struct nitrox_device *ndev)
+{
+ u64 value;
+
+ value = nitrox_read_csr(ndev, POM_INT);
+ nitrox_write_csr(ndev, POM_INT, value);
+ dev_err_ratelimited(DEV(ndev), "POM_INT 0x%016llx\n", value);
+}
+
+static void clear_pem_err_intr(struct nitrox_device *ndev)
+{
+ u64 value;
+
+ value = nitrox_read_csr(ndev, PEM0_INT);
+ nitrox_write_csr(ndev, PEM0_INT, value);
+ dev_err_ratelimited(DEV(ndev), "PEM(0)_INT 0x%016llx\n", value);
+}
+
+static void clear_lbc_err_intr(struct nitrox_device *ndev)
+{
+ union lbc_int lbc_int;
+ u64 value, offset;
+ int i;
+
+ lbc_int.value = nitrox_read_csr(ndev, LBC_INT);
+ dev_err_ratelimited(DEV(ndev), "LBC_INT 0x%016llx\n", lbc_int.value);
+
+ if (lbc_int.s.dma_rd_err) {
+ for (i = 0; i < NR_CLUSTERS; i++) {
+ offset = EFL_CORE_VF_ERR_INT0X(i);
+ value = nitrox_read_csr(ndev, offset);
+ nitrox_write_csr(ndev, offset, value);
+ offset = EFL_CORE_VF_ERR_INT1X(i);
+ value = nitrox_read_csr(ndev, offset);
+ nitrox_write_csr(ndev, offset, value);
+ }
+ }
+
+ if (lbc_int.s.cam_soft_err) {
+ dev_err_ratelimited(DEV(ndev), "CAM_SOFT_ERR, invalidating LBC\n");
+ invalidate_lbc(ndev);
+ }
+
+ if (lbc_int.s.pref_dat_len_mismatch_err) {
+ offset = LBC_PLM_VF1_64_INT;
+ value = nitrox_read_csr(ndev, offset);
+ nitrox_write_csr(ndev, offset, value);
+ offset = LBC_PLM_VF65_128_INT;
+ value = nitrox_read_csr(ndev, offset);
+ nitrox_write_csr(ndev, offset, value);
+ }
+
+ if (lbc_int.s.rd_dat_len_mismatch_err) {
+ offset = LBC_ELM_VF1_64_INT;
+ value = nitrox_read_csr(ndev, offset);
+ nitrox_write_csr(ndev, offset, value);
+ offset = LBC_ELM_VF65_128_INT;
+ value = nitrox_read_csr(ndev, offset);
+ nitrox_write_csr(ndev, offset, value);
+ }
+ nitrox_write_csr(ndev, LBC_INT, lbc_int.value);
+}
+
+static void clear_efl_err_intr(struct nitrox_device *ndev)
+{
+ int i;
+
+ for (i = 0; i < NR_CLUSTERS; i++) {
+ union efl_core_int core_int;
+ u64 value, offset;
+
+ offset = EFL_CORE_INTX(i);
+ core_int.value = nitrox_read_csr(ndev, offset);
+ nitrox_write_csr(ndev, offset, core_int.value);
+ dev_err_ratelimited(DEV(ndev), "ELF_CORE(%d)_INT 0x%016llx\n",
+ i, core_int.value);
+ if (core_int.s.se_err) {
+ offset = EFL_CORE_SE_ERR_INTX(i);
+ value = nitrox_read_csr(ndev, offset);
+ nitrox_write_csr(ndev, offset, value);
+ }
+ }
+}
+
+static void clear_bmi_err_intr(struct nitrox_device *ndev)
+{
+ u64 value;
+
+ value = nitrox_read_csr(ndev, BMI_INT);
+ nitrox_write_csr(ndev, BMI_INT, value);
+ dev_err_ratelimited(DEV(ndev), "BMI_INT 0x%016llx\n", value);
+}
+
+static void nps_core_int_tasklet(unsigned long data)
+{
+ struct nitrox_q_vector *qvec = (void *)(uintptr_t)(data);
+ struct nitrox_device *ndev = qvec->ndev;
+
+ /* if pf mode do queue recovery */
+ if (ndev->mode == __NDEV_MODE_PF) {
+ } else {
+ /**
+ * if VF(s) enabled communicate the error information
+ * to VF(s)
+ */
+ }
+}
+
+/*
+ * nps_core_int_isr - interrupt handler for NITROX errors and
+ * mailbox communication
+ */
+static irqreturn_t nps_core_int_isr(int irq, void *data)
+{
+ struct nitrox_q_vector *qvec = data;
+ struct nitrox_device *ndev = qvec->ndev;
+ union nps_core_int_active core_int;
+
+ core_int.value = nitrox_read_csr(ndev, NPS_CORE_INT_ACTIVE);
+
+ if (core_int.s.nps_core)
+ clear_nps_core_err_intr(ndev);
+
+ if (core_int.s.nps_pkt)
+ clear_nps_pkt_err_intr(ndev);
+
+ if (core_int.s.pom)
+ clear_pom_err_intr(ndev);
+
+ if (core_int.s.pem)
+ clear_pem_err_intr(ndev);
+
+ if (core_int.s.lbc)
+ clear_lbc_err_intr(ndev);
+
+ if (core_int.s.efl)
+ clear_efl_err_intr(ndev);
+
+ if (core_int.s.bmi)
+ clear_bmi_err_intr(ndev);
+
+ /* Mailbox interrupt */
+ if (core_int.s.mbox)
+ nitrox_pf2vf_mbox_handler(ndev);
+
+ /* If more work callback the ISR, set resend */
+ core_int.s.resend = 1;
+ nitrox_write_csr(ndev, NPS_CORE_INT_ACTIVE, core_int.value);
+
+ return IRQ_HANDLED;
+}
+
+void nitrox_unregister_interrupts(struct nitrox_device *ndev)
+{
+ struct pci_dev *pdev = ndev->pdev;
+ int i;
+
+ for (i = 0; i < ndev->num_vecs; i++) {
+ struct nitrox_q_vector *qvec;
+ int vec;
+
+ qvec = ndev->qvec + i;
+ if (!qvec->valid)
+ continue;
+
+ /* get the vector number */
+ vec = pci_irq_vector(pdev, i);
+ irq_set_affinity_hint(vec, NULL);
+ free_irq(vec, qvec);
+
+ tasklet_disable(&qvec->resp_tasklet);
+ tasklet_kill(&qvec->resp_tasklet);
+ qvec->valid = false;
+ }
+ kfree(ndev->qvec);
+ ndev->qvec = NULL;
+ pci_free_irq_vectors(pdev);
+}
+
+int nitrox_register_interrupts(struct nitrox_device *ndev)
+{
+ struct pci_dev *pdev = ndev->pdev;
+ struct nitrox_q_vector *qvec;
+ int nr_vecs, vec, cpu;
+ int ret, i;
+
+ /*
+ * PF MSI-X vectors
+ *
+ * Entry 0: NPS PKT ring 0
+ * Entry 1: AQMQ ring 0
+ * Entry 2: ZQM ring 0
+ * Entry 3: NPS PKT ring 1
+ * Entry 4: AQMQ ring 1
+ * Entry 5: ZQM ring 1
+ * ....
+ * Entry 192: NPS_CORE_INT_ACTIVE
+ */
+ nr_vecs = pci_msix_vec_count(pdev);
+ if (nr_vecs < 0) {
+ dev_err(DEV(ndev), "Error in getting vec count %d\n", nr_vecs);
+ return nr_vecs;
+ }
+
+ /* Enable MSI-X */
+ ret = pci_alloc_irq_vectors(pdev, nr_vecs, nr_vecs, PCI_IRQ_MSIX);
+ if (ret < 0) {
+ dev_err(DEV(ndev), "msix vectors %d alloc failed\n", nr_vecs);
+ return ret;
+ }
+ ndev->num_vecs = nr_vecs;
+
+ ndev->qvec = kcalloc(nr_vecs, sizeof(*qvec), GFP_KERNEL);
+ if (!ndev->qvec) {
+ pci_free_irq_vectors(pdev);
+ return -ENOMEM;
+ }
+
+ /* request irqs for packet rings/ports */
+ for (i = PKT_RING_MSIX_BASE; i < (nr_vecs - 1); i += NR_RING_VECTORS) {
+ qvec = &ndev->qvec[i];
+
+ qvec->ring = i / NR_RING_VECTORS;
+ if (qvec->ring >= ndev->nr_queues)
+ break;
+
+ qvec->cmdq = &ndev->pkt_inq[qvec->ring];
+ snprintf(qvec->name, IRQ_NAMESZ, "nitrox-pkt%d", qvec->ring);
+ /* get the vector number */
+ vec = pci_irq_vector(pdev, i);
+ ret = request_irq(vec, nps_pkt_slc_isr, 0, qvec->name, qvec);
+ if (ret) {
+ dev_err(DEV(ndev), "irq failed for pkt ring/port%d\n",
+ qvec->ring);
+ goto irq_fail;
+ }
+ cpu = qvec->ring % num_online_cpus();
+ irq_set_affinity_hint(vec, get_cpu_mask(cpu));
+
+ tasklet_init(&qvec->resp_tasklet, pkt_slc_resp_tasklet,
+ (unsigned long)qvec);
+ qvec->valid = true;
+ }
+
+ /* request irqs for non ring vectors */
+ i = NON_RING_MSIX_BASE;
+ qvec = &ndev->qvec[i];
+ qvec->ndev = ndev;
+
+ snprintf(qvec->name, IRQ_NAMESZ, "nitrox-core-int%d", i);
+ /* get the vector number */
+ vec = pci_irq_vector(pdev, i);
+ ret = request_irq(vec, nps_core_int_isr, 0, qvec->name, qvec);
+ if (ret) {
+ dev_err(DEV(ndev), "irq failed for nitrox-core-int%d\n", i);
+ goto irq_fail;
+ }
+ cpu = num_online_cpus();
+ irq_set_affinity_hint(vec, get_cpu_mask(cpu));
+
+ tasklet_init(&qvec->resp_tasklet, nps_core_int_tasklet,
+ (unsigned long)qvec);
+ qvec->valid = true;
+
+ return 0;
+
+irq_fail:
+ nitrox_unregister_interrupts(ndev);
+ return ret;
+}
+
+void nitrox_sriov_unregister_interrupts(struct nitrox_device *ndev)
+{
+ struct pci_dev *pdev = ndev->pdev;
+ int i;
+
+ for (i = 0; i < ndev->num_vecs; i++) {
+ struct nitrox_q_vector *qvec;
+ int vec;
+
+ qvec = ndev->qvec + i;
+ if (!qvec->valid)
+ continue;
+
+ vec = ndev->iov.msix.vector;
+ irq_set_affinity_hint(vec, NULL);
+ free_irq(vec, qvec);
+
+ tasklet_disable(&qvec->resp_tasklet);
+ tasklet_kill(&qvec->resp_tasklet);
+ qvec->valid = false;
+ }
+ kfree(ndev->qvec);
+ ndev->qvec = NULL;
+ pci_disable_msix(pdev);
+}
+
+int nitrox_sriov_register_interupts(struct nitrox_device *ndev)
+{
+ struct pci_dev *pdev = ndev->pdev;
+ struct nitrox_q_vector *qvec;
+ int vec, cpu;
+ int ret;
+
+ /**
+ * only non ring vectors i.e Entry 192 is available
+ * for PF in SR-IOV mode.
+ */
+ ndev->iov.msix.entry = NON_RING_MSIX_BASE;
+ ret = pci_enable_msix_exact(pdev, &ndev->iov.msix, NR_NON_RING_VECTORS);
+ if (ret) {
+ dev_err(DEV(ndev), "failed to allocate nps-core-int%d\n",
+ NON_RING_MSIX_BASE);
+ return ret;
+ }
+
+ qvec = kcalloc(NR_NON_RING_VECTORS, sizeof(*qvec), GFP_KERNEL);
+ if (!qvec) {
+ pci_disable_msix(pdev);
+ return -ENOMEM;
+ }
+ qvec->ndev = ndev;
+
+ ndev->qvec = qvec;
+ ndev->num_vecs = NR_NON_RING_VECTORS;
+ snprintf(qvec->name, IRQ_NAMESZ, "nitrox-core-int%d",
+ NON_RING_MSIX_BASE);
+
+ vec = ndev->iov.msix.vector;
+ ret = request_irq(vec, nps_core_int_isr, 0, qvec->name, qvec);
+ if (ret) {
+ dev_err(DEV(ndev), "irq failed for nitrox-core-int%d\n",
+ NON_RING_MSIX_BASE);
+ goto iov_irq_fail;
+ }
+ cpu = num_online_cpus();
+ irq_set_affinity_hint(vec, get_cpu_mask(cpu));
+
+ tasklet_init(&qvec->resp_tasklet, nps_core_int_tasklet,
+ (unsigned long)qvec);
+ qvec->valid = true;
+
+ return 0;
+
+iov_irq_fail:
+ nitrox_sriov_unregister_interrupts(ndev);
+ return ret;
+}
diff --git a/drivers/crypto/cavium/nitrox/nitrox_isr.h b/drivers/crypto/cavium/nitrox/nitrox_isr.h
new file mode 100644
index 000000000..2bb123cd2
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/nitrox_isr.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __NITROX_ISR_H
+#define __NITROX_ISR_H
+
+#include "nitrox_dev.h"
+
+int nitrox_register_interrupts(struct nitrox_device *ndev);
+void nitrox_unregister_interrupts(struct nitrox_device *ndev);
+int nitrox_sriov_register_interupts(struct nitrox_device *ndev);
+void nitrox_sriov_unregister_interrupts(struct nitrox_device *ndev);
+
+#ifdef CONFIG_PCI_IOV
+int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs);
+#else
+static inline int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs)
+{
+ return 0;
+}
+#endif
+
+#endif /* __NITROX_ISR_H */
diff --git a/drivers/crypto/cavium/nitrox/nitrox_lib.c b/drivers/crypto/cavium/nitrox/nitrox_lib.c
new file mode 100644
index 000000000..a5cdc2b48
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/nitrox_lib.c
@@ -0,0 +1,300 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/cpumask.h>
+#include <linux/dma-mapping.h>
+#include <linux/dmapool.h>
+#include <linux/delay.h>
+#include <linux/gfp.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/pci_regs.h>
+#include <linux/vmalloc.h>
+#include <linux/pci.h>
+
+#include "nitrox_dev.h"
+#include "nitrox_common.h"
+#include "nitrox_req.h"
+#include "nitrox_csr.h"
+
+#define CRYPTO_CTX_SIZE 256
+
+/* packet inuput ring alignments */
+#define PKTIN_Q_ALIGN_BYTES 16
+/* AQM Queue input alignments */
+#define AQM_Q_ALIGN_BYTES 32
+
+static int nitrox_cmdq_init(struct nitrox_cmdq *cmdq, int align_bytes)
+{
+ struct nitrox_device *ndev = cmdq->ndev;
+
+ cmdq->qsize = (ndev->qlen * cmdq->instr_size) + align_bytes;
+ cmdq->unalign_base = dma_alloc_coherent(DEV(ndev), cmdq->qsize,
+ &cmdq->unalign_dma,
+ GFP_KERNEL);
+ if (!cmdq->unalign_base)
+ return -ENOMEM;
+
+ cmdq->dma = PTR_ALIGN(cmdq->unalign_dma, align_bytes);
+ cmdq->base = cmdq->unalign_base + (cmdq->dma - cmdq->unalign_dma);
+ cmdq->write_idx = 0;
+
+ spin_lock_init(&cmdq->cmd_qlock);
+ spin_lock_init(&cmdq->resp_qlock);
+ spin_lock_init(&cmdq->backlog_qlock);
+
+ INIT_LIST_HEAD(&cmdq->response_head);
+ INIT_LIST_HEAD(&cmdq->backlog_head);
+ INIT_WORK(&cmdq->backlog_qflush, backlog_qflush_work);
+
+ atomic_set(&cmdq->pending_count, 0);
+ atomic_set(&cmdq->backlog_count, 0);
+ return 0;
+}
+
+static void nitrox_cmdq_reset(struct nitrox_cmdq *cmdq)
+{
+ cmdq->write_idx = 0;
+ atomic_set(&cmdq->pending_count, 0);
+ atomic_set(&cmdq->backlog_count, 0);
+}
+
+static void nitrox_cmdq_cleanup(struct nitrox_cmdq *cmdq)
+{
+ struct nitrox_device *ndev;
+
+ if (!cmdq)
+ return;
+
+ if (!cmdq->unalign_base)
+ return;
+
+ ndev = cmdq->ndev;
+ cancel_work_sync(&cmdq->backlog_qflush);
+
+ dma_free_coherent(DEV(ndev), cmdq->qsize,
+ cmdq->unalign_base, cmdq->unalign_dma);
+ nitrox_cmdq_reset(cmdq);
+
+ cmdq->dbell_csr_addr = NULL;
+ cmdq->compl_cnt_csr_addr = NULL;
+ cmdq->unalign_base = NULL;
+ cmdq->base = NULL;
+ cmdq->unalign_dma = 0;
+ cmdq->dma = 0;
+ cmdq->qsize = 0;
+ cmdq->instr_size = 0;
+}
+
+static void nitrox_free_aqm_queues(struct nitrox_device *ndev)
+{
+ int i;
+
+ for (i = 0; i < ndev->nr_queues; i++) {
+ nitrox_cmdq_cleanup(ndev->aqmq[i]);
+ kfree_sensitive(ndev->aqmq[i]);
+ ndev->aqmq[i] = NULL;
+ }
+}
+
+static int nitrox_alloc_aqm_queues(struct nitrox_device *ndev)
+{
+ int i, err;
+
+ for (i = 0; i < ndev->nr_queues; i++) {
+ struct nitrox_cmdq *cmdq;
+ u64 offset;
+
+ cmdq = kzalloc_node(sizeof(*cmdq), GFP_KERNEL, ndev->node);
+ if (!cmdq) {
+ err = -ENOMEM;
+ goto aqmq_fail;
+ }
+
+ cmdq->ndev = ndev;
+ cmdq->qno = i;
+ cmdq->instr_size = sizeof(struct aqmq_command_s);
+
+ /* AQM Queue Doorbell Counter Register Address */
+ offset = AQMQ_DRBLX(i);
+ cmdq->dbell_csr_addr = NITROX_CSR_ADDR(ndev, offset);
+ /* AQM Queue Commands Completed Count Register Address */
+ offset = AQMQ_CMD_CNTX(i);
+ cmdq->compl_cnt_csr_addr = NITROX_CSR_ADDR(ndev, offset);
+
+ err = nitrox_cmdq_init(cmdq, AQM_Q_ALIGN_BYTES);
+ if (err) {
+ kfree_sensitive(cmdq);
+ goto aqmq_fail;
+ }
+ ndev->aqmq[i] = cmdq;
+ }
+
+ return 0;
+
+aqmq_fail:
+ nitrox_free_aqm_queues(ndev);
+ return err;
+}
+
+static void nitrox_free_pktin_queues(struct nitrox_device *ndev)
+{
+ int i;
+
+ for (i = 0; i < ndev->nr_queues; i++) {
+ struct nitrox_cmdq *cmdq = &ndev->pkt_inq[i];
+
+ nitrox_cmdq_cleanup(cmdq);
+ }
+ kfree(ndev->pkt_inq);
+ ndev->pkt_inq = NULL;
+}
+
+static int nitrox_alloc_pktin_queues(struct nitrox_device *ndev)
+{
+ int i, err;
+
+ ndev->pkt_inq = kcalloc_node(ndev->nr_queues,
+ sizeof(struct nitrox_cmdq),
+ GFP_KERNEL, ndev->node);
+ if (!ndev->pkt_inq)
+ return -ENOMEM;
+
+ for (i = 0; i < ndev->nr_queues; i++) {
+ struct nitrox_cmdq *cmdq;
+ u64 offset;
+
+ cmdq = &ndev->pkt_inq[i];
+ cmdq->ndev = ndev;
+ cmdq->qno = i;
+ cmdq->instr_size = sizeof(struct nps_pkt_instr);
+
+ /* packet input ring doorbell address */
+ offset = NPS_PKT_IN_INSTR_BAOFF_DBELLX(i);
+ cmdq->dbell_csr_addr = NITROX_CSR_ADDR(ndev, offset);
+ /* packet solicit port completion count address */
+ offset = NPS_PKT_SLC_CNTSX(i);
+ cmdq->compl_cnt_csr_addr = NITROX_CSR_ADDR(ndev, offset);
+
+ err = nitrox_cmdq_init(cmdq, PKTIN_Q_ALIGN_BYTES);
+ if (err)
+ goto pktq_fail;
+ }
+ return 0;
+
+pktq_fail:
+ nitrox_free_pktin_queues(ndev);
+ return err;
+}
+
+static int create_crypto_dma_pool(struct nitrox_device *ndev)
+{
+ size_t size;
+
+ /* Crypto context pool, 16 byte aligned */
+ size = CRYPTO_CTX_SIZE + sizeof(struct ctx_hdr);
+ ndev->ctx_pool = dma_pool_create("nitrox-context",
+ DEV(ndev), size, 16, 0);
+ if (!ndev->ctx_pool)
+ return -ENOMEM;
+
+ return 0;
+}
+
+static void destroy_crypto_dma_pool(struct nitrox_device *ndev)
+{
+ if (!ndev->ctx_pool)
+ return;
+
+ dma_pool_destroy(ndev->ctx_pool);
+ ndev->ctx_pool = NULL;
+}
+
+/*
+ * crypto_alloc_context - Allocate crypto context from pool
+ * @ndev: NITROX Device
+ */
+void *crypto_alloc_context(struct nitrox_device *ndev)
+{
+ struct ctx_hdr *ctx;
+ struct crypto_ctx_hdr *chdr;
+ void *vaddr;
+ dma_addr_t dma;
+
+ chdr = kmalloc(sizeof(*chdr), GFP_KERNEL);
+ if (!chdr)
+ return NULL;
+
+ vaddr = dma_pool_zalloc(ndev->ctx_pool, GFP_KERNEL, &dma);
+ if (!vaddr) {
+ kfree(chdr);
+ return NULL;
+ }
+
+ /* fill meta data */
+ ctx = vaddr;
+ ctx->pool = ndev->ctx_pool;
+ ctx->dma = dma;
+ ctx->ctx_dma = dma + sizeof(struct ctx_hdr);
+
+ chdr->pool = ndev->ctx_pool;
+ chdr->dma = dma;
+ chdr->vaddr = vaddr;
+
+ return chdr;
+}
+
+/**
+ * crypto_free_context - Free crypto context to pool
+ * @ctx: context to free
+ */
+void crypto_free_context(void *ctx)
+{
+ struct crypto_ctx_hdr *ctxp;
+
+ if (!ctx)
+ return;
+
+ ctxp = ctx;
+ dma_pool_free(ctxp->pool, ctxp->vaddr, ctxp->dma);
+ kfree(ctxp);
+}
+
+/**
+ * nitrox_common_sw_init - allocate software resources.
+ * @ndev: NITROX device
+ *
+ * Allocates crypto context pools and command queues etc.
+ *
+ * Return: 0 on success, or a negative error code on error.
+ */
+int nitrox_common_sw_init(struct nitrox_device *ndev)
+{
+ int err = 0;
+
+ /* per device crypto context pool */
+ err = create_crypto_dma_pool(ndev);
+ if (err)
+ return err;
+
+ err = nitrox_alloc_pktin_queues(ndev);
+ if (err)
+ destroy_crypto_dma_pool(ndev);
+
+ err = nitrox_alloc_aqm_queues(ndev);
+ if (err) {
+ nitrox_free_pktin_queues(ndev);
+ destroy_crypto_dma_pool(ndev);
+ }
+
+ return err;
+}
+
+/**
+ * nitrox_common_sw_cleanup - free software resources.
+ * @ndev: NITROX device
+ */
+void nitrox_common_sw_cleanup(struct nitrox_device *ndev)
+{
+ nitrox_free_aqm_queues(ndev);
+ nitrox_free_pktin_queues(ndev);
+ destroy_crypto_dma_pool(ndev);
+}
diff --git a/drivers/crypto/cavium/nitrox/nitrox_main.c b/drivers/crypto/cavium/nitrox/nitrox_main.c
new file mode 100644
index 000000000..432a61aca
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/nitrox_main.c
@@ -0,0 +1,583 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include <linux/aer.h>
+#include <linux/delay.h>
+#include <linux/firmware.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/pci.h>
+#include <linux/pci_ids.h>
+
+#include "nitrox_dev.h"
+#include "nitrox_common.h"
+#include "nitrox_csr.h"
+#include "nitrox_hal.h"
+#include "nitrox_isr.h"
+#include "nitrox_debugfs.h"
+
+#define CNN55XX_DEV_ID 0x12
+#define UCODE_HLEN 48
+#define DEFAULT_SE_GROUP 0
+#define DEFAULT_AE_GROUP 0
+
+#define DRIVER_VERSION "1.2"
+#define CNN55XX_UCD_BLOCK_SIZE 32768
+#define CNN55XX_MAX_UCODE_SIZE (CNN55XX_UCD_BLOCK_SIZE * 2)
+#define FW_DIR "cavium/"
+/* SE microcode */
+#define SE_FW FW_DIR "cnn55xx_se.fw"
+/* AE microcode */
+#define AE_FW FW_DIR "cnn55xx_ae.fw"
+
+static const char nitrox_driver_name[] = "CNN55XX";
+
+static LIST_HEAD(ndevlist);
+static DEFINE_MUTEX(devlist_lock);
+static unsigned int num_devices;
+
+/*
+ * nitrox_pci_tbl - PCI Device ID Table
+ */
+static const struct pci_device_id nitrox_pci_tbl[] = {
+ {PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0},
+ /* required last entry */
+ {0, }
+};
+MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl);
+
+static unsigned int qlen = DEFAULT_CMD_QLEN;
+module_param(qlen, uint, 0644);
+MODULE_PARM_DESC(qlen, "Command queue length - default 2048");
+
+/**
+ * struct ucode - Firmware Header
+ * @id: microcode ID
+ * @version: firmware version
+ * @code_size: code section size
+ * @raz: alignment
+ * @code: code section
+ */
+struct ucode {
+ u8 id;
+ char version[VERSION_LEN - 1];
+ __be32 code_size;
+ u8 raz[12];
+ u64 code[];
+};
+
+/*
+ * write_to_ucd_unit - Write Firmware to NITROX UCD unit
+ */
+static void write_to_ucd_unit(struct nitrox_device *ndev, u32 ucode_size,
+ u64 *ucode_data, int block_num)
+{
+ u32 code_size;
+ u64 offset, data;
+ int i = 0;
+
+ /*
+ * UCD structure
+ *
+ * -------------
+ * | BLK 7 |
+ * -------------
+ * | BLK 6 |
+ * -------------
+ * | ... |
+ * -------------
+ * | BLK 0 |
+ * -------------
+ * Total of 8 blocks, each size 32KB
+ */
+
+ /* set the block number */
+ offset = UCD_UCODE_LOAD_BLOCK_NUM;
+ nitrox_write_csr(ndev, offset, block_num);
+
+ code_size = roundup(ucode_size, 16);
+ while (code_size) {
+ data = ucode_data[i];
+ /* write 8 bytes at a time */
+ offset = UCD_UCODE_LOAD_IDX_DATAX(i);
+ nitrox_write_csr(ndev, offset, data);
+ code_size -= 8;
+ i++;
+ }
+
+ usleep_range(300, 400);
+}
+
+static int nitrox_load_fw(struct nitrox_device *ndev)
+{
+ const struct firmware *fw;
+ const char *fw_name;
+ struct ucode *ucode;
+ u64 *ucode_data;
+ u64 offset;
+ union ucd_core_eid_ucode_block_num core_2_eid_val;
+ union aqm_grp_execmsk_lo aqm_grp_execmask_lo;
+ union aqm_grp_execmsk_hi aqm_grp_execmask_hi;
+ u32 ucode_size;
+ int ret, i = 0;
+
+ fw_name = SE_FW;
+ dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
+
+ ret = request_firmware(&fw, fw_name, DEV(ndev));
+ if (ret < 0) {
+ dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
+ return ret;
+ }
+
+ ucode = (struct ucode *)fw->data;
+
+ ucode_size = be32_to_cpu(ucode->code_size) * 2;
+ if (!ucode_size || ucode_size > CNN55XX_MAX_UCODE_SIZE) {
+ dev_err(DEV(ndev), "Invalid ucode size: %u for firmware %s\n",
+ ucode_size, fw_name);
+ release_firmware(fw);
+ return -EINVAL;
+ }
+ ucode_data = ucode->code;
+
+ /* copy the firmware version */
+ memcpy(&ndev->hw.fw_name[0][0], ucode->version, (VERSION_LEN - 2));
+ ndev->hw.fw_name[0][VERSION_LEN - 1] = '\0';
+
+ /* Load SE Firmware on UCD Block 0 */
+ write_to_ucd_unit(ndev, ucode_size, ucode_data, 0);
+
+ release_firmware(fw);
+
+ /* put all SE cores in DEFAULT_SE_GROUP */
+ offset = POM_GRP_EXECMASKX(DEFAULT_SE_GROUP);
+ nitrox_write_csr(ndev, offset, (~0ULL));
+
+ /* write block number and firmware length
+ * bit:<2:0> block number
+ * bit:3 is set SE uses 32KB microcode
+ * bit:3 is clear SE uses 64KB microcode
+ */
+ core_2_eid_val.value = 0ULL;
+ core_2_eid_val.ucode_blk = 0;
+ if (ucode_size <= CNN55XX_UCD_BLOCK_SIZE)
+ core_2_eid_val.ucode_len = 1;
+ else
+ core_2_eid_val.ucode_len = 0;
+
+ for (i = 0; i < ndev->hw.se_cores; i++) {
+ offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i);
+ nitrox_write_csr(ndev, offset, core_2_eid_val.value);
+ }
+
+
+ fw_name = AE_FW;
+ dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
+
+ ret = request_firmware(&fw, fw_name, DEV(ndev));
+ if (ret < 0) {
+ dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
+ return ret;
+ }
+
+ ucode = (struct ucode *)fw->data;
+
+ ucode_size = be32_to_cpu(ucode->code_size) * 2;
+ if (!ucode_size || ucode_size > CNN55XX_MAX_UCODE_SIZE) {
+ dev_err(DEV(ndev), "Invalid ucode size: %u for firmware %s\n",
+ ucode_size, fw_name);
+ release_firmware(fw);
+ return -EINVAL;
+ }
+ ucode_data = ucode->code;
+
+ /* copy the firmware version */
+ memcpy(&ndev->hw.fw_name[1][0], ucode->version, (VERSION_LEN - 2));
+ ndev->hw.fw_name[1][VERSION_LEN - 1] = '\0';
+
+ /* Load AE Firmware on UCD Block 2 */
+ write_to_ucd_unit(ndev, ucode_size, ucode_data, 2);
+
+ release_firmware(fw);
+
+ /* put all AE cores in DEFAULT_AE_GROUP */
+ offset = AQM_GRP_EXECMSK_LOX(DEFAULT_AE_GROUP);
+ aqm_grp_execmask_lo.exec_0_to_39 = 0xFFFFFFFFFFULL;
+ nitrox_write_csr(ndev, offset, aqm_grp_execmask_lo.value);
+ offset = AQM_GRP_EXECMSK_HIX(DEFAULT_AE_GROUP);
+ aqm_grp_execmask_hi.exec_40_to_79 = 0xFFFFFFFFFFULL;
+ nitrox_write_csr(ndev, offset, aqm_grp_execmask_hi.value);
+
+ /* write block number and firmware length
+ * bit:<2:0> block number
+ * bit:3 is set AE uses 32KB microcode
+ * bit:3 is clear AE uses 64KB microcode
+ */
+ core_2_eid_val.value = 0ULL;
+ core_2_eid_val.ucode_blk = 2;
+ if (ucode_size <= CNN55XX_UCD_BLOCK_SIZE)
+ core_2_eid_val.ucode_len = 1;
+ else
+ core_2_eid_val.ucode_len = 0;
+
+ for (i = 0; i < ndev->hw.ae_cores; i++) {
+ offset = UCD_AE_EID_UCODE_BLOCK_NUMX(i);
+ nitrox_write_csr(ndev, offset, core_2_eid_val.value);
+ }
+
+ return 0;
+}
+
+/**
+ * nitrox_add_to_devlist - add NITROX device to global device list
+ * @ndev: NITROX device
+ */
+static int nitrox_add_to_devlist(struct nitrox_device *ndev)
+{
+ struct nitrox_device *dev;
+ int ret = 0;
+
+ INIT_LIST_HEAD(&ndev->list);
+ refcount_set(&ndev->refcnt, 1);
+
+ mutex_lock(&devlist_lock);
+ list_for_each_entry(dev, &ndevlist, list) {
+ if (dev == ndev) {
+ ret = -EEXIST;
+ goto unlock;
+ }
+ }
+ ndev->idx = num_devices++;
+ list_add_tail(&ndev->list, &ndevlist);
+unlock:
+ mutex_unlock(&devlist_lock);
+ return ret;
+}
+
+/**
+ * nitrox_remove_from_devlist - remove NITROX device from
+ * global device list
+ * @ndev: NITROX device
+ */
+static void nitrox_remove_from_devlist(struct nitrox_device *ndev)
+{
+ mutex_lock(&devlist_lock);
+ list_del(&ndev->list);
+ num_devices--;
+ mutex_unlock(&devlist_lock);
+}
+
+struct nitrox_device *nitrox_get_first_device(void)
+{
+ struct nitrox_device *ndev = NULL, *iter;
+
+ mutex_lock(&devlist_lock);
+ list_for_each_entry(iter, &ndevlist, list) {
+ if (nitrox_ready(iter)) {
+ ndev = iter;
+ break;
+ }
+ }
+ mutex_unlock(&devlist_lock);
+ if (!ndev)
+ return NULL;
+
+ refcount_inc(&ndev->refcnt);
+ /* barrier to sync with other cpus */
+ smp_mb__after_atomic();
+ return ndev;
+}
+
+void nitrox_put_device(struct nitrox_device *ndev)
+{
+ if (!ndev)
+ return;
+
+ refcount_dec(&ndev->refcnt);
+ /* barrier to sync with other cpus */
+ smp_mb__after_atomic();
+}
+
+static int nitrox_device_flr(struct pci_dev *pdev)
+{
+ int pos = 0;
+
+ pos = pci_save_state(pdev);
+ if (pos) {
+ dev_err(&pdev->dev, "Failed to save pci state\n");
+ return -ENOMEM;
+ }
+
+ pcie_reset_flr(pdev, PCI_RESET_DO_RESET);
+
+ pci_restore_state(pdev);
+
+ return 0;
+}
+
+static int nitrox_pf_sw_init(struct nitrox_device *ndev)
+{
+ int err;
+
+ err = nitrox_common_sw_init(ndev);
+ if (err)
+ return err;
+
+ err = nitrox_register_interrupts(ndev);
+ if (err)
+ nitrox_common_sw_cleanup(ndev);
+
+ return err;
+}
+
+static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev)
+{
+ nitrox_unregister_interrupts(ndev);
+ nitrox_common_sw_cleanup(ndev);
+}
+
+/**
+ * nitrox_bist_check - Check NITROX BIST registers status
+ * @ndev: NITROX device
+ */
+static int nitrox_bist_check(struct nitrox_device *ndev)
+{
+ u64 value = 0;
+ int i;
+
+ for (i = 0; i < NR_CLUSTERS; i++) {
+ value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i));
+ value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i));
+ }
+ value += nitrox_read_csr(ndev, UCD_BIST_STATUS);
+ value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG);
+ value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG);
+ value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG);
+ value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG);
+ value += nitrox_read_csr(ndev, POM_BIST_REG);
+ value += nitrox_read_csr(ndev, BMI_BIST_REG);
+ value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT);
+ value += nitrox_read_csr(ndev, BMO_BIST_REG);
+ value += nitrox_read_csr(ndev, LBC_BIST_STATUS);
+ value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0));
+ if (value)
+ return -EIO;
+ return 0;
+}
+
+static int nitrox_pf_hw_init(struct nitrox_device *ndev)
+{
+ int err;
+
+ err = nitrox_bist_check(ndev);
+ if (err) {
+ dev_err(&ndev->pdev->dev, "BIST check failed\n");
+ return err;
+ }
+ /* get cores information */
+ nitrox_get_hwinfo(ndev);
+
+ nitrox_config_nps_core_unit(ndev);
+ nitrox_config_aqm_unit(ndev);
+ nitrox_config_nps_pkt_unit(ndev);
+ nitrox_config_pom_unit(ndev);
+ nitrox_config_efl_unit(ndev);
+ /* configure IO units */
+ nitrox_config_bmi_unit(ndev);
+ nitrox_config_bmo_unit(ndev);
+ /* configure Local Buffer Cache */
+ nitrox_config_lbc_unit(ndev);
+ nitrox_config_rand_unit(ndev);
+
+ /* load firmware on cores */
+ err = nitrox_load_fw(ndev);
+ if (err)
+ return err;
+
+ nitrox_config_emu_unit(ndev);
+
+ return 0;
+}
+
+/**
+ * nitrox_probe - NITROX Initialization function.
+ * @pdev: PCI device information struct
+ * @id: entry in nitrox_pci_tbl
+ *
+ * Return: 0, if the driver is bound to the device, or
+ * a negative error if there is failure.
+ */
+static int nitrox_probe(struct pci_dev *pdev,
+ const struct pci_device_id *id)
+{
+ struct nitrox_device *ndev;
+ int err;
+
+ dev_info_once(&pdev->dev, "%s driver version %s\n",
+ nitrox_driver_name, DRIVER_VERSION);
+
+ err = pci_enable_device_mem(pdev);
+ if (err)
+ return err;
+
+ /* do FLR */
+ err = nitrox_device_flr(pdev);
+ if (err) {
+ dev_err(&pdev->dev, "FLR failed\n");
+ goto flr_fail;
+ }
+
+ if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
+ dev_dbg(&pdev->dev, "DMA to 64-BIT address\n");
+ } else {
+ err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
+ if (err) {
+ dev_err(&pdev->dev, "DMA configuration failed\n");
+ goto flr_fail;
+ }
+ }
+
+ err = pci_request_mem_regions(pdev, nitrox_driver_name);
+ if (err)
+ goto flr_fail;
+ pci_set_master(pdev);
+
+ ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
+ if (!ndev) {
+ err = -ENOMEM;
+ goto ndev_fail;
+ }
+
+ pci_set_drvdata(pdev, ndev);
+ ndev->pdev = pdev;
+
+ /* add to device list */
+ nitrox_add_to_devlist(ndev);
+
+ ndev->hw.vendor_id = pdev->vendor;
+ ndev->hw.device_id = pdev->device;
+ ndev->hw.revision_id = pdev->revision;
+ /* command timeout in jiffies */
+ ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT);
+ ndev->node = dev_to_node(&pdev->dev);
+ if (ndev->node == NUMA_NO_NODE)
+ ndev->node = 0;
+
+ ndev->bar_addr = ioremap(pci_resource_start(pdev, 0),
+ pci_resource_len(pdev, 0));
+ if (!ndev->bar_addr) {
+ err = -EIO;
+ goto ioremap_err;
+ }
+ /* allocate command queus based on cpus, max queues are 64 */
+ ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus());
+ ndev->qlen = qlen;
+
+ err = nitrox_pf_sw_init(ndev);
+ if (err)
+ goto pf_sw_fail;
+
+ err = nitrox_pf_hw_init(ndev);
+ if (err)
+ goto pf_hw_fail;
+
+ nitrox_debugfs_init(ndev);
+
+ /* clear the statistics */
+ atomic64_set(&ndev->stats.posted, 0);
+ atomic64_set(&ndev->stats.completed, 0);
+ atomic64_set(&ndev->stats.dropped, 0);
+
+ atomic_set(&ndev->state, __NDEV_READY);
+ /* barrier to sync with other cpus */
+ smp_mb__after_atomic();
+
+ err = nitrox_crypto_register();
+ if (err)
+ goto crypto_fail;
+
+ return 0;
+
+crypto_fail:
+ nitrox_debugfs_exit(ndev);
+ atomic_set(&ndev->state, __NDEV_NOT_READY);
+ /* barrier to sync with other cpus */
+ smp_mb__after_atomic();
+pf_hw_fail:
+ nitrox_pf_sw_cleanup(ndev);
+pf_sw_fail:
+ iounmap(ndev->bar_addr);
+ioremap_err:
+ nitrox_remove_from_devlist(ndev);
+ kfree(ndev);
+ pci_set_drvdata(pdev, NULL);
+ndev_fail:
+ pci_release_mem_regions(pdev);
+flr_fail:
+ pci_disable_device(pdev);
+ return err;
+}
+
+/**
+ * nitrox_remove - Unbind the driver from the device.
+ * @pdev: PCI device information struct
+ */
+static void nitrox_remove(struct pci_dev *pdev)
+{
+ struct nitrox_device *ndev = pci_get_drvdata(pdev);
+
+ if (!ndev)
+ return;
+
+ if (!refcount_dec_and_test(&ndev->refcnt)) {
+ dev_err(DEV(ndev), "Device refcnt not zero (%d)\n",
+ refcount_read(&ndev->refcnt));
+ return;
+ }
+
+ dev_info(DEV(ndev), "Removing Device %x:%x\n",
+ ndev->hw.vendor_id, ndev->hw.device_id);
+
+ atomic_set(&ndev->state, __NDEV_NOT_READY);
+ /* barrier to sync with other cpus */
+ smp_mb__after_atomic();
+
+ nitrox_remove_from_devlist(ndev);
+
+ /* disable SR-IOV */
+ nitrox_sriov_configure(pdev, 0);
+ nitrox_crypto_unregister();
+ nitrox_debugfs_exit(ndev);
+ nitrox_pf_sw_cleanup(ndev);
+
+ iounmap(ndev->bar_addr);
+ kfree(ndev);
+
+ pci_set_drvdata(pdev, NULL);
+ pci_release_mem_regions(pdev);
+ pci_disable_device(pdev);
+}
+
+static void nitrox_shutdown(struct pci_dev *pdev)
+{
+ pci_set_drvdata(pdev, NULL);
+ pci_release_mem_regions(pdev);
+ pci_disable_device(pdev);
+}
+
+static struct pci_driver nitrox_driver = {
+ .name = nitrox_driver_name,
+ .id_table = nitrox_pci_tbl,
+ .probe = nitrox_probe,
+ .remove = nitrox_remove,
+ .shutdown = nitrox_shutdown,
+ .sriov_configure = nitrox_sriov_configure,
+};
+
+module_pci_driver(nitrox_driver);
+
+MODULE_AUTHOR("Srikanth Jampala <Jampala.Srikanth@cavium.com>");
+MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " ");
+MODULE_LICENSE("GPL");
+MODULE_VERSION(DRIVER_VERSION);
+MODULE_FIRMWARE(SE_FW);
diff --git a/drivers/crypto/cavium/nitrox/nitrox_mbx.c b/drivers/crypto/cavium/nitrox/nitrox_mbx.c
new file mode 100644
index 000000000..d4e06999a
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/nitrox_mbx.c
@@ -0,0 +1,218 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/bitmap.h>
+#include <linux/workqueue.h>
+
+#include "nitrox_csr.h"
+#include "nitrox_hal.h"
+#include "nitrox_dev.h"
+#include "nitrox_mbx.h"
+
+#define RING_TO_VFNO(_x, _y) ((_x) / (_y))
+
+/*
+ * mbx_msg_type - Mailbox message types
+ */
+enum mbx_msg_type {
+ MBX_MSG_TYPE_NOP,
+ MBX_MSG_TYPE_REQ,
+ MBX_MSG_TYPE_ACK,
+ MBX_MSG_TYPE_NACK,
+};
+
+/*
+ * mbx_msg_opcode - Mailbox message opcodes
+ */
+enum mbx_msg_opcode {
+ MSG_OP_VF_MODE = 1,
+ MSG_OP_VF_UP,
+ MSG_OP_VF_DOWN,
+ MSG_OP_CHIPID_VFID,
+ MSG_OP_MCODE_INFO = 11,
+};
+
+struct pf2vf_work {
+ struct nitrox_vfdev *vfdev;
+ struct nitrox_device *ndev;
+ struct work_struct pf2vf_resp;
+};
+
+static inline u64 pf2vf_read_mbox(struct nitrox_device *ndev, int ring)
+{
+ u64 reg_addr;
+
+ reg_addr = NPS_PKT_MBOX_VF_PF_PFDATAX(ring);
+ return nitrox_read_csr(ndev, reg_addr);
+}
+
+static inline void pf2vf_write_mbox(struct nitrox_device *ndev, u64 value,
+ int ring)
+{
+ u64 reg_addr;
+
+ reg_addr = NPS_PKT_MBOX_PF_VF_PFDATAX(ring);
+ nitrox_write_csr(ndev, reg_addr, value);
+}
+
+static void pf2vf_send_response(struct nitrox_device *ndev,
+ struct nitrox_vfdev *vfdev)
+{
+ union mbox_msg msg;
+
+ msg.value = vfdev->msg.value;
+
+ switch (vfdev->msg.opcode) {
+ case MSG_OP_VF_MODE:
+ msg.data = ndev->mode;
+ break;
+ case MSG_OP_VF_UP:
+ vfdev->nr_queues = vfdev->msg.data;
+ atomic_set(&vfdev->state, __NDEV_READY);
+ break;
+ case MSG_OP_CHIPID_VFID:
+ msg.id.chipid = ndev->idx;
+ msg.id.vfid = vfdev->vfno;
+ break;
+ case MSG_OP_VF_DOWN:
+ vfdev->nr_queues = 0;
+ atomic_set(&vfdev->state, __NDEV_NOT_READY);
+ break;
+ case MSG_OP_MCODE_INFO:
+ msg.data = 0;
+ msg.mcode_info.count = 2;
+ msg.mcode_info.info = MCODE_TYPE_SE_SSL | (MCODE_TYPE_AE << 5);
+ msg.mcode_info.next_se_grp = 1;
+ msg.mcode_info.next_ae_grp = 1;
+ break;
+ default:
+ msg.type = MBX_MSG_TYPE_NOP;
+ break;
+ }
+
+ if (msg.type == MBX_MSG_TYPE_NOP)
+ return;
+
+ /* send ACK to VF */
+ msg.type = MBX_MSG_TYPE_ACK;
+ pf2vf_write_mbox(ndev, msg.value, vfdev->ring);
+
+ vfdev->msg.value = 0;
+ atomic64_inc(&vfdev->mbx_resp);
+}
+
+static void pf2vf_resp_handler(struct work_struct *work)
+{
+ struct pf2vf_work *pf2vf_resp = container_of(work, struct pf2vf_work,
+ pf2vf_resp);
+ struct nitrox_vfdev *vfdev = pf2vf_resp->vfdev;
+ struct nitrox_device *ndev = pf2vf_resp->ndev;
+
+ switch (vfdev->msg.type) {
+ case MBX_MSG_TYPE_REQ:
+ /* process the request from VF */
+ pf2vf_send_response(ndev, vfdev);
+ break;
+ case MBX_MSG_TYPE_ACK:
+ case MBX_MSG_TYPE_NACK:
+ break;
+ }
+
+ kfree(pf2vf_resp);
+}
+
+void nitrox_pf2vf_mbox_handler(struct nitrox_device *ndev)
+{
+ DECLARE_BITMAP(csr, BITS_PER_TYPE(u64));
+ struct nitrox_vfdev *vfdev;
+ struct pf2vf_work *pfwork;
+ u64 value, reg_addr;
+ u32 i;
+ int vfno;
+
+ /* loop for VF(0..63) */
+ reg_addr = NPS_PKT_MBOX_INT_LO;
+ value = nitrox_read_csr(ndev, reg_addr);
+ bitmap_from_u64(csr, value);
+ for_each_set_bit(i, csr, BITS_PER_TYPE(csr)) {
+ /* get the vfno from ring */
+ vfno = RING_TO_VFNO(i, ndev->iov.max_vf_queues);
+ vfdev = ndev->iov.vfdev + vfno;
+ vfdev->ring = i;
+ /* fill the vf mailbox data */
+ vfdev->msg.value = pf2vf_read_mbox(ndev, vfdev->ring);
+ pfwork = kzalloc(sizeof(*pfwork), GFP_ATOMIC);
+ if (!pfwork)
+ continue;
+
+ pfwork->vfdev = vfdev;
+ pfwork->ndev = ndev;
+ INIT_WORK(&pfwork->pf2vf_resp, pf2vf_resp_handler);
+ queue_work(ndev->iov.pf2vf_wq, &pfwork->pf2vf_resp);
+ /* clear the corresponding vf bit */
+ nitrox_write_csr(ndev, reg_addr, BIT_ULL(i));
+ }
+
+ /* loop for VF(64..127) */
+ reg_addr = NPS_PKT_MBOX_INT_HI;
+ value = nitrox_read_csr(ndev, reg_addr);
+ bitmap_from_u64(csr, value);
+ for_each_set_bit(i, csr, BITS_PER_TYPE(csr)) {
+ /* get the vfno from ring */
+ vfno = RING_TO_VFNO(i + 64, ndev->iov.max_vf_queues);
+ vfdev = ndev->iov.vfdev + vfno;
+ vfdev->ring = (i + 64);
+ /* fill the vf mailbox data */
+ vfdev->msg.value = pf2vf_read_mbox(ndev, vfdev->ring);
+
+ pfwork = kzalloc(sizeof(*pfwork), GFP_ATOMIC);
+ if (!pfwork)
+ continue;
+
+ pfwork->vfdev = vfdev;
+ pfwork->ndev = ndev;
+ INIT_WORK(&pfwork->pf2vf_resp, pf2vf_resp_handler);
+ queue_work(ndev->iov.pf2vf_wq, &pfwork->pf2vf_resp);
+ /* clear the corresponding vf bit */
+ nitrox_write_csr(ndev, reg_addr, BIT_ULL(i));
+ }
+}
+
+int nitrox_mbox_init(struct nitrox_device *ndev)
+{
+ struct nitrox_vfdev *vfdev;
+ int i;
+
+ ndev->iov.vfdev = kcalloc(ndev->iov.num_vfs,
+ sizeof(struct nitrox_vfdev), GFP_KERNEL);
+ if (!ndev->iov.vfdev)
+ return -ENOMEM;
+
+ for (i = 0; i < ndev->iov.num_vfs; i++) {
+ vfdev = ndev->iov.vfdev + i;
+ vfdev->vfno = i;
+ }
+
+ /* allocate pf2vf response workqueue */
+ ndev->iov.pf2vf_wq = alloc_workqueue("nitrox_pf2vf", 0, 0);
+ if (!ndev->iov.pf2vf_wq) {
+ kfree(ndev->iov.vfdev);
+ ndev->iov.vfdev = NULL;
+ return -ENOMEM;
+ }
+ /* enable pf2vf mailbox interrupts */
+ enable_pf2vf_mbox_interrupts(ndev);
+
+ return 0;
+}
+
+void nitrox_mbox_cleanup(struct nitrox_device *ndev)
+{
+ /* disable pf2vf mailbox interrupts */
+ disable_pf2vf_mbox_interrupts(ndev);
+ /* destroy workqueue */
+ if (ndev->iov.pf2vf_wq)
+ destroy_workqueue(ndev->iov.pf2vf_wq);
+
+ kfree(ndev->iov.vfdev);
+ ndev->iov.pf2vf_wq = NULL;
+ ndev->iov.vfdev = NULL;
+}
diff --git a/drivers/crypto/cavium/nitrox/nitrox_mbx.h b/drivers/crypto/cavium/nitrox/nitrox_mbx.h
new file mode 100644
index 000000000..7c93d0282
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/nitrox_mbx.h
@@ -0,0 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __NITROX_MBX_H
+#define __NITROX_MBX_H
+
+int nitrox_mbox_init(struct nitrox_device *ndev);
+void nitrox_mbox_cleanup(struct nitrox_device *ndev);
+void nitrox_pf2vf_mbox_handler(struct nitrox_device *ndev);
+
+#endif /* __NITROX_MBX_H */
diff --git a/drivers/crypto/cavium/nitrox/nitrox_req.h b/drivers/crypto/cavium/nitrox/nitrox_req.h
new file mode 100644
index 000000000..6bf088bcd
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/nitrox_req.h
@@ -0,0 +1,728 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __NITROX_REQ_H
+#define __NITROX_REQ_H
+
+#include <linux/dma-mapping.h>
+#include <crypto/aes.h>
+
+#include "nitrox_dev.h"
+
+#define PENDING_SIG 0xFFFFFFFFFFFFFFFFUL
+#define PRIO 4001
+
+typedef void (*sereq_completion_t)(void *req, int err);
+
+/**
+ * struct gphdr - General purpose Header
+ * @param0: first parameter.
+ * @param1: second parameter.
+ * @param2: third parameter.
+ * @param3: fourth parameter.
+ *
+ * Params tell the iv and enc/dec data offsets.
+ */
+struct gphdr {
+ __be16 param0;
+ __be16 param1;
+ __be16 param2;
+ __be16 param3;
+};
+
+/**
+ * struct se_req_ctrl - SE request information.
+ * @arg: Minor number of the opcode
+ * @ctxc: Context control.
+ * @unca: Uncertainity enabled.
+ * @info: Additional information for SE cores.
+ * @ctxl: Context length in bytes.
+ * @uddl: User defined data length
+ */
+union se_req_ctrl {
+ u64 value;
+ struct {
+ u64 raz : 22;
+ u64 arg : 8;
+ u64 ctxc : 2;
+ u64 unca : 1;
+ u64 info : 3;
+ u64 unc : 8;
+ u64 ctxl : 12;
+ u64 uddl : 8;
+ } s;
+};
+
+#define MAX_IV_LEN 16
+
+/**
+ * struct se_crypto_request - SE crypto request structure.
+ * @opcode: Request opcode (enc/dec)
+ * @flags: flags from crypto subsystem
+ * @ctx_handle: Crypto context handle.
+ * @gph: GP Header
+ * @ctrl: Request Information.
+ * @orh: ORH address
+ * @comp: completion address
+ * @src: Input sglist
+ * @dst: Output sglist
+ */
+struct se_crypto_request {
+ u8 opcode;
+ gfp_t gfp;
+ u32 flags;
+ u64 ctx_handle;
+
+ struct gphdr gph;
+ union se_req_ctrl ctrl;
+ u64 *orh;
+ u64 *comp;
+
+ struct scatterlist *src;
+ struct scatterlist *dst;
+};
+
+/* Crypto opcodes */
+#define FLEXI_CRYPTO_ENCRYPT_HMAC 0x33
+#define ENCRYPT 0
+#define DECRYPT 1
+
+/* IV from context */
+#define IV_FROM_CTX 0
+/* IV from Input data */
+#define IV_FROM_DPTR 1
+
+/**
+ * cipher opcodes for firmware
+ */
+enum flexi_cipher {
+ CIPHER_NULL = 0,
+ CIPHER_3DES_CBC,
+ CIPHER_3DES_ECB,
+ CIPHER_AES_CBC,
+ CIPHER_AES_ECB,
+ CIPHER_AES_CFB,
+ CIPHER_AES_CTR,
+ CIPHER_AES_GCM,
+ CIPHER_AES_XTS,
+ CIPHER_AES_CCM,
+ CIPHER_AES_CBC_CTS,
+ CIPHER_AES_ECB_CTS,
+ CIPHER_INVALID
+};
+
+enum flexi_auth {
+ AUTH_NULL = 0,
+ AUTH_MD5,
+ AUTH_SHA1,
+ AUTH_SHA2_SHA224,
+ AUTH_SHA2_SHA256,
+ AUTH_SHA2_SHA384,
+ AUTH_SHA2_SHA512,
+ AUTH_GMAC,
+ AUTH_INVALID
+};
+
+/**
+ * struct crypto_keys - Crypto keys
+ * @key: Encryption key or KEY1 for AES-XTS
+ * @iv: Encryption IV or Tweak for AES-XTS
+ */
+struct crypto_keys {
+ union {
+ u8 key[AES_MAX_KEY_SIZE];
+ u8 key1[AES_MAX_KEY_SIZE];
+ } u;
+ u8 iv[AES_BLOCK_SIZE];
+};
+
+/**
+ * struct auth_keys - Authentication keys
+ * @ipad: IPAD or KEY2 for AES-XTS
+ * @opad: OPAD or AUTH KEY if auth_input_type = 1
+ */
+struct auth_keys {
+ union {
+ u8 ipad[64];
+ u8 key2[64];
+ } u;
+ u8 opad[64];
+};
+
+union fc_ctx_flags {
+ __be64 f;
+ u64 fu;
+ struct {
+#if defined(__BIG_ENDIAN_BITFIELD)
+ u64 cipher_type : 4;
+ u64 reserved_59 : 1;
+ u64 aes_keylen : 2;
+ u64 iv_source : 1;
+ u64 hash_type : 4;
+ u64 reserved_49_51 : 3;
+ u64 auth_input_type: 1;
+ u64 mac_len : 8;
+ u64 reserved_0_39 : 40;
+#else
+ u64 reserved_0_39 : 40;
+ u64 mac_len : 8;
+ u64 auth_input_type: 1;
+ u64 reserved_49_51 : 3;
+ u64 hash_type : 4;
+ u64 iv_source : 1;
+ u64 aes_keylen : 2;
+ u64 reserved_59 : 1;
+ u64 cipher_type : 4;
+#endif
+ } w0;
+};
+/**
+ * struct flexi_crypto_context - Crypto context
+ * @cipher_type: Encryption cipher type
+ * @aes_keylen: AES key length
+ * @iv_source: Encryption IV source
+ * @hash_type: Authentication type
+ * @auth_input_type: Authentication input type
+ * 1 - Authentication IV and KEY, microcode calculates OPAD/IPAD
+ * 0 - Authentication OPAD/IPAD
+ * @mac_len: mac length
+ * @crypto: Crypto keys
+ * @auth: Authentication keys
+ */
+struct flexi_crypto_context {
+ union fc_ctx_flags flags;
+ struct crypto_keys crypto;
+ struct auth_keys auth;
+};
+
+struct crypto_ctx_hdr {
+ struct dma_pool *pool;
+ dma_addr_t dma;
+ void *vaddr;
+};
+
+struct nitrox_crypto_ctx {
+ struct nitrox_device *ndev;
+ union {
+ u64 ctx_handle;
+ struct flexi_crypto_context *fctx;
+ } u;
+ struct crypto_ctx_hdr *chdr;
+ sereq_completion_t callback;
+};
+
+struct nitrox_kcrypt_request {
+ struct se_crypto_request creq;
+ u8 *src;
+ u8 *dst;
+ u8 *iv_out;
+};
+
+/**
+ * struct nitrox_aead_rctx - AEAD request context
+ * @nkreq: Base request context
+ * @cryptlen: Encryption/Decryption data length
+ * @assoclen: AAD length
+ * @srclen: Input buffer length
+ * @dstlen: Output buffer length
+ * @iv: IV data
+ * @ivsize: IV data length
+ * @flags: AEAD req flags
+ * @ctx_handle: Device context handle
+ * @src: Source sglist
+ * @dst: Destination sglist
+ * @ctrl_arg: Identifies the request type (ENCRYPT/DECRYPT)
+ */
+struct nitrox_aead_rctx {
+ struct nitrox_kcrypt_request nkreq;
+ unsigned int cryptlen;
+ unsigned int assoclen;
+ unsigned int srclen;
+ unsigned int dstlen;
+ u8 *iv;
+ int ivsize;
+ u32 flags;
+ u64 ctx_handle;
+ struct scatterlist *src;
+ struct scatterlist *dst;
+ u8 ctrl_arg;
+};
+
+/**
+ * struct nitrox_rfc4106_rctx - rfc4106 cipher request context
+ * @base: AEAD request context
+ * @src: Source sglist
+ * @dst: Destination sglist
+ * @assoc: AAD
+ */
+struct nitrox_rfc4106_rctx {
+ struct nitrox_aead_rctx base;
+ struct scatterlist src[3];
+ struct scatterlist dst[3];
+ u8 assoc[20];
+};
+
+/**
+ * struct pkt_instr_hdr - Packet Instruction Header
+ * @g: Gather used
+ * When [G] is set and [GSZ] != 0, the instruction is
+ * indirect gather instruction.
+ * When [G] is set and [GSZ] = 0, the instruction is
+ * direct gather instruction.
+ * @gsz: Number of pointers in the indirect gather list
+ * @ihi: When set hardware duplicates the 1st 8 bytes of pkt_instr_hdr
+ * and adds them to the packet after the pkt_instr_hdr but before any UDD
+ * @ssz: Not used by the input hardware. But can become slc_store_int[SSZ]
+ * when [IHI] is set.
+ * @fsz: The number of front data bytes directly included in the
+ * PCIe instruction.
+ * @tlen: The length of the input packet in bytes, include:
+ * - 16B pkt_hdr
+ * - Inline context bytes if any,
+ * - UDD if any,
+ * - packet payload bytes
+ */
+union pkt_instr_hdr {
+ __be64 bev;
+ u64 value;
+ struct {
+#if defined(__BIG_ENDIAN_BITFIELD)
+ u64 raz_48_63 : 16;
+ u64 g : 1;
+ u64 gsz : 7;
+ u64 ihi : 1;
+ u64 ssz : 7;
+ u64 raz_30_31 : 2;
+ u64 fsz : 6;
+ u64 raz_16_23 : 8;
+ u64 tlen : 16;
+#else
+ u64 tlen : 16;
+ u64 raz_16_23 : 8;
+ u64 fsz : 6;
+ u64 raz_30_31 : 2;
+ u64 ssz : 7;
+ u64 ihi : 1;
+ u64 gsz : 7;
+ u64 g : 1;
+ u64 raz_48_63 : 16;
+#endif
+ } s;
+};
+
+/**
+ * struct pkt_hdr - Packet Input Header
+ * @opcode: Request opcode (Major)
+ * @arg: Request opcode (Minor)
+ * @ctxc: Context control.
+ * @unca: When set [UNC] is the uncertainty count for an input packet.
+ * The hardware uses uncertainty counts to predict
+ * output buffer use and avoid deadlock.
+ * @info: Not used by input hardware. Available for use
+ * during SE processing.
+ * @destport: The expected destination port/ring/channel for the packet.
+ * @unc: Uncertainty count for an input packet.
+ * @grp: SE group that will process the input packet.
+ * @ctxl: Context Length in 64-bit words.
+ * @uddl: User-defined data (UDD) length in bytes.
+ * @ctxp: Context pointer. CTXP<63,2:0> must be zero in all cases.
+ */
+union pkt_hdr {
+ __be64 bev[2];
+ u64 value[2];
+ struct {
+#if defined(__BIG_ENDIAN_BITFIELD)
+ u64 opcode : 8;
+ u64 arg : 8;
+ u64 ctxc : 2;
+ u64 unca : 1;
+ u64 raz_44 : 1;
+ u64 info : 3;
+ u64 destport : 9;
+ u64 unc : 8;
+ u64 raz_19_23 : 5;
+ u64 grp : 3;
+ u64 raz_15 : 1;
+ u64 ctxl : 7;
+ u64 uddl : 8;
+#else
+ u64 uddl : 8;
+ u64 ctxl : 7;
+ u64 raz_15 : 1;
+ u64 grp : 3;
+ u64 raz_19_23 : 5;
+ u64 unc : 8;
+ u64 destport : 9;
+ u64 info : 3;
+ u64 raz_44 : 1;
+ u64 unca : 1;
+ u64 ctxc : 2;
+ u64 arg : 8;
+ u64 opcode : 8;
+#endif
+ __be64 ctxp;
+ } s;
+};
+
+/**
+ * struct slc_store_info - Solicited Paceket Output Store Information.
+ * @ssz: The number of scatterlist pointers for the solicited output port
+ * packet.
+ * @rptr: The result pointer for the solicited output port packet.
+ * If [SSZ]=0, [RPTR] must point directly to a buffer on the remote
+ * host that is large enough to hold the entire output packet.
+ * If [SSZ]!=0, [RPTR] must point to an array of ([SSZ]+3)/4
+ * sglist components at [RPTR] on the remote host.
+ */
+union slc_store_info {
+ __be64 bev[2];
+ u64 value[2];
+ struct {
+#if defined(__BIG_ENDIAN_BITFIELD)
+ u64 raz_39_63 : 25;
+ u64 ssz : 7;
+ u64 raz_0_31 : 32;
+#else
+ u64 raz_0_31 : 32;
+ u64 ssz : 7;
+ u64 raz_39_63 : 25;
+#endif
+ __be64 rptr;
+ } s;
+};
+
+/**
+ * struct nps_pkt_instr - NPS Packet Instruction of SE cores.
+ * @dptr0 : Input pointer points to buffer in remote host.
+ * @ih: Packet Instruction Header (8 bytes)
+ * @irh: Packet Input Header (16 bytes)
+ * @slc: Solicited Packet Output Store Information (16 bytes)
+ * @fdata: Front data
+ *
+ * 64-Byte Instruction Format
+ */
+struct nps_pkt_instr {
+ __be64 dptr0;
+ union pkt_instr_hdr ih;
+ union pkt_hdr irh;
+ union slc_store_info slc;
+ u64 fdata[2];
+};
+
+/**
+ * struct aqmq_command_s - The 32 byte command for AE processing.
+ * @opcode: Request opcode
+ * @param1: Request control parameter 1
+ * @param2: Request control parameter 2
+ * @dlen: Input length
+ * @dptr: Input pointer points to buffer in remote host
+ * @rptr: Result pointer points to buffer in remote host
+ * @grp: AQM Group (0..7)
+ * @cptr: Context pointer
+ */
+struct aqmq_command_s {
+ __be16 opcode;
+ __be16 param1;
+ __be16 param2;
+ __be16 dlen;
+ __be64 dptr;
+ __be64 rptr;
+ union {
+ __be64 word3;
+#if defined(__BIG_ENDIAN_BITFIELD)
+ u64 grp : 3;
+ u64 cptr : 61;
+#else
+ u64 cptr : 61;
+ u64 grp : 3;
+#endif
+ };
+};
+
+/**
+ * struct ctx_hdr - Book keeping data about the crypto context
+ * @pool: Pool used to allocate crypto context
+ * @dma: Base DMA address of the crypto context
+ * @ctx_dma: Actual usable crypto context for NITROX
+ */
+struct ctx_hdr {
+ struct dma_pool *pool;
+ dma_addr_t dma;
+ dma_addr_t ctx_dma;
+};
+
+/*
+ * struct sglist_component - SG list component format
+ * @len0: The number of bytes at [PTR0] on the remote host.
+ * @len1: The number of bytes at [PTR1] on the remote host.
+ * @len2: The number of bytes at [PTR2] on the remote host.
+ * @len3: The number of bytes at [PTR3] on the remote host.
+ * @dma0: First pointer point to buffer in remote host.
+ * @dma1: Second pointer point to buffer in remote host.
+ * @dma2: Third pointer point to buffer in remote host.
+ * @dma3: Fourth pointer point to buffer in remote host.
+ */
+struct nitrox_sgcomp {
+ __be16 len[4];
+ __be64 dma[4];
+};
+
+/*
+ * strutct nitrox_sgtable - SG list information
+ * @sgmap_cnt: Number of buffers mapped
+ * @total_bytes: Total bytes in sglist.
+ * @sgcomp_len: Total sglist components length.
+ * @sgcomp_dma: DMA address of sglist component.
+ * @sg: crypto request buffer.
+ * @sgcomp: sglist component for NITROX.
+ */
+struct nitrox_sgtable {
+ u8 sgmap_cnt;
+ u16 total_bytes;
+ u32 sgcomp_len;
+ dma_addr_t sgcomp_dma;
+ struct scatterlist *sg;
+ struct nitrox_sgcomp *sgcomp;
+};
+
+/* Response Header Length */
+#define ORH_HLEN 8
+/* Completion bytes Length */
+#define COMP_HLEN 8
+
+struct resp_hdr {
+ u64 *orh;
+ u64 *completion;
+};
+
+typedef void (*completion_t)(void *arg, int err);
+
+/**
+ * struct nitrox_softreq - Represents the NIROX Request.
+ * @response: response list entry
+ * @backlog: Backlog list entry
+ * @ndev: Device used to submit the request
+ * @cmdq: Command queue for submission
+ * @resp: Response headers
+ * @instr: 64B instruction
+ * @in: SG table for input
+ * @out SG table for output
+ * @tstamp: Request submitted time in jiffies
+ * @callback: callback after request completion/timeout
+ * @cb_arg: callback argument
+ */
+struct nitrox_softreq {
+ struct list_head response;
+ struct list_head backlog;
+
+ u32 flags;
+ gfp_t gfp;
+ atomic_t status;
+
+ struct nitrox_device *ndev;
+ struct nitrox_cmdq *cmdq;
+
+ struct nps_pkt_instr instr;
+ struct resp_hdr resp;
+ struct nitrox_sgtable in;
+ struct nitrox_sgtable out;
+
+ unsigned long tstamp;
+
+ completion_t callback;
+ void *cb_arg;
+};
+
+static inline int flexi_aes_keylen(int keylen)
+{
+ int aes_keylen;
+
+ switch (keylen) {
+ case AES_KEYSIZE_128:
+ aes_keylen = 1;
+ break;
+ case AES_KEYSIZE_192:
+ aes_keylen = 2;
+ break;
+ case AES_KEYSIZE_256:
+ aes_keylen = 3;
+ break;
+ default:
+ aes_keylen = -EINVAL;
+ break;
+ }
+ return aes_keylen;
+}
+
+static inline void *alloc_req_buf(int nents, int extralen, gfp_t gfp)
+{
+ size_t size;
+
+ size = sizeof(struct scatterlist) * nents;
+ size += extralen;
+
+ return kzalloc(size, gfp);
+}
+
+/**
+ * create_single_sg - Point SG entry to the data
+ * @sg: Destination SG list
+ * @buf: Data
+ * @buflen: Data length
+ *
+ * Returns next free entry in the destination SG list
+ **/
+static inline struct scatterlist *create_single_sg(struct scatterlist *sg,
+ void *buf, int buflen)
+{
+ sg_set_buf(sg, buf, buflen);
+ sg++;
+ return sg;
+}
+
+/**
+ * create_multi_sg - Create multiple sg entries with buflen data length from
+ * source sglist
+ * @to_sg: Destination SG list
+ * @from_sg: Source SG list
+ * @buflen: Data length
+ *
+ * Returns next free entry in the destination SG list
+ **/
+static inline struct scatterlist *create_multi_sg(struct scatterlist *to_sg,
+ struct scatterlist *from_sg,
+ int buflen)
+{
+ struct scatterlist *sg = to_sg;
+ unsigned int sglen;
+
+ for (; buflen && from_sg; buflen -= sglen) {
+ sglen = from_sg->length;
+ if (sglen > buflen)
+ sglen = buflen;
+
+ sg_set_buf(sg, sg_virt(from_sg), sglen);
+ from_sg = sg_next(from_sg);
+ sg++;
+ }
+
+ return sg;
+}
+
+static inline void set_orh_value(u64 *orh)
+{
+ WRITE_ONCE(*orh, PENDING_SIG);
+}
+
+static inline void set_comp_value(u64 *comp)
+{
+ WRITE_ONCE(*comp, PENDING_SIG);
+}
+
+static inline int alloc_src_req_buf(struct nitrox_kcrypt_request *nkreq,
+ int nents, int ivsize)
+{
+ struct se_crypto_request *creq = &nkreq->creq;
+
+ nkreq->src = alloc_req_buf(nents, ivsize, creq->gfp);
+ if (!nkreq->src)
+ return -ENOMEM;
+
+ return 0;
+}
+
+static inline void nitrox_creq_copy_iv(char *dst, char *src, int size)
+{
+ memcpy(dst, src, size);
+}
+
+static inline struct scatterlist *nitrox_creq_src_sg(char *iv, int ivsize)
+{
+ return (struct scatterlist *)(iv + ivsize);
+}
+
+static inline void nitrox_creq_set_src_sg(struct nitrox_kcrypt_request *nkreq,
+ int nents, int ivsize,
+ struct scatterlist *src, int buflen)
+{
+ char *iv = nkreq->src;
+ struct scatterlist *sg;
+ struct se_crypto_request *creq = &nkreq->creq;
+
+ creq->src = nitrox_creq_src_sg(iv, ivsize);
+ sg = creq->src;
+ sg_init_table(sg, nents);
+
+ /* Input format:
+ * +----+----------------+
+ * | IV | SRC sg entries |
+ * +----+----------------+
+ */
+
+ /* IV */
+ sg = create_single_sg(sg, iv, ivsize);
+ /* SRC entries */
+ create_multi_sg(sg, src, buflen);
+}
+
+static inline int alloc_dst_req_buf(struct nitrox_kcrypt_request *nkreq,
+ int nents)
+{
+ int extralen = ORH_HLEN + COMP_HLEN;
+ struct se_crypto_request *creq = &nkreq->creq;
+
+ nkreq->dst = alloc_req_buf(nents, extralen, creq->gfp);
+ if (!nkreq->dst)
+ return -ENOMEM;
+
+ return 0;
+}
+
+static inline void nitrox_creq_set_orh(struct nitrox_kcrypt_request *nkreq)
+{
+ struct se_crypto_request *creq = &nkreq->creq;
+
+ creq->orh = (u64 *)(nkreq->dst);
+ set_orh_value(creq->orh);
+}
+
+static inline void nitrox_creq_set_comp(struct nitrox_kcrypt_request *nkreq)
+{
+ struct se_crypto_request *creq = &nkreq->creq;
+
+ creq->comp = (u64 *)(nkreq->dst + ORH_HLEN);
+ set_comp_value(creq->comp);
+}
+
+static inline struct scatterlist *nitrox_creq_dst_sg(char *dst)
+{
+ return (struct scatterlist *)(dst + ORH_HLEN + COMP_HLEN);
+}
+
+static inline void nitrox_creq_set_dst_sg(struct nitrox_kcrypt_request *nkreq,
+ int nents, int ivsize,
+ struct scatterlist *dst, int buflen)
+{
+ struct se_crypto_request *creq = &nkreq->creq;
+ struct scatterlist *sg;
+ char *iv = nkreq->src;
+
+ creq->dst = nitrox_creq_dst_sg(nkreq->dst);
+ sg = creq->dst;
+ sg_init_table(sg, nents);
+
+ /* Output format:
+ * +-----+----+----------------+-----------------+
+ * | ORH | IV | DST sg entries | COMPLETION Bytes|
+ * +-----+----+----------------+-----------------+
+ */
+
+ /* ORH */
+ sg = create_single_sg(sg, creq->orh, ORH_HLEN);
+ /* IV */
+ sg = create_single_sg(sg, iv, ivsize);
+ /* DST entries */
+ sg = create_multi_sg(sg, dst, buflen);
+ /* COMPLETION Bytes */
+ create_single_sg(sg, creq->comp, COMP_HLEN);
+}
+
+#endif /* __NITROX_REQ_H */
diff --git a/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c b/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c
new file mode 100644
index 000000000..55c18da4a
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c
@@ -0,0 +1,607 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/gfp.h>
+#include <linux/workqueue.h>
+#include <crypto/internal/skcipher.h>
+
+#include "nitrox_common.h"
+#include "nitrox_dev.h"
+#include "nitrox_req.h"
+#include "nitrox_csr.h"
+
+/* SLC_STORE_INFO */
+#define MIN_UDD_LEN 16
+/* PKT_IN_HDR + SLC_STORE_INFO */
+#define FDATA_SIZE 32
+/* Base destination port for the solicited requests */
+#define SOLICIT_BASE_DPORT 256
+
+#define REQ_NOT_POSTED 1
+#define REQ_BACKLOG 2
+#define REQ_POSTED 3
+
+/*
+ * Response codes from SE microcode
+ * 0x00 - Success
+ * Completion with no error
+ * 0x43 - ERR_GC_DATA_LEN_INVALID
+ * Invalid Data length if Encryption Data length is
+ * less than 16 bytes for AES-XTS and AES-CTS.
+ * 0x45 - ERR_GC_CTX_LEN_INVALID
+ * Invalid context length: CTXL != 23 words.
+ * 0x4F - ERR_GC_DOCSIS_CIPHER_INVALID
+ * DOCSIS support is enabled with other than
+ * AES/DES-CBC mode encryption.
+ * 0x50 - ERR_GC_DOCSIS_OFFSET_INVALID
+ * Authentication offset is other than 0 with
+ * Encryption IV source = 0.
+ * Authentication offset is other than 8 (DES)/16 (AES)
+ * with Encryption IV source = 1
+ * 0x51 - ERR_GC_CRC32_INVALID_SELECTION
+ * CRC32 is enabled for other than DOCSIS encryption.
+ * 0x52 - ERR_GC_AES_CCM_FLAG_INVALID
+ * Invalid flag options in AES-CCM IV.
+ */
+
+static inline int incr_index(int index, int count, int max)
+{
+ if ((index + count) >= max)
+ index = index + count - max;
+ else
+ index += count;
+
+ return index;
+}
+
+static void softreq_unmap_sgbufs(struct nitrox_softreq *sr)
+{
+ struct nitrox_device *ndev = sr->ndev;
+ struct device *dev = DEV(ndev);
+
+
+ dma_unmap_sg(dev, sr->in.sg, sg_nents(sr->in.sg),
+ DMA_BIDIRECTIONAL);
+ dma_unmap_single(dev, sr->in.sgcomp_dma, sr->in.sgcomp_len,
+ DMA_TO_DEVICE);
+ kfree(sr->in.sgcomp);
+ sr->in.sg = NULL;
+ sr->in.sgmap_cnt = 0;
+
+ dma_unmap_sg(dev, sr->out.sg, sg_nents(sr->out.sg),
+ DMA_BIDIRECTIONAL);
+ dma_unmap_single(dev, sr->out.sgcomp_dma, sr->out.sgcomp_len,
+ DMA_TO_DEVICE);
+ kfree(sr->out.sgcomp);
+ sr->out.sg = NULL;
+ sr->out.sgmap_cnt = 0;
+}
+
+static void softreq_destroy(struct nitrox_softreq *sr)
+{
+ softreq_unmap_sgbufs(sr);
+ kfree(sr);
+}
+
+/**
+ * create_sg_component - create SG componets for N5 device.
+ * @sr: Request structure
+ * @sgtbl: SG table
+ * @map_nents: number of dma mapped entries
+ *
+ * Component structure
+ *
+ * 63 48 47 32 31 16 15 0
+ * --------------------------------------
+ * | LEN0 | LEN1 | LEN2 | LEN3 |
+ * |-------------------------------------
+ * | PTR0 |
+ * --------------------------------------
+ * | PTR1 |
+ * --------------------------------------
+ * | PTR2 |
+ * --------------------------------------
+ * | PTR3 |
+ * --------------------------------------
+ *
+ * Returns 0 if success or a negative errno code on error.
+ */
+static int create_sg_component(struct nitrox_softreq *sr,
+ struct nitrox_sgtable *sgtbl, int map_nents)
+{
+ struct nitrox_device *ndev = sr->ndev;
+ struct nitrox_sgcomp *sgcomp;
+ struct scatterlist *sg;
+ dma_addr_t dma;
+ size_t sz_comp;
+ int i, j, nr_sgcomp;
+
+ nr_sgcomp = roundup(map_nents, 4) / 4;
+
+ /* each component holds 4 dma pointers */
+ sz_comp = nr_sgcomp * sizeof(*sgcomp);
+ sgcomp = kzalloc(sz_comp, sr->gfp);
+ if (!sgcomp)
+ return -ENOMEM;
+
+ sgtbl->sgcomp = sgcomp;
+
+ sg = sgtbl->sg;
+ /* populate device sg component */
+ for (i = 0; i < nr_sgcomp; i++) {
+ for (j = 0; j < 4 && sg; j++) {
+ sgcomp[i].len[j] = cpu_to_be16(sg_dma_len(sg));
+ sgcomp[i].dma[j] = cpu_to_be64(sg_dma_address(sg));
+ sg = sg_next(sg);
+ }
+ }
+ /* map the device sg component */
+ dma = dma_map_single(DEV(ndev), sgtbl->sgcomp, sz_comp, DMA_TO_DEVICE);
+ if (dma_mapping_error(DEV(ndev), dma)) {
+ kfree(sgtbl->sgcomp);
+ sgtbl->sgcomp = NULL;
+ return -ENOMEM;
+ }
+
+ sgtbl->sgcomp_dma = dma;
+ sgtbl->sgcomp_len = sz_comp;
+
+ return 0;
+}
+
+/**
+ * dma_map_inbufs - DMA map input sglist and creates sglist component
+ * for N5 device.
+ * @sr: Request structure
+ * @req: Crypto request structre
+ *
+ * Returns 0 if successful or a negative errno code on error.
+ */
+static int dma_map_inbufs(struct nitrox_softreq *sr,
+ struct se_crypto_request *req)
+{
+ struct device *dev = DEV(sr->ndev);
+ struct scatterlist *sg;
+ int i, nents, ret = 0;
+
+ nents = dma_map_sg(dev, req->src, sg_nents(req->src),
+ DMA_BIDIRECTIONAL);
+ if (!nents)
+ return -EINVAL;
+
+ for_each_sg(req->src, sg, nents, i)
+ sr->in.total_bytes += sg_dma_len(sg);
+
+ sr->in.sg = req->src;
+ sr->in.sgmap_cnt = nents;
+ ret = create_sg_component(sr, &sr->in, sr->in.sgmap_cnt);
+ if (ret)
+ goto incomp_err;
+
+ return 0;
+
+incomp_err:
+ dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_BIDIRECTIONAL);
+ sr->in.sgmap_cnt = 0;
+ return ret;
+}
+
+static int dma_map_outbufs(struct nitrox_softreq *sr,
+ struct se_crypto_request *req)
+{
+ struct device *dev = DEV(sr->ndev);
+ int nents, ret = 0;
+
+ nents = dma_map_sg(dev, req->dst, sg_nents(req->dst),
+ DMA_BIDIRECTIONAL);
+ if (!nents)
+ return -EINVAL;
+
+ sr->out.sg = req->dst;
+ sr->out.sgmap_cnt = nents;
+ ret = create_sg_component(sr, &sr->out, sr->out.sgmap_cnt);
+ if (ret)
+ goto outcomp_map_err;
+
+ return 0;
+
+outcomp_map_err:
+ dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_BIDIRECTIONAL);
+ sr->out.sgmap_cnt = 0;
+ sr->out.sg = NULL;
+ return ret;
+}
+
+static inline int softreq_map_iobuf(struct nitrox_softreq *sr,
+ struct se_crypto_request *creq)
+{
+ int ret;
+
+ ret = dma_map_inbufs(sr, creq);
+ if (ret)
+ return ret;
+
+ ret = dma_map_outbufs(sr, creq);
+ if (ret)
+ softreq_unmap_sgbufs(sr);
+
+ return ret;
+}
+
+static inline void backlog_list_add(struct nitrox_softreq *sr,
+ struct nitrox_cmdq *cmdq)
+{
+ INIT_LIST_HEAD(&sr->backlog);
+
+ spin_lock_bh(&cmdq->backlog_qlock);
+ list_add_tail(&sr->backlog, &cmdq->backlog_head);
+ atomic_inc(&cmdq->backlog_count);
+ atomic_set(&sr->status, REQ_BACKLOG);
+ spin_unlock_bh(&cmdq->backlog_qlock);
+}
+
+static inline void response_list_add(struct nitrox_softreq *sr,
+ struct nitrox_cmdq *cmdq)
+{
+ INIT_LIST_HEAD(&sr->response);
+
+ spin_lock_bh(&cmdq->resp_qlock);
+ list_add_tail(&sr->response, &cmdq->response_head);
+ spin_unlock_bh(&cmdq->resp_qlock);
+}
+
+static inline void response_list_del(struct nitrox_softreq *sr,
+ struct nitrox_cmdq *cmdq)
+{
+ spin_lock_bh(&cmdq->resp_qlock);
+ list_del(&sr->response);
+ spin_unlock_bh(&cmdq->resp_qlock);
+}
+
+static struct nitrox_softreq *
+get_first_response_entry(struct nitrox_cmdq *cmdq)
+{
+ return list_first_entry_or_null(&cmdq->response_head,
+ struct nitrox_softreq, response);
+}
+
+static inline bool cmdq_full(struct nitrox_cmdq *cmdq, int qlen)
+{
+ if (atomic_inc_return(&cmdq->pending_count) > qlen) {
+ atomic_dec(&cmdq->pending_count);
+ /* sync with other cpus */
+ smp_mb__after_atomic();
+ return true;
+ }
+ /* sync with other cpus */
+ smp_mb__after_atomic();
+ return false;
+}
+
+/**
+ * post_se_instr - Post SE instruction to Packet Input ring
+ * @sr: Request structure
+ * @cmdq: Command queue structure
+ *
+ * Returns 0 if successful or a negative error code,
+ * if no space in ring.
+ */
+static void post_se_instr(struct nitrox_softreq *sr,
+ struct nitrox_cmdq *cmdq)
+{
+ struct nitrox_device *ndev = sr->ndev;
+ int idx;
+ u8 *ent;
+
+ spin_lock_bh(&cmdq->cmd_qlock);
+
+ idx = cmdq->write_idx;
+ /* copy the instruction */
+ ent = cmdq->base + (idx * cmdq->instr_size);
+ memcpy(ent, &sr->instr, cmdq->instr_size);
+
+ atomic_set(&sr->status, REQ_POSTED);
+ response_list_add(sr, cmdq);
+ sr->tstamp = jiffies;
+ /* flush the command queue updates */
+ dma_wmb();
+
+ /* Ring doorbell with count 1 */
+ writeq(1, cmdq->dbell_csr_addr);
+
+ cmdq->write_idx = incr_index(idx, 1, ndev->qlen);
+
+ spin_unlock_bh(&cmdq->cmd_qlock);
+
+ /* increment the posted command count */
+ atomic64_inc(&ndev->stats.posted);
+}
+
+static int post_backlog_cmds(struct nitrox_cmdq *cmdq)
+{
+ struct nitrox_device *ndev = cmdq->ndev;
+ struct nitrox_softreq *sr, *tmp;
+ int ret = 0;
+
+ if (!atomic_read(&cmdq->backlog_count))
+ return 0;
+
+ spin_lock_bh(&cmdq->backlog_qlock);
+
+ list_for_each_entry_safe(sr, tmp, &cmdq->backlog_head, backlog) {
+ /* submit until space available */
+ if (unlikely(cmdq_full(cmdq, ndev->qlen))) {
+ ret = -ENOSPC;
+ break;
+ }
+ /* delete from backlog list */
+ list_del(&sr->backlog);
+ atomic_dec(&cmdq->backlog_count);
+ /* sync with other cpus */
+ smp_mb__after_atomic();
+
+ /* post the command */
+ post_se_instr(sr, cmdq);
+ }
+ spin_unlock_bh(&cmdq->backlog_qlock);
+
+ return ret;
+}
+
+static int nitrox_enqueue_request(struct nitrox_softreq *sr)
+{
+ struct nitrox_cmdq *cmdq = sr->cmdq;
+ struct nitrox_device *ndev = sr->ndev;
+
+ /* try to post backlog requests */
+ post_backlog_cmds(cmdq);
+
+ if (unlikely(cmdq_full(cmdq, ndev->qlen))) {
+ if (!(sr->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
+ /* increment drop count */
+ atomic64_inc(&ndev->stats.dropped);
+ return -ENOSPC;
+ }
+ /* add to backlog list */
+ backlog_list_add(sr, cmdq);
+ return -EINPROGRESS;
+ }
+ post_se_instr(sr, cmdq);
+
+ return -EINPROGRESS;
+}
+
+/**
+ * nitrox_process_se_request - Send request to SE core
+ * @ndev: NITROX device
+ * @req: Crypto request
+ * @callback: Completion callback
+ * @cb_arg: Completion callback arguments
+ *
+ * Returns 0 on success, or a negative error code.
+ */
+int nitrox_process_se_request(struct nitrox_device *ndev,
+ struct se_crypto_request *req,
+ completion_t callback,
+ void *cb_arg)
+{
+ struct nitrox_softreq *sr;
+ dma_addr_t ctx_handle = 0;
+ int qno, ret = 0;
+
+ if (!nitrox_ready(ndev))
+ return -ENODEV;
+
+ sr = kzalloc(sizeof(*sr), req->gfp);
+ if (!sr)
+ return -ENOMEM;
+
+ sr->ndev = ndev;
+ sr->flags = req->flags;
+ sr->gfp = req->gfp;
+ sr->callback = callback;
+ sr->cb_arg = cb_arg;
+
+ atomic_set(&sr->status, REQ_NOT_POSTED);
+
+ sr->resp.orh = req->orh;
+ sr->resp.completion = req->comp;
+
+ ret = softreq_map_iobuf(sr, req);
+ if (ret) {
+ kfree(sr);
+ return ret;
+ }
+
+ /* get the context handle */
+ if (req->ctx_handle) {
+ struct ctx_hdr *hdr;
+ u8 *ctx_ptr;
+
+ ctx_ptr = (u8 *)(uintptr_t)req->ctx_handle;
+ hdr = (struct ctx_hdr *)(ctx_ptr - sizeof(struct ctx_hdr));
+ ctx_handle = hdr->ctx_dma;
+ }
+
+ /* select the queue */
+ qno = smp_processor_id() % ndev->nr_queues;
+
+ sr->cmdq = &ndev->pkt_inq[qno];
+
+ /*
+ * 64-Byte Instruction Format
+ *
+ * ----------------------
+ * | DPTR0 | 8 bytes
+ * ----------------------
+ * | PKT_IN_INSTR_HDR | 8 bytes
+ * ----------------------
+ * | PKT_IN_HDR | 16 bytes
+ * ----------------------
+ * | SLC_INFO | 16 bytes
+ * ----------------------
+ * | Front data | 16 bytes
+ * ----------------------
+ */
+
+ /* fill the packet instruction */
+ /* word 0 */
+ sr->instr.dptr0 = cpu_to_be64(sr->in.sgcomp_dma);
+
+ /* word 1 */
+ sr->instr.ih.value = 0;
+ sr->instr.ih.s.g = 1;
+ sr->instr.ih.s.gsz = sr->in.sgmap_cnt;
+ sr->instr.ih.s.ssz = sr->out.sgmap_cnt;
+ sr->instr.ih.s.fsz = FDATA_SIZE + sizeof(struct gphdr);
+ sr->instr.ih.s.tlen = sr->instr.ih.s.fsz + sr->in.total_bytes;
+ sr->instr.ih.bev = cpu_to_be64(sr->instr.ih.value);
+
+ /* word 2 */
+ sr->instr.irh.value[0] = 0;
+ sr->instr.irh.s.uddl = MIN_UDD_LEN;
+ /* context length in 64-bit words */
+ sr->instr.irh.s.ctxl = (req->ctrl.s.ctxl / 8);
+ /* offset from solicit base port 256 */
+ sr->instr.irh.s.destport = SOLICIT_BASE_DPORT + qno;
+ sr->instr.irh.s.ctxc = req->ctrl.s.ctxc;
+ sr->instr.irh.s.arg = req->ctrl.s.arg;
+ sr->instr.irh.s.opcode = req->opcode;
+ sr->instr.irh.bev[0] = cpu_to_be64(sr->instr.irh.value[0]);
+
+ /* word 3 */
+ sr->instr.irh.s.ctxp = cpu_to_be64(ctx_handle);
+
+ /* word 4 */
+ sr->instr.slc.value[0] = 0;
+ sr->instr.slc.s.ssz = sr->out.sgmap_cnt;
+ sr->instr.slc.bev[0] = cpu_to_be64(sr->instr.slc.value[0]);
+
+ /* word 5 */
+ sr->instr.slc.s.rptr = cpu_to_be64(sr->out.sgcomp_dma);
+
+ /*
+ * No conversion for front data,
+ * It goes into payload
+ * put GP Header in front data
+ */
+ sr->instr.fdata[0] = *((u64 *)&req->gph);
+ sr->instr.fdata[1] = 0;
+
+ ret = nitrox_enqueue_request(sr);
+ if (ret == -ENOSPC)
+ goto send_fail;
+
+ return ret;
+
+send_fail:
+ softreq_destroy(sr);
+ return ret;
+}
+
+static inline int cmd_timeout(unsigned long tstamp, unsigned long timeout)
+{
+ return time_after_eq(jiffies, (tstamp + timeout));
+}
+
+void backlog_qflush_work(struct work_struct *work)
+{
+ struct nitrox_cmdq *cmdq;
+
+ cmdq = container_of(work, struct nitrox_cmdq, backlog_qflush);
+ post_backlog_cmds(cmdq);
+}
+
+static bool sr_completed(struct nitrox_softreq *sr)
+{
+ u64 orh = READ_ONCE(*sr->resp.orh);
+ unsigned long timeout = jiffies + msecs_to_jiffies(1);
+
+ if ((orh != PENDING_SIG) && (orh & 0xff))
+ return true;
+
+ while (READ_ONCE(*sr->resp.completion) == PENDING_SIG) {
+ if (time_after(jiffies, timeout)) {
+ pr_err("comp not done\n");
+ return false;
+ }
+ }
+
+ return true;
+}
+
+/**
+ * process_response_list - process completed requests
+ * @cmdq: Command queue structure
+ *
+ * Returns the number of responses processed.
+ */
+static void process_response_list(struct nitrox_cmdq *cmdq)
+{
+ struct nitrox_device *ndev = cmdq->ndev;
+ struct nitrox_softreq *sr;
+ int req_completed = 0, err = 0, budget;
+ completion_t callback;
+ void *cb_arg;
+
+ /* check all pending requests */
+ budget = atomic_read(&cmdq->pending_count);
+
+ while (req_completed < budget) {
+ sr = get_first_response_entry(cmdq);
+ if (!sr)
+ break;
+
+ if (atomic_read(&sr->status) != REQ_POSTED)
+ break;
+
+ /* check orh and completion bytes updates */
+ if (!sr_completed(sr)) {
+ /* request not completed, check for timeout */
+ if (!cmd_timeout(sr->tstamp, ndev->timeout))
+ break;
+ dev_err_ratelimited(DEV(ndev),
+ "Request timeout, orh 0x%016llx\n",
+ READ_ONCE(*sr->resp.orh));
+ }
+ atomic_dec(&cmdq->pending_count);
+ atomic64_inc(&ndev->stats.completed);
+ /* sync with other cpus */
+ smp_mb__after_atomic();
+ /* remove from response list */
+ response_list_del(sr, cmdq);
+ /* ORH error code */
+ err = READ_ONCE(*sr->resp.orh) & 0xff;
+ callback = sr->callback;
+ cb_arg = sr->cb_arg;
+ softreq_destroy(sr);
+ if (callback)
+ callback(cb_arg, err);
+
+ req_completed++;
+ }
+}
+
+/*
+ * pkt_slc_resp_tasklet - post processing of SE responses
+ */
+void pkt_slc_resp_tasklet(unsigned long data)
+{
+ struct nitrox_q_vector *qvec = (void *)(uintptr_t)(data);
+ struct nitrox_cmdq *cmdq = qvec->cmdq;
+ union nps_pkt_slc_cnts slc_cnts;
+
+ /* read completion count */
+ slc_cnts.value = readq(cmdq->compl_cnt_csr_addr);
+ /* resend the interrupt if more work to do */
+ slc_cnts.s.resend = 1;
+
+ process_response_list(cmdq);
+
+ /*
+ * clear the interrupt with resend bit enabled,
+ * MSI-X interrupt generates if Completion count > Threshold
+ */
+ writeq(slc_cnts.value, cmdq->compl_cnt_csr_addr);
+
+ if (atomic_read(&cmdq->backlog_count))
+ schedule_work(&cmdq->backlog_qflush);
+}
diff --git a/drivers/crypto/cavium/nitrox/nitrox_skcipher.c b/drivers/crypto/cavium/nitrox/nitrox_skcipher.c
new file mode 100644
index 000000000..248b4fff1
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/nitrox_skcipher.c
@@ -0,0 +1,552 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/crypto.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/printk.h>
+
+#include <crypto/aes.h>
+#include <crypto/skcipher.h>
+#include <crypto/scatterwalk.h>
+#include <crypto/ctr.h>
+#include <crypto/internal/des.h>
+#include <crypto/xts.h>
+
+#include "nitrox_dev.h"
+#include "nitrox_common.h"
+#include "nitrox_req.h"
+
+struct nitrox_cipher {
+ const char *name;
+ enum flexi_cipher value;
+};
+
+/*
+ * supported cipher list
+ */
+static const struct nitrox_cipher flexi_cipher_table[] = {
+ { "null", CIPHER_NULL },
+ { "cbc(des3_ede)", CIPHER_3DES_CBC },
+ { "ecb(des3_ede)", CIPHER_3DES_ECB },
+ { "cbc(aes)", CIPHER_AES_CBC },
+ { "ecb(aes)", CIPHER_AES_ECB },
+ { "cfb(aes)", CIPHER_AES_CFB },
+ { "rfc3686(ctr(aes))", CIPHER_AES_CTR },
+ { "xts(aes)", CIPHER_AES_XTS },
+ { "cts(cbc(aes))", CIPHER_AES_CBC_CTS },
+ { NULL, CIPHER_INVALID }
+};
+
+static enum flexi_cipher flexi_cipher_type(const char *name)
+{
+ const struct nitrox_cipher *cipher = flexi_cipher_table;
+
+ while (cipher->name) {
+ if (!strcmp(cipher->name, name))
+ break;
+ cipher++;
+ }
+ return cipher->value;
+}
+
+static void free_src_sglist(struct skcipher_request *skreq)
+{
+ struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);
+
+ kfree(nkreq->src);
+}
+
+static void free_dst_sglist(struct skcipher_request *skreq)
+{
+ struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);
+
+ kfree(nkreq->dst);
+}
+
+static void nitrox_skcipher_callback(void *arg, int err)
+{
+ struct skcipher_request *skreq = arg;
+
+ free_src_sglist(skreq);
+ free_dst_sglist(skreq);
+ if (err) {
+ pr_err_ratelimited("request failed status 0x%0x\n", err);
+ err = -EINVAL;
+ }
+
+ skcipher_request_complete(skreq, err);
+}
+
+static void nitrox_cbc_cipher_callback(void *arg, int err)
+{
+ struct skcipher_request *skreq = arg;
+ struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);
+ struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(skreq);
+ int ivsize = crypto_skcipher_ivsize(cipher);
+ unsigned int start = skreq->cryptlen - ivsize;
+
+ if (err) {
+ nitrox_skcipher_callback(arg, err);
+ return;
+ }
+
+ if (nkreq->creq.ctrl.s.arg == ENCRYPT) {
+ scatterwalk_map_and_copy(skreq->iv, skreq->dst, start, ivsize,
+ 0);
+ } else {
+ if (skreq->src != skreq->dst) {
+ scatterwalk_map_and_copy(skreq->iv, skreq->src, start,
+ ivsize, 0);
+ } else {
+ memcpy(skreq->iv, nkreq->iv_out, ivsize);
+ kfree(nkreq->iv_out);
+ }
+ }
+
+ nitrox_skcipher_callback(arg, err);
+}
+
+static int nitrox_skcipher_init(struct crypto_skcipher *tfm)
+{
+ struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(tfm);
+ struct crypto_ctx_hdr *chdr;
+
+ /* get the first device */
+ nctx->ndev = nitrox_get_first_device();
+ if (!nctx->ndev)
+ return -ENODEV;
+
+ /* allocate nitrox crypto context */
+ chdr = crypto_alloc_context(nctx->ndev);
+ if (!chdr) {
+ nitrox_put_device(nctx->ndev);
+ return -ENOMEM;
+ }
+
+ nctx->callback = nitrox_skcipher_callback;
+ nctx->chdr = chdr;
+ nctx->u.ctx_handle = (uintptr_t)((u8 *)chdr->vaddr +
+ sizeof(struct ctx_hdr));
+ crypto_skcipher_set_reqsize(tfm, crypto_skcipher_reqsize(tfm) +
+ sizeof(struct nitrox_kcrypt_request));
+ return 0;
+}
+
+static int nitrox_cbc_init(struct crypto_skcipher *tfm)
+{
+ int err;
+ struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(tfm);
+
+ err = nitrox_skcipher_init(tfm);
+ if (err)
+ return err;
+
+ nctx->callback = nitrox_cbc_cipher_callback;
+ return 0;
+}
+
+static void nitrox_skcipher_exit(struct crypto_skcipher *tfm)
+{
+ struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(tfm);
+
+ /* free the nitrox crypto context */
+ if (nctx->u.ctx_handle) {
+ struct flexi_crypto_context *fctx = nctx->u.fctx;
+
+ memzero_explicit(&fctx->crypto, sizeof(struct crypto_keys));
+ memzero_explicit(&fctx->auth, sizeof(struct auth_keys));
+ crypto_free_context((void *)nctx->chdr);
+ }
+ nitrox_put_device(nctx->ndev);
+
+ nctx->u.ctx_handle = 0;
+ nctx->ndev = NULL;
+}
+
+static inline int nitrox_skcipher_setkey(struct crypto_skcipher *cipher,
+ int aes_keylen, const u8 *key,
+ unsigned int keylen)
+{
+ struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher);
+ struct nitrox_crypto_ctx *nctx = crypto_tfm_ctx(tfm);
+ struct flexi_crypto_context *fctx;
+ union fc_ctx_flags *flags;
+ enum flexi_cipher cipher_type;
+ const char *name;
+
+ name = crypto_tfm_alg_name(tfm);
+ cipher_type = flexi_cipher_type(name);
+ if (unlikely(cipher_type == CIPHER_INVALID)) {
+ pr_err("unsupported cipher: %s\n", name);
+ return -EINVAL;
+ }
+
+ /* fill crypto context */
+ fctx = nctx->u.fctx;
+ flags = &fctx->flags;
+ flags->f = 0;
+ flags->w0.cipher_type = cipher_type;
+ flags->w0.aes_keylen = aes_keylen;
+ flags->w0.iv_source = IV_FROM_DPTR;
+ flags->f = cpu_to_be64(*(u64 *)&flags->w0);
+ /* copy the key to context */
+ memcpy(fctx->crypto.u.key, key, keylen);
+
+ return 0;
+}
+
+static int nitrox_aes_setkey(struct crypto_skcipher *cipher, const u8 *key,
+ unsigned int keylen)
+{
+ int aes_keylen;
+
+ aes_keylen = flexi_aes_keylen(keylen);
+ if (aes_keylen < 0)
+ return -EINVAL;
+ return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen);
+}
+
+static int alloc_src_sglist(struct skcipher_request *skreq, int ivsize)
+{
+ struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);
+ int nents = sg_nents(skreq->src) + 1;
+ int ret;
+
+ /* Allocate buffer to hold IV and input scatterlist array */
+ ret = alloc_src_req_buf(nkreq, nents, ivsize);
+ if (ret)
+ return ret;
+
+ nitrox_creq_copy_iv(nkreq->src, skreq->iv, ivsize);
+ nitrox_creq_set_src_sg(nkreq, nents, ivsize, skreq->src,
+ skreq->cryptlen);
+
+ return 0;
+}
+
+static int alloc_dst_sglist(struct skcipher_request *skreq, int ivsize)
+{
+ struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);
+ int nents = sg_nents(skreq->dst) + 3;
+ int ret;
+
+ /* Allocate buffer to hold ORH, COMPLETION and output scatterlist
+ * array
+ */
+ ret = alloc_dst_req_buf(nkreq, nents);
+ if (ret)
+ return ret;
+
+ nitrox_creq_set_orh(nkreq);
+ nitrox_creq_set_comp(nkreq);
+ nitrox_creq_set_dst_sg(nkreq, nents, ivsize, skreq->dst,
+ skreq->cryptlen);
+
+ return 0;
+}
+
+static int nitrox_skcipher_crypt(struct skcipher_request *skreq, bool enc)
+{
+ struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(skreq);
+ struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(cipher);
+ struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);
+ int ivsize = crypto_skcipher_ivsize(cipher);
+ struct se_crypto_request *creq;
+ int ret;
+
+ creq = &nkreq->creq;
+ creq->flags = skreq->base.flags;
+ creq->gfp = (skreq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
+ GFP_KERNEL : GFP_ATOMIC;
+
+ /* fill the request */
+ creq->ctrl.value = 0;
+ creq->opcode = FLEXI_CRYPTO_ENCRYPT_HMAC;
+ creq->ctrl.s.arg = (enc ? ENCRYPT : DECRYPT);
+ /* param0: length of the data to be encrypted */
+ creq->gph.param0 = cpu_to_be16(skreq->cryptlen);
+ creq->gph.param1 = 0;
+ /* param2: encryption data offset */
+ creq->gph.param2 = cpu_to_be16(ivsize);
+ creq->gph.param3 = 0;
+
+ creq->ctx_handle = nctx->u.ctx_handle;
+ creq->ctrl.s.ctxl = sizeof(struct flexi_crypto_context);
+
+ ret = alloc_src_sglist(skreq, ivsize);
+ if (ret)
+ return ret;
+
+ ret = alloc_dst_sglist(skreq, ivsize);
+ if (ret) {
+ free_src_sglist(skreq);
+ return ret;
+ }
+
+ /* send the crypto request */
+ return nitrox_process_se_request(nctx->ndev, creq, nctx->callback,
+ skreq);
+}
+
+static int nitrox_cbc_decrypt(struct skcipher_request *skreq)
+{
+ struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);
+ struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(skreq);
+ int ivsize = crypto_skcipher_ivsize(cipher);
+ gfp_t flags = (skreq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
+ GFP_KERNEL : GFP_ATOMIC;
+ unsigned int start = skreq->cryptlen - ivsize;
+
+ if (skreq->src != skreq->dst)
+ return nitrox_skcipher_crypt(skreq, false);
+
+ nkreq->iv_out = kmalloc(ivsize, flags);
+ if (!nkreq->iv_out)
+ return -ENOMEM;
+
+ scatterwalk_map_and_copy(nkreq->iv_out, skreq->src, start, ivsize, 0);
+ return nitrox_skcipher_crypt(skreq, false);
+}
+
+static int nitrox_aes_encrypt(struct skcipher_request *skreq)
+{
+ return nitrox_skcipher_crypt(skreq, true);
+}
+
+static int nitrox_aes_decrypt(struct skcipher_request *skreq)
+{
+ return nitrox_skcipher_crypt(skreq, false);
+}
+
+static int nitrox_3des_setkey(struct crypto_skcipher *cipher,
+ const u8 *key, unsigned int keylen)
+{
+ return verify_skcipher_des3_key(cipher, key) ?:
+ nitrox_skcipher_setkey(cipher, 0, key, keylen);
+}
+
+static int nitrox_3des_encrypt(struct skcipher_request *skreq)
+{
+ return nitrox_skcipher_crypt(skreq, true);
+}
+
+static int nitrox_3des_decrypt(struct skcipher_request *skreq)
+{
+ return nitrox_skcipher_crypt(skreq, false);
+}
+
+static int nitrox_aes_xts_setkey(struct crypto_skcipher *cipher,
+ const u8 *key, unsigned int keylen)
+{
+ struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher);
+ struct nitrox_crypto_ctx *nctx = crypto_tfm_ctx(tfm);
+ struct flexi_crypto_context *fctx;
+ int aes_keylen, ret;
+
+ ret = xts_check_key(tfm, key, keylen);
+ if (ret)
+ return ret;
+
+ keylen /= 2;
+
+ aes_keylen = flexi_aes_keylen(keylen);
+ if (aes_keylen < 0)
+ return -EINVAL;
+
+ fctx = nctx->u.fctx;
+ /* copy KEY2 */
+ memcpy(fctx->auth.u.key2, (key + keylen), keylen);
+
+ return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen);
+}
+
+static int nitrox_aes_ctr_rfc3686_setkey(struct crypto_skcipher *cipher,
+ const u8 *key, unsigned int keylen)
+{
+ struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher);
+ struct nitrox_crypto_ctx *nctx = crypto_tfm_ctx(tfm);
+ struct flexi_crypto_context *fctx;
+ int aes_keylen;
+
+ if (keylen < CTR_RFC3686_NONCE_SIZE)
+ return -EINVAL;
+
+ fctx = nctx->u.fctx;
+
+ memcpy(fctx->crypto.iv, key + (keylen - CTR_RFC3686_NONCE_SIZE),
+ CTR_RFC3686_NONCE_SIZE);
+
+ keylen -= CTR_RFC3686_NONCE_SIZE;
+
+ aes_keylen = flexi_aes_keylen(keylen);
+ if (aes_keylen < 0)
+ return -EINVAL;
+ return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen);
+}
+
+static struct skcipher_alg nitrox_skciphers[] = { {
+ .base = {
+ .cra_name = "cbc(aes)",
+ .cra_driver_name = "n5_cbc(aes)",
+ .cra_priority = PRIO,
+ .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ },
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = nitrox_aes_setkey,
+ .encrypt = nitrox_aes_encrypt,
+ .decrypt = nitrox_cbc_decrypt,
+ .init = nitrox_cbc_init,
+ .exit = nitrox_skcipher_exit,
+}, {
+ .base = {
+ .cra_name = "ecb(aes)",
+ .cra_driver_name = "n5_ecb(aes)",
+ .cra_priority = PRIO,
+ .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ },
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = nitrox_aes_setkey,
+ .encrypt = nitrox_aes_encrypt,
+ .decrypt = nitrox_aes_decrypt,
+ .init = nitrox_skcipher_init,
+ .exit = nitrox_skcipher_exit,
+}, {
+ .base = {
+ .cra_name = "cfb(aes)",
+ .cra_driver_name = "n5_cfb(aes)",
+ .cra_priority = PRIO,
+ .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ },
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = nitrox_aes_setkey,
+ .encrypt = nitrox_aes_encrypt,
+ .decrypt = nitrox_aes_decrypt,
+ .init = nitrox_skcipher_init,
+ .exit = nitrox_skcipher_exit,
+}, {
+ .base = {
+ .cra_name = "xts(aes)",
+ .cra_driver_name = "n5_xts(aes)",
+ .cra_priority = PRIO,
+ .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ },
+ .min_keysize = 2 * AES_MIN_KEY_SIZE,
+ .max_keysize = 2 * AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = nitrox_aes_xts_setkey,
+ .encrypt = nitrox_aes_encrypt,
+ .decrypt = nitrox_aes_decrypt,
+ .init = nitrox_skcipher_init,
+ .exit = nitrox_skcipher_exit,
+}, {
+ .base = {
+ .cra_name = "rfc3686(ctr(aes))",
+ .cra_driver_name = "n5_rfc3686(ctr(aes))",
+ .cra_priority = PRIO,
+ .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ },
+ .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
+ .ivsize = CTR_RFC3686_IV_SIZE,
+ .init = nitrox_skcipher_init,
+ .exit = nitrox_skcipher_exit,
+ .setkey = nitrox_aes_ctr_rfc3686_setkey,
+ .encrypt = nitrox_aes_encrypt,
+ .decrypt = nitrox_aes_decrypt,
+}, {
+ .base = {
+ .cra_name = "cts(cbc(aes))",
+ .cra_driver_name = "n5_cts(cbc(aes))",
+ .cra_priority = PRIO,
+ .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ },
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = nitrox_aes_setkey,
+ .encrypt = nitrox_aes_encrypt,
+ .decrypt = nitrox_aes_decrypt,
+ .init = nitrox_skcipher_init,
+ .exit = nitrox_skcipher_exit,
+}, {
+ .base = {
+ .cra_name = "cbc(des3_ede)",
+ .cra_driver_name = "n5_cbc(des3_ede)",
+ .cra_priority = PRIO,
+ .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
+ .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ },
+ .min_keysize = DES3_EDE_KEY_SIZE,
+ .max_keysize = DES3_EDE_KEY_SIZE,
+ .ivsize = DES3_EDE_BLOCK_SIZE,
+ .setkey = nitrox_3des_setkey,
+ .encrypt = nitrox_3des_encrypt,
+ .decrypt = nitrox_cbc_decrypt,
+ .init = nitrox_cbc_init,
+ .exit = nitrox_skcipher_exit,
+}, {
+ .base = {
+ .cra_name = "ecb(des3_ede)",
+ .cra_driver_name = "n5_ecb(des3_ede)",
+ .cra_priority = PRIO,
+ .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
+ .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ },
+ .min_keysize = DES3_EDE_KEY_SIZE,
+ .max_keysize = DES3_EDE_KEY_SIZE,
+ .ivsize = DES3_EDE_BLOCK_SIZE,
+ .setkey = nitrox_3des_setkey,
+ .encrypt = nitrox_3des_encrypt,
+ .decrypt = nitrox_3des_decrypt,
+ .init = nitrox_skcipher_init,
+ .exit = nitrox_skcipher_exit,
+}
+
+};
+
+int nitrox_register_skciphers(void)
+{
+ return crypto_register_skciphers(nitrox_skciphers,
+ ARRAY_SIZE(nitrox_skciphers));
+}
+
+void nitrox_unregister_skciphers(void)
+{
+ crypto_unregister_skciphers(nitrox_skciphers,
+ ARRAY_SIZE(nitrox_skciphers));
+}
diff --git a/drivers/crypto/cavium/nitrox/nitrox_sriov.c b/drivers/crypto/cavium/nitrox/nitrox_sriov.c
new file mode 100644
index 000000000..43287f847
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/nitrox_sriov.c
@@ -0,0 +1,234 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/pci.h>
+#include <linux/delay.h>
+
+#include "nitrox_dev.h"
+#include "nitrox_hal.h"
+#include "nitrox_common.h"
+#include "nitrox_isr.h"
+#include "nitrox_mbx.h"
+
+/**
+ * num_vfs_valid - validate VF count
+ * @num_vfs: number of VF(s)
+ */
+static inline bool num_vfs_valid(int num_vfs)
+{
+ bool valid = false;
+
+ switch (num_vfs) {
+ case 16:
+ case 32:
+ case 64:
+ case 128:
+ valid = true;
+ break;
+ }
+
+ return valid;
+}
+
+static inline enum vf_mode num_vfs_to_mode(int num_vfs)
+{
+ enum vf_mode mode = 0;
+
+ switch (num_vfs) {
+ case 0:
+ mode = __NDEV_MODE_PF;
+ break;
+ case 16:
+ mode = __NDEV_MODE_VF16;
+ break;
+ case 32:
+ mode = __NDEV_MODE_VF32;
+ break;
+ case 64:
+ mode = __NDEV_MODE_VF64;
+ break;
+ case 128:
+ mode = __NDEV_MODE_VF128;
+ break;
+ }
+
+ return mode;
+}
+
+static inline int vf_mode_to_nr_queues(enum vf_mode mode)
+{
+ int nr_queues = 0;
+
+ switch (mode) {
+ case __NDEV_MODE_PF:
+ nr_queues = MAX_PF_QUEUES;
+ break;
+ case __NDEV_MODE_VF16:
+ nr_queues = 8;
+ break;
+ case __NDEV_MODE_VF32:
+ nr_queues = 4;
+ break;
+ case __NDEV_MODE_VF64:
+ nr_queues = 2;
+ break;
+ case __NDEV_MODE_VF128:
+ nr_queues = 1;
+ break;
+ }
+
+ return nr_queues;
+}
+
+static void nitrox_pf_cleanup(struct nitrox_device *ndev)
+{
+ /* PF has no queues in SR-IOV mode */
+ atomic_set(&ndev->state, __NDEV_NOT_READY);
+ /* unregister crypto algorithms */
+ nitrox_crypto_unregister();
+
+ /* cleanup PF resources */
+ nitrox_unregister_interrupts(ndev);
+ nitrox_common_sw_cleanup(ndev);
+}
+
+/**
+ * nitrox_pf_reinit - re-initialize PF resources once SR-IOV is disabled
+ * @ndev: NITROX device
+ */
+static int nitrox_pf_reinit(struct nitrox_device *ndev)
+{
+ int err;
+
+ /* allocate resources for PF */
+ err = nitrox_common_sw_init(ndev);
+ if (err)
+ return err;
+
+ err = nitrox_register_interrupts(ndev);
+ if (err) {
+ nitrox_common_sw_cleanup(ndev);
+ return err;
+ }
+
+ /* configure the AQM queues */
+ nitrox_config_aqm_rings(ndev);
+
+ /* configure the packet queues */
+ nitrox_config_pkt_input_rings(ndev);
+ nitrox_config_pkt_solicit_ports(ndev);
+
+ /* set device to ready state */
+ atomic_set(&ndev->state, __NDEV_READY);
+
+ /* register crypto algorithms */
+ return nitrox_crypto_register();
+}
+
+static void nitrox_sriov_cleanup(struct nitrox_device *ndev)
+{
+ /* unregister interrupts for PF in SR-IOV */
+ nitrox_sriov_unregister_interrupts(ndev);
+ nitrox_mbox_cleanup(ndev);
+}
+
+static int nitrox_sriov_init(struct nitrox_device *ndev)
+{
+ int ret;
+
+ /* register interrupts for PF in SR-IOV */
+ ret = nitrox_sriov_register_interupts(ndev);
+ if (ret)
+ return ret;
+
+ ret = nitrox_mbox_init(ndev);
+ if (ret)
+ goto sriov_init_fail;
+
+ return 0;
+
+sriov_init_fail:
+ nitrox_sriov_cleanup(ndev);
+ return ret;
+}
+
+static int nitrox_sriov_enable(struct pci_dev *pdev, int num_vfs)
+{
+ struct nitrox_device *ndev = pci_get_drvdata(pdev);
+ int err;
+
+ if (!num_vfs_valid(num_vfs)) {
+ dev_err(DEV(ndev), "Invalid num_vfs %d\n", num_vfs);
+ return -EINVAL;
+ }
+
+ if (pci_num_vf(pdev) == num_vfs)
+ return num_vfs;
+
+ err = pci_enable_sriov(pdev, num_vfs);
+ if (err) {
+ dev_err(DEV(ndev), "failed to enable PCI sriov %d\n", err);
+ return err;
+ }
+ dev_info(DEV(ndev), "Enabled VF(s) %d\n", num_vfs);
+
+ ndev->mode = num_vfs_to_mode(num_vfs);
+ ndev->iov.num_vfs = num_vfs;
+ ndev->iov.max_vf_queues = vf_mode_to_nr_queues(ndev->mode);
+ /* set bit in flags */
+ set_bit(__NDEV_SRIOV_BIT, &ndev->flags);
+
+ /* cleanup PF resources */
+ nitrox_pf_cleanup(ndev);
+
+ /* PF SR-IOV mode initialization */
+ err = nitrox_sriov_init(ndev);
+ if (err)
+ goto iov_fail;
+
+ config_nps_core_vfcfg_mode(ndev, ndev->mode);
+ return num_vfs;
+
+iov_fail:
+ pci_disable_sriov(pdev);
+ /* clear bit in flags */
+ clear_bit(__NDEV_SRIOV_BIT, &ndev->flags);
+ ndev->iov.num_vfs = 0;
+ ndev->mode = __NDEV_MODE_PF;
+ /* reset back to working mode in PF */
+ nitrox_pf_reinit(ndev);
+ return err;
+}
+
+static int nitrox_sriov_disable(struct pci_dev *pdev)
+{
+ struct nitrox_device *ndev = pci_get_drvdata(pdev);
+
+ if (!test_bit(__NDEV_SRIOV_BIT, &ndev->flags))
+ return 0;
+
+ if (pci_vfs_assigned(pdev)) {
+ dev_warn(DEV(ndev), "VFs are attached to VM. Can't disable SR-IOV\n");
+ return -EPERM;
+ }
+ pci_disable_sriov(pdev);
+ /* clear bit in flags */
+ clear_bit(__NDEV_SRIOV_BIT, &ndev->flags);
+
+ ndev->iov.num_vfs = 0;
+ ndev->iov.max_vf_queues = 0;
+ ndev->mode = __NDEV_MODE_PF;
+
+ /* cleanup PF SR-IOV resources */
+ nitrox_sriov_cleanup(ndev);
+
+ config_nps_core_vfcfg_mode(ndev, ndev->mode);
+
+ return nitrox_pf_reinit(ndev);
+}
+
+int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs)
+{
+ if (!num_vfs)
+ return nitrox_sriov_disable(pdev);
+
+ return nitrox_sriov_enable(pdev, num_vfs);
+}