From ace9429bb58fd418f0c81d4c2835699bddf6bde6 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Thu, 11 Apr 2024 10:27:49 +0200 Subject: Adding upstream version 6.6.15. Signed-off-by: Daniel Baumann --- drivers/crypto/starfive/Kconfig | 22 + drivers/crypto/starfive/Makefile | 4 + drivers/crypto/starfive/jh7110-aes.c | 1024 +++++++++++++++++++++++++++++++++ drivers/crypto/starfive/jh7110-cryp.c | 279 +++++++++ drivers/crypto/starfive/jh7110-cryp.h | 236 ++++++++ drivers/crypto/starfive/jh7110-hash.c | 915 +++++++++++++++++++++++++++++ drivers/crypto/starfive/jh7110-rsa.c | 617 ++++++++++++++++++++ 7 files changed, 3097 insertions(+) create mode 100644 drivers/crypto/starfive/Kconfig create mode 100644 drivers/crypto/starfive/Makefile create mode 100644 drivers/crypto/starfive/jh7110-aes.c create mode 100644 drivers/crypto/starfive/jh7110-cryp.c create mode 100644 drivers/crypto/starfive/jh7110-cryp.h create mode 100644 drivers/crypto/starfive/jh7110-hash.c create mode 100644 drivers/crypto/starfive/jh7110-rsa.c (limited to 'drivers/crypto/starfive') diff --git a/drivers/crypto/starfive/Kconfig b/drivers/crypto/starfive/Kconfig new file mode 100644 index 0000000000..2cb192502c --- /dev/null +++ b/drivers/crypto/starfive/Kconfig @@ -0,0 +1,22 @@ +# +# StarFive crypto drivers configuration +# + +config CRYPTO_DEV_JH7110 + tristate "StarFive JH7110 cryptographic engine driver" + depends on SOC_STARFIVE || AMBA_PL08X || COMPILE_TEST + depends on HAS_DMA + select CRYPTO_ENGINE + select CRYPTO_HMAC + select CRYPTO_SHA256 + select CRYPTO_SHA512 + select CRYPTO_SM3_GENERIC + select CRYPTO_RSA + select CRYPTO_AES + select CRYPTO_CCM + help + Support for StarFive JH7110 crypto hardware acceleration engine. + This module provides acceleration for public key algo, + skciphers, AEAD and hash functions. + + If you choose 'M' here, this module will be called jh7110-crypto. diff --git a/drivers/crypto/starfive/Makefile b/drivers/crypto/starfive/Makefile new file mode 100644 index 0000000000..8c137afe58 --- /dev/null +++ b/drivers/crypto/starfive/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_CRYPTO_DEV_JH7110) += jh7110-crypto.o +jh7110-crypto-objs := jh7110-cryp.o jh7110-hash.o jh7110-rsa.o jh7110-aes.o diff --git a/drivers/crypto/starfive/jh7110-aes.c b/drivers/crypto/starfive/jh7110-aes.c new file mode 100644 index 0000000000..9378e6682f --- /dev/null +++ b/drivers/crypto/starfive/jh7110-aes.c @@ -0,0 +1,1024 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * StarFive AES acceleration driver + * + * Copyright (c) 2022 StarFive Technology + */ + +#include +#include +#include +#include +#include +#include "jh7110-cryp.h" +#include +#include +#include +#include +#include + +#define STARFIVE_AES_REGS_OFFSET 0x100 +#define STARFIVE_AES_AESDIO0R (STARFIVE_AES_REGS_OFFSET + 0x0) +#define STARFIVE_AES_KEY0 (STARFIVE_AES_REGS_OFFSET + 0x4) +#define STARFIVE_AES_KEY1 (STARFIVE_AES_REGS_OFFSET + 0x8) +#define STARFIVE_AES_KEY2 (STARFIVE_AES_REGS_OFFSET + 0xC) +#define STARFIVE_AES_KEY3 (STARFIVE_AES_REGS_OFFSET + 0x10) +#define STARFIVE_AES_KEY4 (STARFIVE_AES_REGS_OFFSET + 0x14) +#define STARFIVE_AES_KEY5 (STARFIVE_AES_REGS_OFFSET + 0x18) +#define STARFIVE_AES_KEY6 (STARFIVE_AES_REGS_OFFSET + 0x1C) +#define STARFIVE_AES_KEY7 (STARFIVE_AES_REGS_OFFSET + 0x20) +#define STARFIVE_AES_CSR (STARFIVE_AES_REGS_OFFSET + 0x24) +#define STARFIVE_AES_IV0 (STARFIVE_AES_REGS_OFFSET + 0x28) +#define STARFIVE_AES_IV1 (STARFIVE_AES_REGS_OFFSET + 0x2C) +#define STARFIVE_AES_IV2 (STARFIVE_AES_REGS_OFFSET + 0x30) +#define STARFIVE_AES_IV3 (STARFIVE_AES_REGS_OFFSET + 0x34) +#define STARFIVE_AES_NONCE0 (STARFIVE_AES_REGS_OFFSET + 0x3C) +#define STARFIVE_AES_NONCE1 (STARFIVE_AES_REGS_OFFSET + 0x40) +#define STARFIVE_AES_NONCE2 (STARFIVE_AES_REGS_OFFSET + 0x44) +#define STARFIVE_AES_NONCE3 (STARFIVE_AES_REGS_OFFSET + 0x48) +#define STARFIVE_AES_ALEN0 (STARFIVE_AES_REGS_OFFSET + 0x4C) +#define STARFIVE_AES_ALEN1 (STARFIVE_AES_REGS_OFFSET + 0x50) +#define STARFIVE_AES_MLEN0 (STARFIVE_AES_REGS_OFFSET + 0x54) +#define STARFIVE_AES_MLEN1 (STARFIVE_AES_REGS_OFFSET + 0x58) +#define STARFIVE_AES_IVLEN (STARFIVE_AES_REGS_OFFSET + 0x5C) + +#define FLG_MODE_MASK GENMASK(2, 0) +#define FLG_ENCRYPT BIT(4) + +/* Misc */ +#define CCM_B0_ADATA 0x40 +#define AES_BLOCK_32 (AES_BLOCK_SIZE / sizeof(u32)) + +static inline int starfive_aes_wait_busy(struct starfive_cryp_dev *cryp) +{ + u32 status; + + return readl_relaxed_poll_timeout(cryp->base + STARFIVE_AES_CSR, status, + !(status & STARFIVE_AES_BUSY), 10, 100000); +} + +static inline int starfive_aes_wait_keydone(struct starfive_cryp_dev *cryp) +{ + u32 status; + + return readl_relaxed_poll_timeout(cryp->base + STARFIVE_AES_CSR, status, + (status & STARFIVE_AES_KEY_DONE), 10, 100000); +} + +static inline int starfive_aes_wait_gcmdone(struct starfive_cryp_dev *cryp) +{ + u32 status; + + return readl_relaxed_poll_timeout(cryp->base + STARFIVE_AES_CSR, status, + (status & STARFIVE_AES_GCM_DONE), 10, 100000); +} + +static inline int is_gcm(struct starfive_cryp_dev *cryp) +{ + return (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_GCM; +} + +static inline int is_encrypt(struct starfive_cryp_dev *cryp) +{ + return cryp->flags & FLG_ENCRYPT; +} + +static void starfive_aes_aead_hw_start(struct starfive_cryp_ctx *ctx, u32 hw_mode) +{ + struct starfive_cryp_dev *cryp = ctx->cryp; + unsigned int value; + + switch (hw_mode) { + case STARFIVE_AES_MODE_GCM: + value = readl(ctx->cryp->base + STARFIVE_AES_CSR); + value |= STARFIVE_AES_GCM_START; + writel(value, cryp->base + STARFIVE_AES_CSR); + starfive_aes_wait_gcmdone(cryp); + break; + case STARFIVE_AES_MODE_CCM: + value = readl(ctx->cryp->base + STARFIVE_AES_CSR); + value |= STARFIVE_AES_CCM_START; + writel(value, cryp->base + STARFIVE_AES_CSR); + break; + } +} + +static inline void starfive_aes_set_ivlen(struct starfive_cryp_ctx *ctx) +{ + struct starfive_cryp_dev *cryp = ctx->cryp; + + if (is_gcm(cryp)) + writel(GCM_AES_IV_SIZE, cryp->base + STARFIVE_AES_IVLEN); + else + writel(AES_BLOCK_SIZE, cryp->base + STARFIVE_AES_IVLEN); +} + +static inline void starfive_aes_set_alen(struct starfive_cryp_ctx *ctx) +{ + struct starfive_cryp_dev *cryp = ctx->cryp; + + writel(upper_32_bits(cryp->assoclen), cryp->base + STARFIVE_AES_ALEN0); + writel(lower_32_bits(cryp->assoclen), cryp->base + STARFIVE_AES_ALEN1); +} + +static inline void starfive_aes_set_mlen(struct starfive_cryp_ctx *ctx) +{ + struct starfive_cryp_dev *cryp = ctx->cryp; + + writel(upper_32_bits(cryp->total_in), cryp->base + STARFIVE_AES_MLEN0); + writel(lower_32_bits(cryp->total_in), cryp->base + STARFIVE_AES_MLEN1); +} + +static inline int starfive_aes_ccm_check_iv(const u8 *iv) +{ + /* 2 <= L <= 8, so 1 <= L' <= 7. */ + if (iv[0] < 1 || iv[0] > 7) + return -EINVAL; + + return 0; +} + +static int starfive_aes_write_iv(struct starfive_cryp_ctx *ctx, u32 *iv) +{ + struct starfive_cryp_dev *cryp = ctx->cryp; + + writel(iv[0], cryp->base + STARFIVE_AES_IV0); + writel(iv[1], cryp->base + STARFIVE_AES_IV1); + writel(iv[2], cryp->base + STARFIVE_AES_IV2); + + if (is_gcm(cryp)) { + if (starfive_aes_wait_gcmdone(cryp)) + return -ETIMEDOUT; + + return 0; + } + + writel(iv[3], cryp->base + STARFIVE_AES_IV3); + + return 0; +} + +static inline void starfive_aes_get_iv(struct starfive_cryp_dev *cryp, u32 *iv) +{ + iv[0] = readl(cryp->base + STARFIVE_AES_IV0); + iv[1] = readl(cryp->base + STARFIVE_AES_IV1); + iv[2] = readl(cryp->base + STARFIVE_AES_IV2); + iv[3] = readl(cryp->base + STARFIVE_AES_IV3); +} + +static inline void starfive_aes_write_nonce(struct starfive_cryp_ctx *ctx, u32 *nonce) +{ + struct starfive_cryp_dev *cryp = ctx->cryp; + + writel(nonce[0], cryp->base + STARFIVE_AES_NONCE0); + writel(nonce[1], cryp->base + STARFIVE_AES_NONCE1); + writel(nonce[2], cryp->base + STARFIVE_AES_NONCE2); + writel(nonce[3], cryp->base + STARFIVE_AES_NONCE3); +} + +static int starfive_aes_write_key(struct starfive_cryp_ctx *ctx) +{ + struct starfive_cryp_dev *cryp = ctx->cryp; + u32 *key = (u32 *)ctx->key; + + if (ctx->keylen >= AES_KEYSIZE_128) { + writel(key[0], cryp->base + STARFIVE_AES_KEY0); + writel(key[1], cryp->base + STARFIVE_AES_KEY1); + writel(key[2], cryp->base + STARFIVE_AES_KEY2); + writel(key[3], cryp->base + STARFIVE_AES_KEY3); + } + + if (ctx->keylen >= AES_KEYSIZE_192) { + writel(key[4], cryp->base + STARFIVE_AES_KEY4); + writel(key[5], cryp->base + STARFIVE_AES_KEY5); + } + + if (ctx->keylen >= AES_KEYSIZE_256) { + writel(key[6], cryp->base + STARFIVE_AES_KEY6); + writel(key[7], cryp->base + STARFIVE_AES_KEY7); + } + + if (starfive_aes_wait_keydone(cryp)) + return -ETIMEDOUT; + + return 0; +} + +static int starfive_aes_ccm_init(struct starfive_cryp_ctx *ctx) +{ + struct starfive_cryp_dev *cryp = ctx->cryp; + u8 iv[AES_BLOCK_SIZE], b0[AES_BLOCK_SIZE]; + unsigned int textlen; + + memcpy(iv, cryp->req.areq->iv, AES_BLOCK_SIZE); + memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1); + + /* Build B0 */ + memcpy(b0, iv, AES_BLOCK_SIZE); + + b0[0] |= (8 * ((cryp->authsize - 2) / 2)); + + if (cryp->assoclen) + b0[0] |= CCM_B0_ADATA; + + textlen = cryp->total_in; + + b0[AES_BLOCK_SIZE - 2] = textlen >> 8; + b0[AES_BLOCK_SIZE - 1] = textlen & 0xFF; + + starfive_aes_write_nonce(ctx, (u32 *)b0); + + return 0; +} + +static int starfive_aes_hw_init(struct starfive_cryp_ctx *ctx) +{ + struct starfive_cryp_request_ctx *rctx = ctx->rctx; + struct starfive_cryp_dev *cryp = ctx->cryp; + u32 hw_mode; + + /* reset */ + rctx->csr.aes.v = 0; + rctx->csr.aes.aesrst = 1; + writel(rctx->csr.aes.v, cryp->base + STARFIVE_AES_CSR); + + /* csr setup */ + hw_mode = cryp->flags & FLG_MODE_MASK; + + rctx->csr.aes.v = 0; + + switch (ctx->keylen) { + case AES_KEYSIZE_128: + rctx->csr.aes.keymode = STARFIVE_AES_KEYMODE_128; + break; + case AES_KEYSIZE_192: + rctx->csr.aes.keymode = STARFIVE_AES_KEYMODE_192; + break; + case AES_KEYSIZE_256: + rctx->csr.aes.keymode = STARFIVE_AES_KEYMODE_256; + break; + } + + rctx->csr.aes.mode = hw_mode; + rctx->csr.aes.cmode = !is_encrypt(cryp); + rctx->csr.aes.ie = 1; + + if (hw_mode == STARFIVE_AES_MODE_CFB || + hw_mode == STARFIVE_AES_MODE_OFB) + rctx->csr.aes.stmode = STARFIVE_AES_MODE_XFB_128; + else + rctx->csr.aes.stmode = STARFIVE_AES_MODE_XFB_1; + + if (cryp->side_chan) { + rctx->csr.aes.delay_aes = 1; + rctx->csr.aes.vaes_start = 1; + } + + writel(rctx->csr.aes.v, cryp->base + STARFIVE_AES_CSR); + + cryp->err = starfive_aes_write_key(ctx); + if (cryp->err) + return cryp->err; + + switch (hw_mode) { + case STARFIVE_AES_MODE_GCM: + starfive_aes_set_alen(ctx); + starfive_aes_set_mlen(ctx); + starfive_aes_set_ivlen(ctx); + starfive_aes_aead_hw_start(ctx, hw_mode); + starfive_aes_write_iv(ctx, (void *)cryp->req.areq->iv); + break; + case STARFIVE_AES_MODE_CCM: + starfive_aes_set_alen(ctx); + starfive_aes_set_mlen(ctx); + starfive_aes_ccm_init(ctx); + starfive_aes_aead_hw_start(ctx, hw_mode); + break; + case STARFIVE_AES_MODE_OFB: + case STARFIVE_AES_MODE_CFB: + case STARFIVE_AES_MODE_CBC: + case STARFIVE_AES_MODE_CTR: + starfive_aes_write_iv(ctx, (void *)cryp->req.sreq->iv); + break; + default: + break; + } + + return cryp->err; +} + +static int starfive_aes_read_authtag(struct starfive_cryp_dev *cryp) +{ + int i, start_addr; + + if (starfive_aes_wait_busy(cryp)) + return dev_err_probe(cryp->dev, -ETIMEDOUT, + "Timeout waiting for tag generation."); + + start_addr = STARFIVE_AES_NONCE0; + + if (is_gcm(cryp)) + for (i = 0; i < AES_BLOCK_32; i++, start_addr += 4) + cryp->tag_out[i] = readl(cryp->base + start_addr); + else + for (i = 0; i < AES_BLOCK_32; i++) + cryp->tag_out[i] = readl(cryp->base + STARFIVE_AES_AESDIO0R); + + if (is_encrypt(cryp)) { + scatterwalk_copychunks(cryp->tag_out, &cryp->out_walk, cryp->authsize, 1); + } else { + scatterwalk_copychunks(cryp->tag_in, &cryp->in_walk, cryp->authsize, 0); + + if (crypto_memneq(cryp->tag_in, cryp->tag_out, cryp->authsize)) + return dev_err_probe(cryp->dev, -EBADMSG, "Failed tag verification\n"); + } + + return 0; +} + +static void starfive_aes_finish_req(struct starfive_cryp_dev *cryp) +{ + union starfive_aes_csr csr; + int err = cryp->err; + + if (!err && cryp->authsize) + err = starfive_aes_read_authtag(cryp); + + if (!err && ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CBC || + (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CTR)) + starfive_aes_get_iv(cryp, (void *)cryp->req.sreq->iv); + + /* reset irq flags*/ + csr.v = 0; + csr.aesrst = 1; + writel(csr.v, cryp->base + STARFIVE_AES_CSR); + + if (cryp->authsize) + crypto_finalize_aead_request(cryp->engine, cryp->req.areq, err); + else + crypto_finalize_skcipher_request(cryp->engine, cryp->req.sreq, + err); +} + +void starfive_aes_done_task(unsigned long param) +{ + struct starfive_cryp_dev *cryp = (struct starfive_cryp_dev *)param; + u32 block[AES_BLOCK_32]; + u32 stat; + int i; + + for (i = 0; i < AES_BLOCK_32; i++) + block[i] = readl(cryp->base + STARFIVE_AES_AESDIO0R); + + scatterwalk_copychunks(block, &cryp->out_walk, min_t(size_t, AES_BLOCK_SIZE, + cryp->total_out), 1); + + cryp->total_out -= min_t(size_t, AES_BLOCK_SIZE, cryp->total_out); + + if (!cryp->total_out) { + starfive_aes_finish_req(cryp); + return; + } + + memset(block, 0, AES_BLOCK_SIZE); + scatterwalk_copychunks(block, &cryp->in_walk, min_t(size_t, AES_BLOCK_SIZE, + cryp->total_in), 0); + cryp->total_in -= min_t(size_t, AES_BLOCK_SIZE, cryp->total_in); + + for (i = 0; i < AES_BLOCK_32; i++) + writel(block[i], cryp->base + STARFIVE_AES_AESDIO0R); + + stat = readl(cryp->base + STARFIVE_IE_MASK_OFFSET); + stat &= ~STARFIVE_IE_MASK_AES_DONE; + writel(stat, cryp->base + STARFIVE_IE_MASK_OFFSET); +} + +static int starfive_aes_gcm_write_adata(struct starfive_cryp_ctx *ctx) +{ + struct starfive_cryp_dev *cryp = ctx->cryp; + struct starfive_cryp_request_ctx *rctx = ctx->rctx; + u32 *buffer; + int total_len, loop; + + total_len = ALIGN(cryp->assoclen, AES_BLOCK_SIZE) / sizeof(unsigned int); + buffer = (u32 *)rctx->adata; + + for (loop = 0; loop < total_len; loop += 4) { + writel(*buffer, cryp->base + STARFIVE_AES_NONCE0); + buffer++; + writel(*buffer, cryp->base + STARFIVE_AES_NONCE1); + buffer++; + writel(*buffer, cryp->base + STARFIVE_AES_NONCE2); + buffer++; + writel(*buffer, cryp->base + STARFIVE_AES_NONCE3); + buffer++; + } + + if (starfive_aes_wait_gcmdone(cryp)) + return dev_err_probe(cryp->dev, -ETIMEDOUT, + "Timeout processing gcm aad block"); + + return 0; +} + +static int starfive_aes_ccm_write_adata(struct starfive_cryp_ctx *ctx) +{ + struct starfive_cryp_dev *cryp = ctx->cryp; + struct starfive_cryp_request_ctx *rctx = ctx->rctx; + u32 *buffer; + u8 *ci; + int total_len, loop; + + total_len = cryp->assoclen; + + ci = rctx->adata; + writeb(*ci, cryp->base + STARFIVE_AES_AESDIO0R); + ci++; + writeb(*ci, cryp->base + STARFIVE_AES_AESDIO0R); + ci++; + total_len -= 2; + buffer = (u32 *)ci; + + for (loop = 0; loop < 3; loop++, buffer++) + writel(*buffer, cryp->base + STARFIVE_AES_AESDIO0R); + + total_len -= 12; + + while (total_len > 0) { + for (loop = 0; loop < AES_BLOCK_32; loop++, buffer++) + writel(*buffer, cryp->base + STARFIVE_AES_AESDIO0R); + + total_len -= AES_BLOCK_SIZE; + } + + if (starfive_aes_wait_busy(cryp)) + return dev_err_probe(cryp->dev, -ETIMEDOUT, + "Timeout processing ccm aad block"); + + return 0; +} + +static int starfive_aes_prepare_req(struct skcipher_request *req, + struct aead_request *areq) +{ + struct starfive_cryp_ctx *ctx; + struct starfive_cryp_request_ctx *rctx; + struct starfive_cryp_dev *cryp; + + if (!req && !areq) + return -EINVAL; + + ctx = req ? crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)) : + crypto_aead_ctx(crypto_aead_reqtfm(areq)); + + cryp = ctx->cryp; + rctx = req ? skcipher_request_ctx(req) : aead_request_ctx(areq); + + if (req) { + cryp->req.sreq = req; + cryp->total_in = req->cryptlen; + cryp->total_out = req->cryptlen; + cryp->assoclen = 0; + cryp->authsize = 0; + } else { + cryp->req.areq = areq; + cryp->assoclen = areq->assoclen; + cryp->authsize = crypto_aead_authsize(crypto_aead_reqtfm(areq)); + if (is_encrypt(cryp)) { + cryp->total_in = areq->cryptlen; + cryp->total_out = areq->cryptlen; + } else { + cryp->total_in = areq->cryptlen - cryp->authsize; + cryp->total_out = cryp->total_in; + } + } + + rctx->in_sg = req ? req->src : areq->src; + scatterwalk_start(&cryp->in_walk, rctx->in_sg); + + rctx->out_sg = req ? req->dst : areq->dst; + scatterwalk_start(&cryp->out_walk, rctx->out_sg); + + if (cryp->assoclen) { + rctx->adata = kzalloc(ALIGN(cryp->assoclen, AES_BLOCK_SIZE), GFP_KERNEL); + if (!rctx->adata) + return dev_err_probe(cryp->dev, -ENOMEM, + "Failed to alloc memory for adata"); + + scatterwalk_copychunks(rctx->adata, &cryp->in_walk, cryp->assoclen, 0); + scatterwalk_copychunks(NULL, &cryp->out_walk, cryp->assoclen, 2); + } + + ctx->rctx = rctx; + + return starfive_aes_hw_init(ctx); +} + +static int starfive_aes_do_one_req(struct crypto_engine *engine, void *areq) +{ + struct skcipher_request *req = + container_of(areq, struct skcipher_request, base); + struct starfive_cryp_ctx *ctx = + crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + struct starfive_cryp_dev *cryp = ctx->cryp; + u32 block[AES_BLOCK_32]; + u32 stat; + int err; + int i; + + err = starfive_aes_prepare_req(req, NULL); + if (err) + return err; + + /* + * Write first plain/ciphertext block to start the module + * then let irq tasklet handle the rest of the data blocks. + */ + scatterwalk_copychunks(block, &cryp->in_walk, min_t(size_t, AES_BLOCK_SIZE, + cryp->total_in), 0); + cryp->total_in -= min_t(size_t, AES_BLOCK_SIZE, cryp->total_in); + + for (i = 0; i < AES_BLOCK_32; i++) + writel(block[i], cryp->base + STARFIVE_AES_AESDIO0R); + + stat = readl(cryp->base + STARFIVE_IE_MASK_OFFSET); + stat &= ~STARFIVE_IE_MASK_AES_DONE; + writel(stat, cryp->base + STARFIVE_IE_MASK_OFFSET); + + return 0; +} + +static int starfive_aes_init_tfm(struct crypto_skcipher *tfm) +{ + struct starfive_cryp_ctx *ctx = crypto_skcipher_ctx(tfm); + + ctx->cryp = starfive_cryp_find_dev(ctx); + if (!ctx->cryp) + return -ENODEV; + + crypto_skcipher_set_reqsize(tfm, sizeof(struct starfive_cryp_request_ctx) + + sizeof(struct skcipher_request)); + + return 0; +} + +static int starfive_aes_aead_do_one_req(struct crypto_engine *engine, void *areq) +{ + struct aead_request *req = + container_of(areq, struct aead_request, base); + struct starfive_cryp_ctx *ctx = + crypto_aead_ctx(crypto_aead_reqtfm(req)); + struct starfive_cryp_dev *cryp = ctx->cryp; + struct starfive_cryp_request_ctx *rctx = ctx->rctx; + u32 block[AES_BLOCK_32]; + u32 stat; + int err; + int i; + + err = starfive_aes_prepare_req(NULL, req); + if (err) + return err; + + if (!cryp->assoclen) + goto write_text; + + if ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CCM) + cryp->err = starfive_aes_ccm_write_adata(ctx); + else + cryp->err = starfive_aes_gcm_write_adata(ctx); + + kfree(rctx->adata); + + if (cryp->err) + return cryp->err; + +write_text: + if (!cryp->total_in) + goto finish_req; + + /* + * Write first plain/ciphertext block to start the module + * then let irq tasklet handle the rest of the data blocks. + */ + scatterwalk_copychunks(block, &cryp->in_walk, min_t(size_t, AES_BLOCK_SIZE, + cryp->total_in), 0); + cryp->total_in -= min_t(size_t, AES_BLOCK_SIZE, cryp->total_in); + + for (i = 0; i < AES_BLOCK_32; i++) + writel(block[i], cryp->base + STARFIVE_AES_AESDIO0R); + + stat = readl(cryp->base + STARFIVE_IE_MASK_OFFSET); + stat &= ~STARFIVE_IE_MASK_AES_DONE; + writel(stat, cryp->base + STARFIVE_IE_MASK_OFFSET); + + return 0; + +finish_req: + starfive_aes_finish_req(cryp); + return 0; +} + +static int starfive_aes_aead_init_tfm(struct crypto_aead *tfm) +{ + struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm); + struct starfive_cryp_dev *cryp = ctx->cryp; + struct crypto_tfm *aead = crypto_aead_tfm(tfm); + struct crypto_alg *alg = aead->__crt_alg; + + ctx->cryp = starfive_cryp_find_dev(ctx); + if (!ctx->cryp) + return -ENODEV; + + if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) { + ctx->aead_fbk = crypto_alloc_aead(alg->cra_name, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->aead_fbk)) + return dev_err_probe(cryp->dev, PTR_ERR(ctx->aead_fbk), + "%s() failed to allocate fallback for %s\n", + __func__, alg->cra_name); + } + + crypto_aead_set_reqsize(tfm, sizeof(struct starfive_cryp_ctx) + + sizeof(struct aead_request)); + + return 0; +} + +static void starfive_aes_aead_exit_tfm(struct crypto_aead *tfm) +{ + struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm); + + crypto_free_aead(ctx->aead_fbk); +} + +static int starfive_aes_crypt(struct skcipher_request *req, unsigned long flags) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct starfive_cryp_ctx *ctx = crypto_skcipher_ctx(tfm); + struct starfive_cryp_dev *cryp = ctx->cryp; + unsigned int blocksize_align = crypto_skcipher_blocksize(tfm) - 1; + + cryp->flags = flags; + + if ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_ECB || + (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CBC) + if (req->cryptlen & blocksize_align) + return -EINVAL; + + return crypto_transfer_skcipher_request_to_engine(cryp->engine, req); +} + +static int starfive_aes_aead_crypt(struct aead_request *req, unsigned long flags) +{ + struct starfive_cryp_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + struct starfive_cryp_dev *cryp = ctx->cryp; + + cryp->flags = flags; + + /* + * HW engine could not perform CCM tag verification on + * non-blocksize aligned text, use fallback algo instead + */ + if (ctx->aead_fbk && !is_encrypt(cryp)) { + struct aead_request *subreq = aead_request_ctx(req); + + aead_request_set_tfm(subreq, ctx->aead_fbk); + aead_request_set_callback(subreq, req->base.flags, + req->base.complete, req->base.data); + aead_request_set_crypt(subreq, req->src, + req->dst, req->cryptlen, req->iv); + aead_request_set_ad(subreq, req->assoclen); + + return crypto_aead_decrypt(subreq); + } + + return crypto_transfer_aead_request_to_engine(cryp->engine, req); +} + +static int starfive_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct starfive_cryp_ctx *ctx = crypto_skcipher_ctx(tfm); + + if (!key || !keylen) + return -EINVAL; + + if (keylen != AES_KEYSIZE_128 && + keylen != AES_KEYSIZE_192 && + keylen != AES_KEYSIZE_256) + return -EINVAL; + + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + + return 0; +} + +static int starfive_aes_aead_setkey(struct crypto_aead *tfm, const u8 *key, + unsigned int keylen) +{ + struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm); + + if (!key || !keylen) + return -EINVAL; + + if (keylen != AES_KEYSIZE_128 && + keylen != AES_KEYSIZE_192 && + keylen != AES_KEYSIZE_256) + return -EINVAL; + + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + + if (ctx->aead_fbk) + return crypto_aead_setkey(ctx->aead_fbk, key, keylen); + + return 0; +} + +static int starfive_aes_gcm_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + return crypto_gcm_check_authsize(authsize); +} + +static int starfive_aes_ccm_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm); + + switch (authsize) { + case 4: + case 6: + case 8: + case 10: + case 12: + case 14: + case 16: + break; + default: + return -EINVAL; + } + + return crypto_aead_setauthsize(ctx->aead_fbk, authsize); +} + +static int starfive_aes_ecb_encrypt(struct skcipher_request *req) +{ + return starfive_aes_crypt(req, STARFIVE_AES_MODE_ECB | FLG_ENCRYPT); +} + +static int starfive_aes_ecb_decrypt(struct skcipher_request *req) +{ + return starfive_aes_crypt(req, STARFIVE_AES_MODE_ECB); +} + +static int starfive_aes_cbc_encrypt(struct skcipher_request *req) +{ + return starfive_aes_crypt(req, STARFIVE_AES_MODE_CBC | FLG_ENCRYPT); +} + +static int starfive_aes_cbc_decrypt(struct skcipher_request *req) +{ + return starfive_aes_crypt(req, STARFIVE_AES_MODE_CBC); +} + +static int starfive_aes_cfb_encrypt(struct skcipher_request *req) +{ + return starfive_aes_crypt(req, STARFIVE_AES_MODE_CFB | FLG_ENCRYPT); +} + +static int starfive_aes_cfb_decrypt(struct skcipher_request *req) +{ + return starfive_aes_crypt(req, STARFIVE_AES_MODE_CFB); +} + +static int starfive_aes_ofb_encrypt(struct skcipher_request *req) +{ + return starfive_aes_crypt(req, STARFIVE_AES_MODE_OFB | FLG_ENCRYPT); +} + +static int starfive_aes_ofb_decrypt(struct skcipher_request *req) +{ + return starfive_aes_crypt(req, STARFIVE_AES_MODE_OFB); +} + +static int starfive_aes_ctr_encrypt(struct skcipher_request *req) +{ + return starfive_aes_crypt(req, STARFIVE_AES_MODE_CTR | FLG_ENCRYPT); +} + +static int starfive_aes_ctr_decrypt(struct skcipher_request *req) +{ + return starfive_aes_crypt(req, STARFIVE_AES_MODE_CTR); +} + +static int starfive_aes_gcm_encrypt(struct aead_request *req) +{ + return starfive_aes_aead_crypt(req, STARFIVE_AES_MODE_GCM | FLG_ENCRYPT); +} + +static int starfive_aes_gcm_decrypt(struct aead_request *req) +{ + return starfive_aes_aead_crypt(req, STARFIVE_AES_MODE_GCM); +} + +static int starfive_aes_ccm_encrypt(struct aead_request *req) +{ + int ret; + + ret = starfive_aes_ccm_check_iv(req->iv); + if (ret) + return ret; + + return starfive_aes_aead_crypt(req, STARFIVE_AES_MODE_CCM | FLG_ENCRYPT); +} + +static int starfive_aes_ccm_decrypt(struct aead_request *req) +{ + int ret; + + ret = starfive_aes_ccm_check_iv(req->iv); + if (ret) + return ret; + + return starfive_aes_aead_crypt(req, STARFIVE_AES_MODE_CCM); +} + +static struct skcipher_engine_alg skcipher_algs[] = { +{ + .base.init = starfive_aes_init_tfm, + .base.setkey = starfive_aes_setkey, + .base.encrypt = starfive_aes_ecb_encrypt, + .base.decrypt = starfive_aes_ecb_decrypt, + .base.min_keysize = AES_MIN_KEY_SIZE, + .base.max_keysize = AES_MAX_KEY_SIZE, + .base.base = { + .cra_name = "ecb(aes)", + .cra_driver_name = "starfive-ecb-aes", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), + .cra_alignmask = 0xf, + .cra_module = THIS_MODULE, + }, + .op = { + .do_one_request = starfive_aes_do_one_req, + }, +}, { + .base.init = starfive_aes_init_tfm, + .base.setkey = starfive_aes_setkey, + .base.encrypt = starfive_aes_cbc_encrypt, + .base.decrypt = starfive_aes_cbc_decrypt, + .base.min_keysize = AES_MIN_KEY_SIZE, + .base.max_keysize = AES_MAX_KEY_SIZE, + .base.ivsize = AES_BLOCK_SIZE, + .base.base = { + .cra_name = "cbc(aes)", + .cra_driver_name = "starfive-cbc-aes", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), + .cra_alignmask = 0xf, + .cra_module = THIS_MODULE, + }, + .op = { + .do_one_request = starfive_aes_do_one_req, + }, +}, { + .base.init = starfive_aes_init_tfm, + .base.setkey = starfive_aes_setkey, + .base.encrypt = starfive_aes_ctr_encrypt, + .base.decrypt = starfive_aes_ctr_decrypt, + .base.min_keysize = AES_MIN_KEY_SIZE, + .base.max_keysize = AES_MAX_KEY_SIZE, + .base.ivsize = AES_BLOCK_SIZE, + .base.base = { + .cra_name = "ctr(aes)", + .cra_driver_name = "starfive-ctr-aes", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), + .cra_alignmask = 0xf, + .cra_module = THIS_MODULE, + }, + .op = { + .do_one_request = starfive_aes_do_one_req, + }, +}, { + .base.init = starfive_aes_init_tfm, + .base.setkey = starfive_aes_setkey, + .base.encrypt = starfive_aes_cfb_encrypt, + .base.decrypt = starfive_aes_cfb_decrypt, + .base.min_keysize = AES_MIN_KEY_SIZE, + .base.max_keysize = AES_MAX_KEY_SIZE, + .base.ivsize = AES_BLOCK_SIZE, + .base.base = { + .cra_name = "cfb(aes)", + .cra_driver_name = "starfive-cfb-aes", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), + .cra_alignmask = 0xf, + .cra_module = THIS_MODULE, + }, + .op = { + .do_one_request = starfive_aes_do_one_req, + }, +}, { + .base.init = starfive_aes_init_tfm, + .base.setkey = starfive_aes_setkey, + .base.encrypt = starfive_aes_ofb_encrypt, + .base.decrypt = starfive_aes_ofb_decrypt, + .base.min_keysize = AES_MIN_KEY_SIZE, + .base.max_keysize = AES_MAX_KEY_SIZE, + .base.ivsize = AES_BLOCK_SIZE, + .base.base = { + .cra_name = "ofb(aes)", + .cra_driver_name = "starfive-ofb-aes", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), + .cra_alignmask = 0xf, + .cra_module = THIS_MODULE, + }, + .op = { + .do_one_request = starfive_aes_do_one_req, + }, +}, +}; + +static struct aead_engine_alg aead_algs[] = { +{ + .base.setkey = starfive_aes_aead_setkey, + .base.setauthsize = starfive_aes_gcm_setauthsize, + .base.encrypt = starfive_aes_gcm_encrypt, + .base.decrypt = starfive_aes_gcm_decrypt, + .base.init = starfive_aes_aead_init_tfm, + .base.exit = starfive_aes_aead_exit_tfm, + .base.ivsize = GCM_AES_IV_SIZE, + .base.maxauthsize = AES_BLOCK_SIZE, + .base.base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "starfive-gcm-aes", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), + .cra_alignmask = 0xf, + .cra_module = THIS_MODULE, + }, + .op = { + .do_one_request = starfive_aes_aead_do_one_req, + }, +}, { + .base.setkey = starfive_aes_aead_setkey, + .base.setauthsize = starfive_aes_ccm_setauthsize, + .base.encrypt = starfive_aes_ccm_encrypt, + .base.decrypt = starfive_aes_ccm_decrypt, + .base.init = starfive_aes_aead_init_tfm, + .base.exit = starfive_aes_aead_exit_tfm, + .base.ivsize = AES_BLOCK_SIZE, + .base.maxauthsize = AES_BLOCK_SIZE, + .base.base = { + .cra_name = "ccm(aes)", + .cra_driver_name = "starfive-ccm-aes", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), + .cra_alignmask = 0xf, + .cra_module = THIS_MODULE, + }, + .op = { + .do_one_request = starfive_aes_aead_do_one_req, + }, +}, +}; + +int starfive_aes_register_algs(void) +{ + int ret; + + ret = crypto_engine_register_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs)); + if (ret) + return ret; + + ret = crypto_engine_register_aeads(aead_algs, ARRAY_SIZE(aead_algs)); + if (ret) + crypto_engine_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs)); + + return ret; +} + +void starfive_aes_unregister_algs(void) +{ + crypto_engine_unregister_aeads(aead_algs, ARRAY_SIZE(aead_algs)); + crypto_engine_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs)); +} diff --git a/drivers/crypto/starfive/jh7110-cryp.c b/drivers/crypto/starfive/jh7110-cryp.c new file mode 100644 index 0000000000..3a67ddc4d9 --- /dev/null +++ b/drivers/crypto/starfive/jh7110-cryp.c @@ -0,0 +1,279 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Cryptographic API. + * + * Support for StarFive hardware cryptographic engine. + * Copyright (c) 2022 StarFive Technology + * + */ + +#include +#include "jh7110-cryp.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DRIVER_NAME "jh7110-crypto" + +struct starfive_dev_list { + struct list_head dev_list; + spinlock_t lock; /* protect dev_list */ +}; + +static struct starfive_dev_list dev_list = { + .dev_list = LIST_HEAD_INIT(dev_list.dev_list), + .lock = __SPIN_LOCK_UNLOCKED(dev_list.lock), +}; + +struct starfive_cryp_dev *starfive_cryp_find_dev(struct starfive_cryp_ctx *ctx) +{ + struct starfive_cryp_dev *cryp = NULL, *tmp; + + spin_lock_bh(&dev_list.lock); + if (!ctx->cryp) { + list_for_each_entry(tmp, &dev_list.dev_list, list) { + cryp = tmp; + break; + } + ctx->cryp = cryp; + } else { + cryp = ctx->cryp; + } + + spin_unlock_bh(&dev_list.lock); + + return cryp; +} + +static u16 side_chan; +module_param(side_chan, ushort, 0); +MODULE_PARM_DESC(side_chan, "Enable side channel mitigation for AES module.\n" + "Enabling this feature will reduce speed performance.\n" + " 0 - Disabled\n" + " other - Enabled"); + +static int starfive_dma_init(struct starfive_cryp_dev *cryp) +{ + dma_cap_mask_t mask; + + dma_cap_zero(mask); + dma_cap_set(DMA_SLAVE, mask); + + cryp->tx = dma_request_chan(cryp->dev, "tx"); + if (IS_ERR(cryp->tx)) + return dev_err_probe(cryp->dev, PTR_ERR(cryp->tx), + "Error requesting tx dma channel.\n"); + + cryp->rx = dma_request_chan(cryp->dev, "rx"); + if (IS_ERR(cryp->rx)) { + dma_release_channel(cryp->tx); + return dev_err_probe(cryp->dev, PTR_ERR(cryp->rx), + "Error requesting rx dma channel.\n"); + } + + return 0; +} + +static void starfive_dma_cleanup(struct starfive_cryp_dev *cryp) +{ + dma_release_channel(cryp->tx); + dma_release_channel(cryp->rx); +} + +static irqreturn_t starfive_cryp_irq(int irq, void *priv) +{ + u32 status; + u32 mask; + struct starfive_cryp_dev *cryp = (struct starfive_cryp_dev *)priv; + + mask = readl(cryp->base + STARFIVE_IE_MASK_OFFSET); + status = readl(cryp->base + STARFIVE_IE_FLAG_OFFSET); + if (status & STARFIVE_IE_FLAG_AES_DONE) { + mask |= STARFIVE_IE_MASK_AES_DONE; + writel(mask, cryp->base + STARFIVE_IE_MASK_OFFSET); + tasklet_schedule(&cryp->aes_done); + } + + if (status & STARFIVE_IE_FLAG_HASH_DONE) { + mask |= STARFIVE_IE_MASK_HASH_DONE; + writel(mask, cryp->base + STARFIVE_IE_MASK_OFFSET); + tasklet_schedule(&cryp->hash_done); + } + + if (status & STARFIVE_IE_FLAG_PKA_DONE) { + mask |= STARFIVE_IE_MASK_PKA_DONE; + writel(mask, cryp->base + STARFIVE_IE_MASK_OFFSET); + complete(&cryp->pka_done); + } + + return IRQ_HANDLED; +} + +static int starfive_cryp_probe(struct platform_device *pdev) +{ + struct starfive_cryp_dev *cryp; + struct resource *res; + int irq; + int ret; + + cryp = devm_kzalloc(&pdev->dev, sizeof(*cryp), GFP_KERNEL); + if (!cryp) + return -ENOMEM; + + platform_set_drvdata(pdev, cryp); + cryp->dev = &pdev->dev; + + cryp->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); + if (IS_ERR(cryp->base)) + return dev_err_probe(&pdev->dev, PTR_ERR(cryp->base), + "Error remapping memory for platform device\n"); + + tasklet_init(&cryp->aes_done, starfive_aes_done_task, (unsigned long)cryp); + tasklet_init(&cryp->hash_done, starfive_hash_done_task, (unsigned long)cryp); + + cryp->phys_base = res->start; + cryp->dma_maxburst = 32; + cryp->side_chan = side_chan; + + cryp->hclk = devm_clk_get(&pdev->dev, "hclk"); + if (IS_ERR(cryp->hclk)) + return dev_err_probe(&pdev->dev, PTR_ERR(cryp->hclk), + "Error getting hardware reference clock\n"); + + cryp->ahb = devm_clk_get(&pdev->dev, "ahb"); + if (IS_ERR(cryp->ahb)) + return dev_err_probe(&pdev->dev, PTR_ERR(cryp->ahb), + "Error getting ahb reference clock\n"); + + cryp->rst = devm_reset_control_get_shared(cryp->dev, NULL); + if (IS_ERR(cryp->rst)) + return dev_err_probe(&pdev->dev, PTR_ERR(cryp->rst), + "Error getting hardware reset line\n"); + + init_completion(&cryp->pka_done); + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + + ret = devm_request_irq(&pdev->dev, irq, starfive_cryp_irq, 0, pdev->name, + (void *)cryp); + if (ret) + return dev_err_probe(&pdev->dev, irq, + "Failed to register interrupt handler\n"); + + clk_prepare_enable(cryp->hclk); + clk_prepare_enable(cryp->ahb); + reset_control_deassert(cryp->rst); + + spin_lock(&dev_list.lock); + list_add(&cryp->list, &dev_list.dev_list); + spin_unlock(&dev_list.lock); + + ret = starfive_dma_init(cryp); + if (ret) + goto err_dma_init; + + /* Initialize crypto engine */ + cryp->engine = crypto_engine_alloc_init(&pdev->dev, 1); + if (!cryp->engine) { + ret = -ENOMEM; + goto err_engine; + } + + ret = crypto_engine_start(cryp->engine); + if (ret) + goto err_engine_start; + + ret = starfive_aes_register_algs(); + if (ret) + goto err_algs_aes; + + ret = starfive_hash_register_algs(); + if (ret) + goto err_algs_hash; + + ret = starfive_rsa_register_algs(); + if (ret) + goto err_algs_rsa; + + return 0; + +err_algs_rsa: + starfive_hash_unregister_algs(); +err_algs_hash: + starfive_aes_unregister_algs(); +err_algs_aes: + crypto_engine_stop(cryp->engine); +err_engine_start: + crypto_engine_exit(cryp->engine); +err_engine: + starfive_dma_cleanup(cryp); +err_dma_init: + spin_lock(&dev_list.lock); + list_del(&cryp->list); + spin_unlock(&dev_list.lock); + + clk_disable_unprepare(cryp->hclk); + clk_disable_unprepare(cryp->ahb); + reset_control_assert(cryp->rst); + + tasklet_kill(&cryp->aes_done); + tasklet_kill(&cryp->hash_done); + + return ret; +} + +static void starfive_cryp_remove(struct platform_device *pdev) +{ + struct starfive_cryp_dev *cryp = platform_get_drvdata(pdev); + + starfive_aes_unregister_algs(); + starfive_hash_unregister_algs(); + starfive_rsa_unregister_algs(); + + tasklet_kill(&cryp->aes_done); + tasklet_kill(&cryp->hash_done); + + crypto_engine_stop(cryp->engine); + crypto_engine_exit(cryp->engine); + + starfive_dma_cleanup(cryp); + + spin_lock(&dev_list.lock); + list_del(&cryp->list); + spin_unlock(&dev_list.lock); + + clk_disable_unprepare(cryp->hclk); + clk_disable_unprepare(cryp->ahb); + reset_control_assert(cryp->rst); +} + +static const struct of_device_id starfive_dt_ids[] __maybe_unused = { + { .compatible = "starfive,jh7110-crypto", .data = NULL}, + {}, +}; +MODULE_DEVICE_TABLE(of, starfive_dt_ids); + +static struct platform_driver starfive_cryp_driver = { + .probe = starfive_cryp_probe, + .remove_new = starfive_cryp_remove, + .driver = { + .name = DRIVER_NAME, + .of_match_table = starfive_dt_ids, + }, +}; + +module_platform_driver(starfive_cryp_driver); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("StarFive JH7110 Cryptographic Module"); diff --git a/drivers/crypto/starfive/jh7110-cryp.h b/drivers/crypto/starfive/jh7110-cryp.h new file mode 100644 index 0000000000..fe011d5047 --- /dev/null +++ b/drivers/crypto/starfive/jh7110-cryp.h @@ -0,0 +1,236 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __STARFIVE_STR_H__ +#define __STARFIVE_STR_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define STARFIVE_ALG_CR_OFFSET 0x0 +#define STARFIVE_ALG_FIFO_OFFSET 0x4 +#define STARFIVE_IE_MASK_OFFSET 0x8 +#define STARFIVE_IE_FLAG_OFFSET 0xc +#define STARFIVE_DMA_IN_LEN_OFFSET 0x10 +#define STARFIVE_DMA_OUT_LEN_OFFSET 0x14 + +#define STARFIVE_IE_MASK_AES_DONE 0x1 +#define STARFIVE_IE_MASK_HASH_DONE 0x4 +#define STARFIVE_IE_MASK_PKA_DONE 0x8 +#define STARFIVE_IE_FLAG_AES_DONE 0x1 +#define STARFIVE_IE_FLAG_HASH_DONE 0x4 +#define STARFIVE_IE_FLAG_PKA_DONE 0x8 + +#define STARFIVE_MSG_BUFFER_SIZE SZ_16K +#define MAX_KEY_SIZE SHA512_BLOCK_SIZE +#define STARFIVE_AES_IV_LEN AES_BLOCK_SIZE +#define STARFIVE_AES_CTR_LEN AES_BLOCK_SIZE + +union starfive_aes_csr { + u32 v; + struct { + u32 cmode :1; +#define STARFIVE_AES_KEYMODE_128 0x0 +#define STARFIVE_AES_KEYMODE_192 0x1 +#define STARFIVE_AES_KEYMODE_256 0x2 + u32 keymode :2; +#define STARFIVE_AES_BUSY BIT(3) + u32 busy :1; + u32 done :1; +#define STARFIVE_AES_KEY_DONE BIT(5) + u32 krdy :1; + u32 aesrst :1; + u32 ie :1; +#define STARFIVE_AES_CCM_START BIT(8) + u32 ccm_start :1; +#define STARFIVE_AES_MODE_ECB 0x0 +#define STARFIVE_AES_MODE_CBC 0x1 +#define STARFIVE_AES_MODE_CFB 0x2 +#define STARFIVE_AES_MODE_OFB 0x3 +#define STARFIVE_AES_MODE_CTR 0x4 +#define STARFIVE_AES_MODE_CCM 0x5 +#define STARFIVE_AES_MODE_GCM 0x6 + u32 mode :3; +#define STARFIVE_AES_GCM_START BIT(12) + u32 gcm_start :1; +#define STARFIVE_AES_GCM_DONE BIT(13) + u32 gcm_done :1; + u32 delay_aes :1; + u32 vaes_start :1; + u32 rsvd_0 :8; +#define STARFIVE_AES_MODE_XFB_1 0x0 +#define STARFIVE_AES_MODE_XFB_128 0x5 + u32 stmode :3; + u32 rsvd_1 :5; + }; +}; + +union starfive_hash_csr { + u32 v; + struct { + u32 start :1; + u32 reset :1; + u32 ie :1; + u32 firstb :1; +#define STARFIVE_HASH_SM3 0x0 +#define STARFIVE_HASH_SHA224 0x3 +#define STARFIVE_HASH_SHA256 0x4 +#define STARFIVE_HASH_SHA384 0x5 +#define STARFIVE_HASH_SHA512 0x6 +#define STARFIVE_HASH_MODE_MASK 0x7 + u32 mode :3; + u32 rsvd_1 :1; + u32 final :1; + u32 rsvd_2 :2; +#define STARFIVE_HASH_HMAC_FLAGS 0x800 + u32 hmac :1; + u32 rsvd_3 :1; +#define STARFIVE_HASH_KEY_DONE BIT(13) + u32 key_done :1; + u32 key_flag :1; + u32 hmac_done :1; +#define STARFIVE_HASH_BUSY BIT(16) + u32 busy :1; + u32 hashdone :1; + u32 rsvd_4 :14; + }; +}; + +union starfive_pka_cacr { + u32 v; + struct { + u32 start :1; + u32 reset :1; + u32 ie :1; + u32 rsvd_0 :1; + u32 fifo_mode :1; + u32 not_r2 :1; + u32 ecc_sub :1; + u32 pre_expf :1; + u32 cmd :4; + u32 rsvd_1 :1; + u32 ctrl_dummy :1; + u32 ctrl_false :1; + u32 cln_done :1; + u32 opsize :6; + u32 rsvd_2 :2; + u32 exposize :6; + u32 rsvd_3 :1; + u32 bigendian :1; + }; +}; + +struct starfive_rsa_key { + u8 *n; + u8 *e; + u8 *d; + int e_bitlen; + int d_bitlen; + int bitlen; + size_t key_sz; +}; + +union starfive_alg_cr { + u32 v; + struct { + u32 start :1; + u32 aes_dma_en :1; + u32 rsvd_0 :1; + u32 hash_dma_en :1; + u32 alg_done :1; + u32 rsvd_1 :3; + u32 clear :1; + u32 rsvd_2 :23; + }; +}; + +struct starfive_cryp_ctx { + struct starfive_cryp_dev *cryp; + struct starfive_cryp_request_ctx *rctx; + + unsigned int hash_mode; + u8 key[MAX_KEY_SIZE]; + int keylen; + bool is_hmac; + struct starfive_rsa_key rsa_key; + struct crypto_akcipher *akcipher_fbk; + struct crypto_ahash *ahash_fbk; + struct crypto_aead *aead_fbk; +}; + +struct starfive_cryp_dev { + struct list_head list; + struct device *dev; + struct clk *hclk; + struct clk *ahb; + struct reset_control *rst; + + void __iomem *base; + phys_addr_t phys_base; + + u32 dma_maxburst; + struct dma_chan *tx; + struct dma_chan *rx; + struct dma_slave_config cfg_in; + struct dma_slave_config cfg_out; + struct scatter_walk in_walk; + struct scatter_walk out_walk; + struct crypto_engine *engine; + struct tasklet_struct aes_done; + struct tasklet_struct hash_done; + struct completion pka_done; + size_t assoclen; + size_t total_in; + size_t total_out; + u32 tag_in[4]; + u32 tag_out[4]; + unsigned int authsize; + unsigned long flags; + int err; + bool side_chan; + union starfive_alg_cr alg_cr; + union { + struct ahash_request *hreq; + struct aead_request *areq; + struct skcipher_request *sreq; + } req; +}; + +struct starfive_cryp_request_ctx { + union { + union starfive_hash_csr hash; + union starfive_pka_cacr pka; + union starfive_aes_csr aes; + } csr; + + struct scatterlist *in_sg; + struct scatterlist *out_sg; + struct ahash_request ahash_fbk_req; + size_t total; + size_t nents; + unsigned int blksize; + unsigned int digsize; + unsigned long in_sg_len; + unsigned char *adata; + u8 rsa_data[] __aligned(sizeof(u32)); +}; + +struct starfive_cryp_dev *starfive_cryp_find_dev(struct starfive_cryp_ctx *ctx); + +int starfive_hash_register_algs(void); +void starfive_hash_unregister_algs(void); + +int starfive_rsa_register_algs(void); +void starfive_rsa_unregister_algs(void); + +int starfive_aes_register_algs(void); +void starfive_aes_unregister_algs(void); + +void starfive_hash_done_task(unsigned long param); +void starfive_aes_done_task(unsigned long param); +#endif diff --git a/drivers/crypto/starfive/jh7110-hash.c b/drivers/crypto/starfive/jh7110-hash.c new file mode 100644 index 0000000000..cc7650198d --- /dev/null +++ b/drivers/crypto/starfive/jh7110-hash.c @@ -0,0 +1,915 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Hash function and HMAC support for StarFive driver + * + * Copyright (c) 2022 StarFive Technology + * + */ + +#include +#include +#include +#include "jh7110-cryp.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define STARFIVE_HASH_REGS_OFFSET 0x300 +#define STARFIVE_HASH_SHACSR (STARFIVE_HASH_REGS_OFFSET + 0x0) +#define STARFIVE_HASH_SHAWDR (STARFIVE_HASH_REGS_OFFSET + 0x4) +#define STARFIVE_HASH_SHARDR (STARFIVE_HASH_REGS_OFFSET + 0x8) +#define STARFIVE_HASH_SHAWSR (STARFIVE_HASH_REGS_OFFSET + 0xC) +#define STARFIVE_HASH_SHAWLEN3 (STARFIVE_HASH_REGS_OFFSET + 0x10) +#define STARFIVE_HASH_SHAWLEN2 (STARFIVE_HASH_REGS_OFFSET + 0x14) +#define STARFIVE_HASH_SHAWLEN1 (STARFIVE_HASH_REGS_OFFSET + 0x18) +#define STARFIVE_HASH_SHAWLEN0 (STARFIVE_HASH_REGS_OFFSET + 0x1C) +#define STARFIVE_HASH_SHAWKR (STARFIVE_HASH_REGS_OFFSET + 0x20) +#define STARFIVE_HASH_SHAWKLEN (STARFIVE_HASH_REGS_OFFSET + 0x24) + +#define STARFIVE_HASH_BUFLEN SHA512_BLOCK_SIZE +#define STARFIVE_HASH_RESET 0x2 + +static inline int starfive_hash_wait_busy(struct starfive_cryp_ctx *ctx) +{ + struct starfive_cryp_dev *cryp = ctx->cryp; + u32 status; + + return readl_relaxed_poll_timeout(cryp->base + STARFIVE_HASH_SHACSR, status, + !(status & STARFIVE_HASH_BUSY), 10, 100000); +} + +static inline int starfive_hash_wait_key_done(struct starfive_cryp_ctx *ctx) +{ + struct starfive_cryp_dev *cryp = ctx->cryp; + u32 status; + + return readl_relaxed_poll_timeout(cryp->base + STARFIVE_HASH_SHACSR, status, + (status & STARFIVE_HASH_KEY_DONE), 10, 100000); +} + +static int starfive_hash_hmac_key(struct starfive_cryp_ctx *ctx) +{ + struct starfive_cryp_request_ctx *rctx = ctx->rctx; + struct starfive_cryp_dev *cryp = ctx->cryp; + int klen = ctx->keylen, loop; + unsigned int *key = (unsigned int *)ctx->key; + unsigned char *cl; + + writel(ctx->keylen, cryp->base + STARFIVE_HASH_SHAWKLEN); + + rctx->csr.hash.hmac = 1; + rctx->csr.hash.key_flag = 1; + + writel(rctx->csr.hash.v, cryp->base + STARFIVE_HASH_SHACSR); + + for (loop = 0; loop < klen / sizeof(unsigned int); loop++, key++) + writel(*key, cryp->base + STARFIVE_HASH_SHAWKR); + + if (klen & 0x3) { + cl = (unsigned char *)key; + for (loop = 0; loop < (klen & 0x3); loop++, cl++) + writeb(*cl, cryp->base + STARFIVE_HASH_SHAWKR); + } + + if (starfive_hash_wait_key_done(ctx)) + return dev_err_probe(cryp->dev, -ETIMEDOUT, "starfive_hash_wait_key_done error\n"); + + return 0; +} + +static void starfive_hash_start(void *param) +{ + struct starfive_cryp_ctx *ctx = param; + struct starfive_cryp_request_ctx *rctx = ctx->rctx; + struct starfive_cryp_dev *cryp = ctx->cryp; + union starfive_alg_cr alg_cr; + union starfive_hash_csr csr; + u32 stat; + + dma_unmap_sg(cryp->dev, rctx->in_sg, rctx->in_sg_len, DMA_TO_DEVICE); + + alg_cr.v = 0; + alg_cr.clear = 1; + + writel(alg_cr.v, cryp->base + STARFIVE_ALG_CR_OFFSET); + + csr.v = readl(cryp->base + STARFIVE_HASH_SHACSR); + csr.firstb = 0; + csr.final = 1; + + stat = readl(cryp->base + STARFIVE_IE_MASK_OFFSET); + stat &= ~STARFIVE_IE_MASK_HASH_DONE; + writel(stat, cryp->base + STARFIVE_IE_MASK_OFFSET); + writel(csr.v, cryp->base + STARFIVE_HASH_SHACSR); +} + +static int starfive_hash_xmit_dma(struct starfive_cryp_ctx *ctx) +{ + struct starfive_cryp_request_ctx *rctx = ctx->rctx; + struct starfive_cryp_dev *cryp = ctx->cryp; + struct dma_async_tx_descriptor *in_desc; + union starfive_alg_cr alg_cr; + int total_len; + int ret; + + if (!rctx->total) { + starfive_hash_start(ctx); + return 0; + } + + writel(rctx->total, cryp->base + STARFIVE_DMA_IN_LEN_OFFSET); + + total_len = rctx->total; + total_len = (total_len & 0x3) ? (((total_len >> 2) + 1) << 2) : total_len; + sg_dma_len(rctx->in_sg) = total_len; + + alg_cr.v = 0; + alg_cr.start = 1; + alg_cr.hash_dma_en = 1; + + writel(alg_cr.v, cryp->base + STARFIVE_ALG_CR_OFFSET); + + ret = dma_map_sg(cryp->dev, rctx->in_sg, rctx->in_sg_len, DMA_TO_DEVICE); + if (!ret) + return dev_err_probe(cryp->dev, -EINVAL, "dma_map_sg() error\n"); + + cryp->cfg_in.direction = DMA_MEM_TO_DEV; + cryp->cfg_in.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + cryp->cfg_in.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + cryp->cfg_in.src_maxburst = cryp->dma_maxburst; + cryp->cfg_in.dst_maxburst = cryp->dma_maxburst; + cryp->cfg_in.dst_addr = cryp->phys_base + STARFIVE_ALG_FIFO_OFFSET; + + dmaengine_slave_config(cryp->tx, &cryp->cfg_in); + + in_desc = dmaengine_prep_slave_sg(cryp->tx, rctx->in_sg, + ret, DMA_MEM_TO_DEV, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + + if (!in_desc) + return -EINVAL; + + in_desc->callback = starfive_hash_start; + in_desc->callback_param = ctx; + + dmaengine_submit(in_desc); + dma_async_issue_pending(cryp->tx); + + return 0; +} + +static int starfive_hash_xmit(struct starfive_cryp_ctx *ctx) +{ + struct starfive_cryp_request_ctx *rctx = ctx->rctx; + struct starfive_cryp_dev *cryp = ctx->cryp; + int ret = 0; + + rctx->csr.hash.v = 0; + rctx->csr.hash.reset = 1; + writel(rctx->csr.hash.v, cryp->base + STARFIVE_HASH_SHACSR); + + if (starfive_hash_wait_busy(ctx)) + return dev_err_probe(cryp->dev, -ETIMEDOUT, "Error resetting engine.\n"); + + rctx->csr.hash.v = 0; + rctx->csr.hash.mode = ctx->hash_mode; + rctx->csr.hash.ie = 1; + + if (ctx->is_hmac) { + ret = starfive_hash_hmac_key(ctx); + if (ret) + return ret; + } else { + rctx->csr.hash.start = 1; + rctx->csr.hash.firstb = 1; + writel(rctx->csr.hash.v, cryp->base + STARFIVE_HASH_SHACSR); + } + + return starfive_hash_xmit_dma(ctx); +} + +static int starfive_hash_copy_hash(struct ahash_request *req) +{ + struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req); + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); + int count, *data; + int mlen; + + if (!req->result) + return 0; + + mlen = rctx->digsize / sizeof(u32); + data = (u32 *)req->result; + + for (count = 0; count < mlen; count++) + data[count] = readl(ctx->cryp->base + STARFIVE_HASH_SHARDR); + + return 0; +} + +void starfive_hash_done_task(unsigned long param) +{ + struct starfive_cryp_dev *cryp = (struct starfive_cryp_dev *)param; + int err = cryp->err; + + if (!err) + err = starfive_hash_copy_hash(cryp->req.hreq); + + /* Reset to clear hash_done in irq register*/ + writel(STARFIVE_HASH_RESET, cryp->base + STARFIVE_HASH_SHACSR); + + crypto_finalize_hash_request(cryp->engine, cryp->req.hreq, err); +} + +static int starfive_hash_check_aligned(struct scatterlist *sg, size_t total, size_t align) +{ + int len = 0; + + if (!total) + return 0; + + if (!IS_ALIGNED(total, align)) + return -EINVAL; + + while (sg) { + if (!IS_ALIGNED(sg->offset, sizeof(u32))) + return -EINVAL; + + if (!IS_ALIGNED(sg->length, align)) + return -EINVAL; + + len += sg->length; + sg = sg_next(sg); + } + + if (len != total) + return -EINVAL; + + return 0; +} + +static int starfive_hash_one_request(struct crypto_engine *engine, void *areq) +{ + struct ahash_request *req = container_of(areq, struct ahash_request, + base); + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); + struct starfive_cryp_dev *cryp = ctx->cryp; + + if (!cryp) + return -ENODEV; + + return starfive_hash_xmit(ctx); +} + +static int starfive_hash_init(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req); + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk); + ahash_request_set_callback(&rctx->ahash_fbk_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + + ahash_request_set_crypt(&rctx->ahash_fbk_req, req->src, + req->result, req->nbytes); + + return crypto_ahash_init(&rctx->ahash_fbk_req); +} + +static int starfive_hash_update(struct ahash_request *req) +{ + struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk); + ahash_request_set_callback(&rctx->ahash_fbk_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + + ahash_request_set_crypt(&rctx->ahash_fbk_req, req->src, + req->result, req->nbytes); + + return crypto_ahash_update(&rctx->ahash_fbk_req); +} + +static int starfive_hash_final(struct ahash_request *req) +{ + struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk); + ahash_request_set_callback(&rctx->ahash_fbk_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + + ahash_request_set_crypt(&rctx->ahash_fbk_req, req->src, + req->result, req->nbytes); + + return crypto_ahash_final(&rctx->ahash_fbk_req); +} + +static int starfive_hash_finup(struct ahash_request *req) +{ + struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk); + ahash_request_set_callback(&rctx->ahash_fbk_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + + ahash_request_set_crypt(&rctx->ahash_fbk_req, req->src, + req->result, req->nbytes); + + return crypto_ahash_finup(&rctx->ahash_fbk_req); +} + +static int starfive_hash_digest_fb(struct ahash_request *req) +{ + struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk); + ahash_request_set_callback(&rctx->ahash_fbk_req, req->base.flags, + req->base.complete, req->base.data); + + ahash_request_set_crypt(&rctx->ahash_fbk_req, req->src, + req->result, req->nbytes); + + return crypto_ahash_digest(&rctx->ahash_fbk_req); +} + +static int starfive_hash_digest(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm); + struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req); + struct starfive_cryp_dev *cryp = ctx->cryp; + + memset(rctx, 0, sizeof(struct starfive_cryp_request_ctx)); + + cryp->req.hreq = req; + rctx->total = req->nbytes; + rctx->in_sg = req->src; + rctx->blksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + rctx->digsize = crypto_ahash_digestsize(tfm); + rctx->in_sg_len = sg_nents_for_len(rctx->in_sg, rctx->total); + ctx->rctx = rctx; + + if (starfive_hash_check_aligned(rctx->in_sg, rctx->total, rctx->blksize)) + return starfive_hash_digest_fb(req); + + return crypto_transfer_hash_request_to_engine(cryp->engine, req); +} + +static int starfive_hash_export(struct ahash_request *req, void *out) +{ + struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk); + ahash_request_set_callback(&rctx->ahash_fbk_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + + return crypto_ahash_export(&rctx->ahash_fbk_req, out); +} + +static int starfive_hash_import(struct ahash_request *req, const void *in) +{ + struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk); + ahash_request_set_callback(&rctx->ahash_fbk_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + + return crypto_ahash_import(&rctx->ahash_fbk_req, in); +} + +static int starfive_hash_init_tfm(struct crypto_ahash *hash, + const char *alg_name, + unsigned int mode) +{ + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash); + + ctx->cryp = starfive_cryp_find_dev(ctx); + + if (!ctx->cryp) + return -ENODEV; + + ctx->ahash_fbk = crypto_alloc_ahash(alg_name, 0, + CRYPTO_ALG_NEED_FALLBACK); + + if (IS_ERR(ctx->ahash_fbk)) + return dev_err_probe(ctx->cryp->dev, PTR_ERR(ctx->ahash_fbk), + "starfive_hash: Could not load fallback driver.\n"); + + crypto_ahash_set_statesize(hash, crypto_ahash_statesize(ctx->ahash_fbk)); + crypto_ahash_set_reqsize(hash, sizeof(struct starfive_cryp_request_ctx) + + crypto_ahash_reqsize(ctx->ahash_fbk)); + + ctx->keylen = 0; + ctx->hash_mode = mode; + + return 0; +} + +static void starfive_hash_exit_tfm(struct crypto_ahash *hash) +{ + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash); + + crypto_free_ahash(ctx->ahash_fbk); +} + +static int starfive_hash_long_setkey(struct starfive_cryp_ctx *ctx, + const u8 *key, unsigned int keylen, + const char *alg_name) +{ + struct crypto_wait wait; + struct ahash_request *req; + struct scatterlist sg; + struct crypto_ahash *ahash_tfm; + u8 *buf; + int ret; + + ahash_tfm = crypto_alloc_ahash(alg_name, 0, 0); + if (IS_ERR(ahash_tfm)) + return PTR_ERR(ahash_tfm); + + req = ahash_request_alloc(ahash_tfm, GFP_KERNEL); + if (!req) { + ret = -ENOMEM; + goto err_free_ahash; + } + + crypto_init_wait(&wait); + ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + crypto_ahash_clear_flags(ahash_tfm, ~0); + + buf = kzalloc(keylen + STARFIVE_HASH_BUFLEN, GFP_KERNEL); + if (!buf) { + ret = -ENOMEM; + goto err_free_req; + } + + memcpy(buf, key, keylen); + sg_init_one(&sg, buf, keylen); + ahash_request_set_crypt(req, &sg, ctx->key, keylen); + + ret = crypto_wait_req(crypto_ahash_digest(req), &wait); + + kfree(buf); +err_free_req: + ahash_request_free(req); +err_free_ahash: + crypto_free_ahash(ahash_tfm); + return ret; +} + +static int starfive_hash_setkey(struct crypto_ahash *hash, + const u8 *key, unsigned int keylen) +{ + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash); + unsigned int digestsize = crypto_ahash_digestsize(hash); + unsigned int blocksize = crypto_ahash_blocksize(hash); + const char *alg_name; + + crypto_ahash_setkey(ctx->ahash_fbk, key, keylen); + + if (keylen <= blocksize) { + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + return 0; + } + + ctx->keylen = digestsize; + + switch (digestsize) { + case SHA224_DIGEST_SIZE: + alg_name = "sha224-starfive"; + break; + case SHA256_DIGEST_SIZE: + if (ctx->hash_mode == STARFIVE_HASH_SM3) + alg_name = "sm3-starfive"; + else + alg_name = "sha256-starfive"; + break; + case SHA384_DIGEST_SIZE: + alg_name = "sha384-starfive"; + break; + case SHA512_DIGEST_SIZE: + alg_name = "sha512-starfive"; + break; + default: + return -EINVAL; + } + + return starfive_hash_long_setkey(ctx, key, keylen, alg_name); +} + +static int starfive_sha224_init_tfm(struct crypto_ahash *hash) +{ + return starfive_hash_init_tfm(hash, "sha224-generic", + STARFIVE_HASH_SHA224); +} + +static int starfive_sha256_init_tfm(struct crypto_ahash *hash) +{ + return starfive_hash_init_tfm(hash, "sha256-generic", + STARFIVE_HASH_SHA256); +} + +static int starfive_sha384_init_tfm(struct crypto_ahash *hash) +{ + return starfive_hash_init_tfm(hash, "sha384-generic", + STARFIVE_HASH_SHA384); +} + +static int starfive_sha512_init_tfm(struct crypto_ahash *hash) +{ + return starfive_hash_init_tfm(hash, "sha512-generic", + STARFIVE_HASH_SHA512); +} + +static int starfive_sm3_init_tfm(struct crypto_ahash *hash) +{ + return starfive_hash_init_tfm(hash, "sm3-generic", + STARFIVE_HASH_SM3); +} + +static int starfive_hmac_sha224_init_tfm(struct crypto_ahash *hash) +{ + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash); + + ctx->is_hmac = true; + + return starfive_hash_init_tfm(hash, "hmac(sha224-generic)", + STARFIVE_HASH_SHA224); +} + +static int starfive_hmac_sha256_init_tfm(struct crypto_ahash *hash) +{ + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash); + + ctx->is_hmac = true; + + return starfive_hash_init_tfm(hash, "hmac(sha256-generic)", + STARFIVE_HASH_SHA256); +} + +static int starfive_hmac_sha384_init_tfm(struct crypto_ahash *hash) +{ + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash); + + ctx->is_hmac = true; + + return starfive_hash_init_tfm(hash, "hmac(sha384-generic)", + STARFIVE_HASH_SHA384); +} + +static int starfive_hmac_sha512_init_tfm(struct crypto_ahash *hash) +{ + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash); + + ctx->is_hmac = true; + + return starfive_hash_init_tfm(hash, "hmac(sha512-generic)", + STARFIVE_HASH_SHA512); +} + +static int starfive_hmac_sm3_init_tfm(struct crypto_ahash *hash) +{ + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash); + + ctx->is_hmac = true; + + return starfive_hash_init_tfm(hash, "hmac(sm3-generic)", + STARFIVE_HASH_SM3); +} + +static struct ahash_engine_alg algs_sha2_sm3[] = { +{ + .base.init = starfive_hash_init, + .base.update = starfive_hash_update, + .base.final = starfive_hash_final, + .base.finup = starfive_hash_finup, + .base.digest = starfive_hash_digest, + .base.export = starfive_hash_export, + .base.import = starfive_hash_import, + .base.init_tfm = starfive_sha224_init_tfm, + .base.exit_tfm = starfive_hash_exit_tfm, + .base.halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct sha256_state), + .base = { + .cra_name = "sha224", + .cra_driver_name = "sha224-starfive", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = starfive_hash_one_request, + }, +}, { + .base.init = starfive_hash_init, + .base.update = starfive_hash_update, + .base.final = starfive_hash_final, + .base.finup = starfive_hash_finup, + .base.digest = starfive_hash_digest, + .base.export = starfive_hash_export, + .base.import = starfive_hash_import, + .base.init_tfm = starfive_hmac_sha224_init_tfm, + .base.exit_tfm = starfive_hash_exit_tfm, + .base.setkey = starfive_hash_setkey, + .base.halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct sha256_state), + .base = { + .cra_name = "hmac(sha224)", + .cra_driver_name = "sha224-hmac-starfive", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = starfive_hash_one_request, + }, +}, { + .base.init = starfive_hash_init, + .base.update = starfive_hash_update, + .base.final = starfive_hash_final, + .base.finup = starfive_hash_finup, + .base.digest = starfive_hash_digest, + .base.export = starfive_hash_export, + .base.import = starfive_hash_import, + .base.init_tfm = starfive_sha256_init_tfm, + .base.exit_tfm = starfive_hash_exit_tfm, + .base.halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct sha256_state), + .base = { + .cra_name = "sha256", + .cra_driver_name = "sha256-starfive", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = starfive_hash_one_request, + }, +}, { + .base.init = starfive_hash_init, + .base.update = starfive_hash_update, + .base.final = starfive_hash_final, + .base.finup = starfive_hash_finup, + .base.digest = starfive_hash_digest, + .base.export = starfive_hash_export, + .base.import = starfive_hash_import, + .base.init_tfm = starfive_hmac_sha256_init_tfm, + .base.exit_tfm = starfive_hash_exit_tfm, + .base.setkey = starfive_hash_setkey, + .base.halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct sha256_state), + .base = { + .cra_name = "hmac(sha256)", + .cra_driver_name = "sha256-hmac-starfive", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = starfive_hash_one_request, + }, +}, { + .base.init = starfive_hash_init, + .base.update = starfive_hash_update, + .base.final = starfive_hash_final, + .base.finup = starfive_hash_finup, + .base.digest = starfive_hash_digest, + .base.export = starfive_hash_export, + .base.import = starfive_hash_import, + .base.init_tfm = starfive_sha384_init_tfm, + .base.exit_tfm = starfive_hash_exit_tfm, + .base.halg = { + .digestsize = SHA384_DIGEST_SIZE, + .statesize = sizeof(struct sha512_state), + .base = { + .cra_name = "sha384", + .cra_driver_name = "sha384-starfive", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = starfive_hash_one_request, + }, +}, { + .base.init = starfive_hash_init, + .base.update = starfive_hash_update, + .base.final = starfive_hash_final, + .base.finup = starfive_hash_finup, + .base.digest = starfive_hash_digest, + .base.export = starfive_hash_export, + .base.import = starfive_hash_import, + .base.init_tfm = starfive_hmac_sha384_init_tfm, + .base.exit_tfm = starfive_hash_exit_tfm, + .base.setkey = starfive_hash_setkey, + .base.halg = { + .digestsize = SHA384_DIGEST_SIZE, + .statesize = sizeof(struct sha512_state), + .base = { + .cra_name = "hmac(sha384)", + .cra_driver_name = "sha384-hmac-starfive", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = starfive_hash_one_request, + }, +}, { + .base.init = starfive_hash_init, + .base.update = starfive_hash_update, + .base.final = starfive_hash_final, + .base.finup = starfive_hash_finup, + .base.digest = starfive_hash_digest, + .base.export = starfive_hash_export, + .base.import = starfive_hash_import, + .base.init_tfm = starfive_sha512_init_tfm, + .base.exit_tfm = starfive_hash_exit_tfm, + .base.halg = { + .digestsize = SHA512_DIGEST_SIZE, + .statesize = sizeof(struct sha512_state), + .base = { + .cra_name = "sha512", + .cra_driver_name = "sha512-starfive", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = starfive_hash_one_request, + }, +}, { + .base.init = starfive_hash_init, + .base.update = starfive_hash_update, + .base.final = starfive_hash_final, + .base.finup = starfive_hash_finup, + .base.digest = starfive_hash_digest, + .base.export = starfive_hash_export, + .base.import = starfive_hash_import, + .base.init_tfm = starfive_hmac_sha512_init_tfm, + .base.exit_tfm = starfive_hash_exit_tfm, + .base.setkey = starfive_hash_setkey, + .base.halg = { + .digestsize = SHA512_DIGEST_SIZE, + .statesize = sizeof(struct sha512_state), + .base = { + .cra_name = "hmac(sha512)", + .cra_driver_name = "sha512-hmac-starfive", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = starfive_hash_one_request, + }, +}, { + .base.init = starfive_hash_init, + .base.update = starfive_hash_update, + .base.final = starfive_hash_final, + .base.finup = starfive_hash_finup, + .base.digest = starfive_hash_digest, + .base.export = starfive_hash_export, + .base.import = starfive_hash_import, + .base.init_tfm = starfive_sm3_init_tfm, + .base.exit_tfm = starfive_hash_exit_tfm, + .base.halg = { + .digestsize = SM3_DIGEST_SIZE, + .statesize = sizeof(struct sm3_state), + .base = { + .cra_name = "sm3", + .cra_driver_name = "sm3-starfive", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SM3_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = starfive_hash_one_request, + }, +}, { + .base.init = starfive_hash_init, + .base.update = starfive_hash_update, + .base.final = starfive_hash_final, + .base.finup = starfive_hash_finup, + .base.digest = starfive_hash_digest, + .base.export = starfive_hash_export, + .base.import = starfive_hash_import, + .base.init_tfm = starfive_hmac_sm3_init_tfm, + .base.exit_tfm = starfive_hash_exit_tfm, + .base.setkey = starfive_hash_setkey, + .base.halg = { + .digestsize = SM3_DIGEST_SIZE, + .statesize = sizeof(struct sm3_state), + .base = { + .cra_name = "hmac(sm3)", + .cra_driver_name = "sm3-hmac-starfive", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SM3_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = starfive_hash_one_request, + }, +}, +}; + +int starfive_hash_register_algs(void) +{ + return crypto_engine_register_ahashes(algs_sha2_sm3, ARRAY_SIZE(algs_sha2_sm3)); +} + +void starfive_hash_unregister_algs(void) +{ + crypto_engine_unregister_ahashes(algs_sha2_sm3, ARRAY_SIZE(algs_sha2_sm3)); +} diff --git a/drivers/crypto/starfive/jh7110-rsa.c b/drivers/crypto/starfive/jh7110-rsa.c new file mode 100644 index 0000000000..f31bbd825f --- /dev/null +++ b/drivers/crypto/starfive/jh7110-rsa.c @@ -0,0 +1,617 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * StarFive Public Key Algo acceleration driver + * + * Copyright (c) 2022 StarFive Technology + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "jh7110-cryp.h" + +#define STARFIVE_PKA_REGS_OFFSET 0x400 +#define STARFIVE_PKA_CACR_OFFSET (STARFIVE_PKA_REGS_OFFSET + 0x0) +#define STARFIVE_PKA_CASR_OFFSET (STARFIVE_PKA_REGS_OFFSET + 0x4) +#define STARFIVE_PKA_CAAR_OFFSET (STARFIVE_PKA_REGS_OFFSET + 0x8) +#define STARFIVE_PKA_CAER_OFFSET (STARFIVE_PKA_REGS_OFFSET + 0x108) +#define STARFIVE_PKA_CANR_OFFSET (STARFIVE_PKA_REGS_OFFSET + 0x208) + +// R^2 mod N and N0' +#define CRYPTO_CMD_PRE 0x0 +// A * R mod N ==> A +#define CRYPTO_CMD_ARN 0x5 +// A * E * R mod N ==> A +#define CRYPTO_CMD_AERN 0x6 +// A * A * R mod N ==> A +#define CRYPTO_CMD_AARN 0x7 + +#define STARFIVE_RSA_MAX_KEYSZ 256 +#define STARFIVE_RSA_RESET 0x2 + +static inline int starfive_pka_wait_done(struct starfive_cryp_ctx *ctx) +{ + struct starfive_cryp_dev *cryp = ctx->cryp; + + return wait_for_completion_timeout(&cryp->pka_done, + usecs_to_jiffies(100000)); +} + +static inline void starfive_pka_irq_mask_clear(struct starfive_cryp_ctx *ctx) +{ + struct starfive_cryp_dev *cryp = ctx->cryp; + u32 stat; + + stat = readl(cryp->base + STARFIVE_IE_MASK_OFFSET); + stat &= ~STARFIVE_IE_MASK_PKA_DONE; + writel(stat, cryp->base + STARFIVE_IE_MASK_OFFSET); + + reinit_completion(&cryp->pka_done); +} + +static void starfive_rsa_free_key(struct starfive_rsa_key *key) +{ + if (key->d) + kfree_sensitive(key->d); + if (key->e) + kfree_sensitive(key->e); + if (key->n) + kfree_sensitive(key->n); + memset(key, 0, sizeof(*key)); +} + +static unsigned int starfive_rsa_get_nbit(u8 *pa, u32 snum, int key_sz) +{ + u32 i; + u8 value; + + i = snum >> 3; + + value = pa[key_sz - i - 1]; + value >>= snum & 0x7; + value &= 0x1; + + return value; +} + +static int starfive_rsa_montgomery_form(struct starfive_cryp_ctx *ctx, + u32 *out, u32 *in, u8 mont, + u32 *mod, int bit_len) +{ + struct starfive_cryp_dev *cryp = ctx->cryp; + struct starfive_cryp_request_ctx *rctx = ctx->rctx; + int count = rctx->total / sizeof(u32) - 1; + int loop; + u32 temp; + u8 opsize; + + opsize = (bit_len - 1) >> 5; + rctx->csr.pka.v = 0; + + writel(rctx->csr.pka.v, cryp->base + STARFIVE_PKA_CACR_OFFSET); + + for (loop = 0; loop <= opsize; loop++) + writel(mod[opsize - loop], cryp->base + STARFIVE_PKA_CANR_OFFSET + loop * 4); + + if (mont) { + rctx->csr.pka.v = 0; + rctx->csr.pka.cln_done = 1; + rctx->csr.pka.opsize = opsize; + rctx->csr.pka.exposize = opsize; + rctx->csr.pka.cmd = CRYPTO_CMD_PRE; + rctx->csr.pka.start = 1; + rctx->csr.pka.not_r2 = 1; + rctx->csr.pka.ie = 1; + + starfive_pka_irq_mask_clear(ctx); + writel(rctx->csr.pka.v, cryp->base + STARFIVE_PKA_CACR_OFFSET); + + if (!starfive_pka_wait_done(ctx)) + return -ETIMEDOUT; + + for (loop = 0; loop <= opsize; loop++) + writel(in[opsize - loop], cryp->base + STARFIVE_PKA_CAAR_OFFSET + loop * 4); + + writel(0x1000000, cryp->base + STARFIVE_PKA_CAER_OFFSET); + + for (loop = 1; loop <= opsize; loop++) + writel(0, cryp->base + STARFIVE_PKA_CAER_OFFSET + loop * 4); + + rctx->csr.pka.v = 0; + rctx->csr.pka.cln_done = 1; + rctx->csr.pka.opsize = opsize; + rctx->csr.pka.exposize = opsize; + rctx->csr.pka.cmd = CRYPTO_CMD_AERN; + rctx->csr.pka.start = 1; + rctx->csr.pka.ie = 1; + + starfive_pka_irq_mask_clear(ctx); + writel(rctx->csr.pka.v, cryp->base + STARFIVE_PKA_CACR_OFFSET); + + if (!starfive_pka_wait_done(ctx)) + return -ETIMEDOUT; + } else { + rctx->csr.pka.v = 0; + rctx->csr.pka.cln_done = 1; + rctx->csr.pka.opsize = opsize; + rctx->csr.pka.exposize = opsize; + rctx->csr.pka.cmd = CRYPTO_CMD_PRE; + rctx->csr.pka.start = 1; + rctx->csr.pka.pre_expf = 1; + rctx->csr.pka.ie = 1; + + starfive_pka_irq_mask_clear(ctx); + writel(rctx->csr.pka.v, cryp->base + STARFIVE_PKA_CACR_OFFSET); + + if (!starfive_pka_wait_done(ctx)) + return -ETIMEDOUT; + + for (loop = 0; loop <= count; loop++) + writel(in[count - loop], cryp->base + STARFIVE_PKA_CAER_OFFSET + loop * 4); + + /*pad with 0 up to opsize*/ + for (loop = count + 1; loop <= opsize; loop++) + writel(0, cryp->base + STARFIVE_PKA_CAER_OFFSET + loop * 4); + + rctx->csr.pka.v = 0; + rctx->csr.pka.cln_done = 1; + rctx->csr.pka.opsize = opsize; + rctx->csr.pka.exposize = opsize; + rctx->csr.pka.cmd = CRYPTO_CMD_ARN; + rctx->csr.pka.start = 1; + rctx->csr.pka.ie = 1; + + starfive_pka_irq_mask_clear(ctx); + writel(rctx->csr.pka.v, cryp->base + STARFIVE_PKA_CACR_OFFSET); + + if (!starfive_pka_wait_done(ctx)) + return -ETIMEDOUT; + } + + for (loop = 0; loop <= opsize; loop++) { + temp = readl(cryp->base + STARFIVE_PKA_CAAR_OFFSET + 0x4 * loop); + out[opsize - loop] = temp; + } + + return 0; +} + +static int starfive_rsa_cpu_start(struct starfive_cryp_ctx *ctx, u32 *result, + u8 *de, u32 *n, int key_sz) +{ + struct starfive_cryp_dev *cryp = ctx->cryp; + struct starfive_cryp_request_ctx *rctx = ctx->rctx; + struct starfive_rsa_key *key = &ctx->rsa_key; + u32 temp; + int ret = 0; + int opsize, mlen, loop; + unsigned int *mta; + + opsize = (key_sz - 1) >> 2; + + mta = kmalloc(key_sz, GFP_KERNEL); + if (!mta) + return -ENOMEM; + + ret = starfive_rsa_montgomery_form(ctx, mta, (u32 *)rctx->rsa_data, + 0, n, key_sz << 3); + if (ret) { + dev_err_probe(cryp->dev, ret, "Conversion to Montgomery failed"); + goto rsa_err; + } + + for (loop = 0; loop <= opsize; loop++) + writel(mta[opsize - loop], + cryp->base + STARFIVE_PKA_CAER_OFFSET + loop * 4); + + for (loop = key->bitlen - 1; loop > 0; loop--) { + mlen = starfive_rsa_get_nbit(de, loop - 1, key_sz); + + rctx->csr.pka.v = 0; + rctx->csr.pka.cln_done = 1; + rctx->csr.pka.opsize = opsize; + rctx->csr.pka.exposize = opsize; + rctx->csr.pka.cmd = CRYPTO_CMD_AARN; + rctx->csr.pka.start = 1; + rctx->csr.pka.ie = 1; + + starfive_pka_irq_mask_clear(ctx); + writel(rctx->csr.pka.v, cryp->base + STARFIVE_PKA_CACR_OFFSET); + + ret = -ETIMEDOUT; + if (!starfive_pka_wait_done(ctx)) + goto rsa_err; + + if (mlen) { + rctx->csr.pka.v = 0; + rctx->csr.pka.cln_done = 1; + rctx->csr.pka.opsize = opsize; + rctx->csr.pka.exposize = opsize; + rctx->csr.pka.cmd = CRYPTO_CMD_AERN; + rctx->csr.pka.start = 1; + rctx->csr.pka.ie = 1; + + starfive_pka_irq_mask_clear(ctx); + writel(rctx->csr.pka.v, cryp->base + STARFIVE_PKA_CACR_OFFSET); + + if (!starfive_pka_wait_done(ctx)) + goto rsa_err; + } + } + + for (loop = 0; loop <= opsize; loop++) { + temp = readl(cryp->base + STARFIVE_PKA_CAAR_OFFSET + 0x4 * loop); + result[opsize - loop] = temp; + } + + ret = starfive_rsa_montgomery_form(ctx, result, result, 1, n, key_sz << 3); + if (ret) + dev_err_probe(cryp->dev, ret, "Conversion from Montgomery failed"); +rsa_err: + kfree(mta); + return ret; +} + +static int starfive_rsa_start(struct starfive_cryp_ctx *ctx, u8 *result, + u8 *de, u8 *n, int key_sz) +{ + return starfive_rsa_cpu_start(ctx, (u32 *)result, de, (u32 *)n, key_sz); +} + +static int starfive_rsa_enc_core(struct starfive_cryp_ctx *ctx, int enc) +{ + struct starfive_cryp_dev *cryp = ctx->cryp; + struct starfive_cryp_request_ctx *rctx = ctx->rctx; + struct starfive_rsa_key *key = &ctx->rsa_key; + int ret = 0; + + writel(STARFIVE_RSA_RESET, cryp->base + STARFIVE_PKA_CACR_OFFSET); + + rctx->total = sg_copy_to_buffer(rctx->in_sg, rctx->nents, + rctx->rsa_data, rctx->total); + + if (enc) { + key->bitlen = key->e_bitlen; + ret = starfive_rsa_start(ctx, rctx->rsa_data, key->e, + key->n, key->key_sz); + } else { + key->bitlen = key->d_bitlen; + ret = starfive_rsa_start(ctx, rctx->rsa_data, key->d, + key->n, key->key_sz); + } + + if (ret) + goto err_rsa_crypt; + + sg_copy_buffer(rctx->out_sg, sg_nents(rctx->out_sg), + rctx->rsa_data, key->key_sz, 0, 0); + +err_rsa_crypt: + writel(STARFIVE_RSA_RESET, cryp->base + STARFIVE_PKA_CACR_OFFSET); + kfree(rctx->rsa_data); + return ret; +} + +static int starfive_rsa_enc(struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct starfive_cryp_ctx *ctx = akcipher_tfm_ctx(tfm); + struct starfive_cryp_dev *cryp = ctx->cryp; + struct starfive_rsa_key *key = &ctx->rsa_key; + struct starfive_cryp_request_ctx *rctx = akcipher_request_ctx(req); + int ret; + + if (!key->key_sz) { + akcipher_request_set_tfm(req, ctx->akcipher_fbk); + ret = crypto_akcipher_encrypt(req); + akcipher_request_set_tfm(req, tfm); + return ret; + } + + if (unlikely(!key->n || !key->e)) + return -EINVAL; + + if (req->dst_len < key->key_sz) + return dev_err_probe(cryp->dev, -EOVERFLOW, + "Output buffer length less than parameter n\n"); + + rctx->in_sg = req->src; + rctx->out_sg = req->dst; + rctx->total = req->src_len; + rctx->nents = sg_nents(rctx->in_sg); + ctx->rctx = rctx; + + return starfive_rsa_enc_core(ctx, 1); +} + +static int starfive_rsa_dec(struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct starfive_cryp_ctx *ctx = akcipher_tfm_ctx(tfm); + struct starfive_cryp_dev *cryp = ctx->cryp; + struct starfive_rsa_key *key = &ctx->rsa_key; + struct starfive_cryp_request_ctx *rctx = akcipher_request_ctx(req); + int ret; + + if (!key->key_sz) { + akcipher_request_set_tfm(req, ctx->akcipher_fbk); + ret = crypto_akcipher_decrypt(req); + akcipher_request_set_tfm(req, tfm); + return ret; + } + + if (unlikely(!key->n || !key->d)) + return -EINVAL; + + if (req->dst_len < key->key_sz) + return dev_err_probe(cryp->dev, -EOVERFLOW, + "Output buffer length less than parameter n\n"); + + rctx->in_sg = req->src; + rctx->out_sg = req->dst; + ctx->rctx = rctx; + rctx->total = req->src_len; + + return starfive_rsa_enc_core(ctx, 0); +} + +static int starfive_rsa_set_n(struct starfive_rsa_key *rsa_key, + const char *value, size_t vlen) +{ + const char *ptr = value; + unsigned int bitslen; + int ret; + + while (!*ptr && vlen) { + ptr++; + vlen--; + } + rsa_key->key_sz = vlen; + bitslen = rsa_key->key_sz << 3; + + /* check valid key size */ + if (bitslen & 0x1f) + return -EINVAL; + + ret = -ENOMEM; + rsa_key->n = kmemdup(ptr, rsa_key->key_sz, GFP_KERNEL); + if (!rsa_key->n) + goto err; + + return 0; + err: + rsa_key->key_sz = 0; + rsa_key->n = NULL; + starfive_rsa_free_key(rsa_key); + return ret; +} + +static int starfive_rsa_set_e(struct starfive_rsa_key *rsa_key, + const char *value, size_t vlen) +{ + const char *ptr = value; + unsigned char pt; + int loop; + + while (!*ptr && vlen) { + ptr++; + vlen--; + } + pt = *ptr; + + if (!rsa_key->key_sz || !vlen || vlen > rsa_key->key_sz) { + rsa_key->e = NULL; + return -EINVAL; + } + + rsa_key->e = kzalloc(rsa_key->key_sz, GFP_KERNEL); + if (!rsa_key->e) + return -ENOMEM; + + for (loop = 8; loop > 0; loop--) { + if (pt >> (loop - 1)) + break; + } + + rsa_key->e_bitlen = (vlen - 1) * 8 + loop; + + memcpy(rsa_key->e + (rsa_key->key_sz - vlen), ptr, vlen); + + return 0; +} + +static int starfive_rsa_set_d(struct starfive_rsa_key *rsa_key, + const char *value, size_t vlen) +{ + const char *ptr = value; + unsigned char pt; + int loop; + int ret; + + while (!*ptr && vlen) { + ptr++; + vlen--; + } + pt = *ptr; + + ret = -EINVAL; + if (!rsa_key->key_sz || !vlen || vlen > rsa_key->key_sz) + goto err; + + ret = -ENOMEM; + rsa_key->d = kzalloc(rsa_key->key_sz, GFP_KERNEL); + if (!rsa_key->d) + goto err; + + for (loop = 8; loop > 0; loop--) { + if (pt >> (loop - 1)) + break; + } + + rsa_key->d_bitlen = (vlen - 1) * 8 + loop; + + memcpy(rsa_key->d + (rsa_key->key_sz - vlen), ptr, vlen); + + return 0; + err: + rsa_key->d = NULL; + return ret; +} + +static int starfive_rsa_setkey(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen, bool private) +{ + struct starfive_cryp_ctx *ctx = akcipher_tfm_ctx(tfm); + struct rsa_key raw_key = {NULL}; + struct starfive_rsa_key *rsa_key = &ctx->rsa_key; + int ret; + + if (private) + ret = rsa_parse_priv_key(&raw_key, key, keylen); + else + ret = rsa_parse_pub_key(&raw_key, key, keylen); + if (ret < 0) + goto err; + + starfive_rsa_free_key(rsa_key); + + /* Use fallback for mod > 256 + 1 byte prefix */ + if (raw_key.n_sz > STARFIVE_RSA_MAX_KEYSZ + 1) + return 0; + + ret = starfive_rsa_set_n(rsa_key, raw_key.n, raw_key.n_sz); + if (ret) + return ret; + + ret = starfive_rsa_set_e(rsa_key, raw_key.e, raw_key.e_sz); + if (ret) + goto err; + + if (private) { + ret = starfive_rsa_set_d(rsa_key, raw_key.d, raw_key.d_sz); + if (ret) + goto err; + } + + if (!rsa_key->n || !rsa_key->e) { + ret = -EINVAL; + goto err; + } + + if (private && !rsa_key->d) { + ret = -EINVAL; + goto err; + } + + return 0; + err: + starfive_rsa_free_key(rsa_key); + return ret; +} + +static int starfive_rsa_set_pub_key(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen) +{ + struct starfive_cryp_ctx *ctx = akcipher_tfm_ctx(tfm); + int ret; + + ret = crypto_akcipher_set_pub_key(ctx->akcipher_fbk, key, keylen); + if (ret) + return ret; + + return starfive_rsa_setkey(tfm, key, keylen, false); +} + +static int starfive_rsa_set_priv_key(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen) +{ + struct starfive_cryp_ctx *ctx = akcipher_tfm_ctx(tfm); + int ret; + + ret = crypto_akcipher_set_priv_key(ctx->akcipher_fbk, key, keylen); + if (ret) + return ret; + + return starfive_rsa_setkey(tfm, key, keylen, true); +} + +static unsigned int starfive_rsa_max_size(struct crypto_akcipher *tfm) +{ + struct starfive_cryp_ctx *ctx = akcipher_tfm_ctx(tfm); + + if (ctx->rsa_key.key_sz) + return ctx->rsa_key.key_sz; + + return crypto_akcipher_maxsize(ctx->akcipher_fbk); +} + +static int starfive_rsa_init_tfm(struct crypto_akcipher *tfm) +{ + struct starfive_cryp_ctx *ctx = akcipher_tfm_ctx(tfm); + + ctx->akcipher_fbk = crypto_alloc_akcipher("rsa-generic", 0, 0); + if (IS_ERR(ctx->akcipher_fbk)) + return PTR_ERR(ctx->akcipher_fbk); + + ctx->cryp = starfive_cryp_find_dev(ctx); + if (!ctx->cryp) { + crypto_free_akcipher(ctx->akcipher_fbk); + return -ENODEV; + } + + akcipher_set_reqsize(tfm, sizeof(struct starfive_cryp_request_ctx) + + sizeof(struct crypto_akcipher) + 32); + + return 0; +} + +static void starfive_rsa_exit_tfm(struct crypto_akcipher *tfm) +{ + struct starfive_cryp_ctx *ctx = akcipher_tfm_ctx(tfm); + struct starfive_rsa_key *key = (struct starfive_rsa_key *)&ctx->rsa_key; + + crypto_free_akcipher(ctx->akcipher_fbk); + starfive_rsa_free_key(key); +} + +static struct akcipher_alg starfive_rsa = { + .encrypt = starfive_rsa_enc, + .decrypt = starfive_rsa_dec, + .sign = starfive_rsa_dec, + .verify = starfive_rsa_enc, + .set_pub_key = starfive_rsa_set_pub_key, + .set_priv_key = starfive_rsa_set_priv_key, + .max_size = starfive_rsa_max_size, + .init = starfive_rsa_init_tfm, + .exit = starfive_rsa_exit_tfm, + .base = { + .cra_name = "rsa", + .cra_driver_name = "starfive-rsa", + .cra_flags = CRYPTO_ALG_TYPE_AKCIPHER | + CRYPTO_ALG_NEED_FALLBACK, + .cra_priority = 3000, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), + }, +}; + +int starfive_rsa_register_algs(void) +{ + return crypto_register_akcipher(&starfive_rsa); +} + +void starfive_rsa_unregister_algs(void) +{ + crypto_unregister_akcipher(&starfive_rsa); +} -- cgit v1.2.3