diff options
Diffstat (limited to 'drivers/crypto/axis/artpec6_crypto.c')
| -rw-r--r-- | drivers/crypto/axis/artpec6_crypto.c | 3204 |
1 files changed, 3204 insertions, 0 deletions
diff --git a/drivers/crypto/axis/artpec6_crypto.c b/drivers/crypto/axis/artpec6_crypto.c new file mode 100644 index 000000000..3caf57ebd --- /dev/null +++ b/drivers/crypto/axis/artpec6_crypto.c @@ -0,0 +1,3204 @@ +/* + * Driver for ARTPEC-6 crypto block using the kernel asynchronous crypto api. + * + * Copyright (C) 2014-2017 Axis Communications AB + */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/bitfield.h> +#include <linux/crypto.h> +#include <linux/debugfs.h> +#include <linux/delay.h> +#include <linux/dma-mapping.h> +#include <linux/fault-inject.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/kernel.h> +#include <linux/list.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/scatterlist.h> +#include <linux/slab.h> + +#include <crypto/aes.h> +#include <crypto/gcm.h> +#include <crypto/internal/aead.h> +#include <crypto/internal/hash.h> +#include <crypto/internal/skcipher.h> +#include <crypto/scatterwalk.h> +#include <crypto/sha.h> +#include <crypto/xts.h> + +/* Max length of a line in all cache levels for Artpec SoCs. */ +#define ARTPEC_CACHE_LINE_MAX 32 + +#define PDMA_OUT_CFG 0x0000 +#define PDMA_OUT_BUF_CFG 0x0004 +#define PDMA_OUT_CMD 0x0008 +#define PDMA_OUT_DESCRQ_PUSH 0x0010 +#define PDMA_OUT_DESCRQ_STAT 0x0014 + +#define A6_PDMA_IN_CFG 0x0028 +#define A6_PDMA_IN_BUF_CFG 0x002c +#define A6_PDMA_IN_CMD 0x0030 +#define A6_PDMA_IN_STATQ_PUSH 0x0038 +#define A6_PDMA_IN_DESCRQ_PUSH 0x0044 +#define A6_PDMA_IN_DESCRQ_STAT 0x0048 +#define A6_PDMA_INTR_MASK 0x0068 +#define A6_PDMA_ACK_INTR 0x006c +#define A6_PDMA_MASKED_INTR 0x0074 + +#define A7_PDMA_IN_CFG 0x002c +#define A7_PDMA_IN_BUF_CFG 0x0030 +#define A7_PDMA_IN_CMD 0x0034 +#define A7_PDMA_IN_STATQ_PUSH 0x003c +#define A7_PDMA_IN_DESCRQ_PUSH 0x0048 +#define A7_PDMA_IN_DESCRQ_STAT 0x004C +#define A7_PDMA_INTR_MASK 0x006c +#define A7_PDMA_ACK_INTR 0x0070 +#define A7_PDMA_MASKED_INTR 0x0078 + +#define PDMA_OUT_CFG_EN BIT(0) + +#define PDMA_OUT_BUF_CFG_DATA_BUF_SIZE GENMASK(4, 0) +#define PDMA_OUT_BUF_CFG_DESCR_BUF_SIZE GENMASK(9, 5) + +#define PDMA_OUT_CMD_START BIT(0) +#define A6_PDMA_OUT_CMD_STOP BIT(3) +#define A7_PDMA_OUT_CMD_STOP BIT(2) + +#define PDMA_OUT_DESCRQ_PUSH_LEN GENMASK(5, 0) +#define PDMA_OUT_DESCRQ_PUSH_ADDR GENMASK(31, 6) + +#define PDMA_OUT_DESCRQ_STAT_LEVEL GENMASK(3, 0) +#define PDMA_OUT_DESCRQ_STAT_SIZE GENMASK(7, 4) + +#define PDMA_IN_CFG_EN BIT(0) + +#define PDMA_IN_BUF_CFG_DATA_BUF_SIZE GENMASK(4, 0) +#define PDMA_IN_BUF_CFG_DESCR_BUF_SIZE GENMASK(9, 5) +#define PDMA_IN_BUF_CFG_STAT_BUF_SIZE GENMASK(14, 10) + +#define PDMA_IN_CMD_START BIT(0) +#define A6_PDMA_IN_CMD_FLUSH_STAT BIT(2) +#define A6_PDMA_IN_CMD_STOP BIT(3) +#define A7_PDMA_IN_CMD_FLUSH_STAT BIT(1) +#define A7_PDMA_IN_CMD_STOP BIT(2) + +#define PDMA_IN_STATQ_PUSH_LEN GENMASK(5, 0) +#define PDMA_IN_STATQ_PUSH_ADDR GENMASK(31, 6) + +#define PDMA_IN_DESCRQ_PUSH_LEN GENMASK(5, 0) +#define PDMA_IN_DESCRQ_PUSH_ADDR GENMASK(31, 6) + +#define PDMA_IN_DESCRQ_STAT_LEVEL GENMASK(3, 0) +#define PDMA_IN_DESCRQ_STAT_SIZE GENMASK(7, 4) + +#define A6_PDMA_INTR_MASK_IN_DATA BIT(2) +#define A6_PDMA_INTR_MASK_IN_EOP BIT(3) +#define A6_PDMA_INTR_MASK_IN_EOP_FLUSH BIT(4) + +#define A7_PDMA_INTR_MASK_IN_DATA BIT(3) +#define A7_PDMA_INTR_MASK_IN_EOP BIT(4) +#define A7_PDMA_INTR_MASK_IN_EOP_FLUSH BIT(5) + +#define A6_CRY_MD_OPER GENMASK(19, 16) + +#define A6_CRY_MD_HASH_SEL_CTX GENMASK(21, 20) +#define A6_CRY_MD_HASH_HMAC_FIN BIT(23) + +#define A6_CRY_MD_CIPHER_LEN GENMASK(21, 20) +#define A6_CRY_MD_CIPHER_DECR BIT(22) +#define A6_CRY_MD_CIPHER_TWEAK BIT(23) +#define A6_CRY_MD_CIPHER_DSEQ BIT(24) + +#define A7_CRY_MD_OPER GENMASK(11, 8) + +#define A7_CRY_MD_HASH_SEL_CTX GENMASK(13, 12) +#define A7_CRY_MD_HASH_HMAC_FIN BIT(15) + +#define A7_CRY_MD_CIPHER_LEN GENMASK(13, 12) +#define A7_CRY_MD_CIPHER_DECR BIT(14) +#define A7_CRY_MD_CIPHER_TWEAK BIT(15) +#define A7_CRY_MD_CIPHER_DSEQ BIT(16) + +/* DMA metadata constants */ +#define regk_crypto_aes_cbc 0x00000002 +#define regk_crypto_aes_ctr 0x00000003 +#define regk_crypto_aes_ecb 0x00000001 +#define regk_crypto_aes_gcm 0x00000004 +#define regk_crypto_aes_xts 0x00000005 +#define regk_crypto_cache 0x00000002 +#define a6_regk_crypto_dlkey 0x0000000a +#define a7_regk_crypto_dlkey 0x0000000e +#define regk_crypto_ext 0x00000001 +#define regk_crypto_hmac_sha1 0x00000007 +#define regk_crypto_hmac_sha256 0x00000009 +#define regk_crypto_hmac_sha384 0x0000000b +#define regk_crypto_hmac_sha512 0x0000000d +#define regk_crypto_init 0x00000000 +#define regk_crypto_key_128 0x00000000 +#define regk_crypto_key_192 0x00000001 +#define regk_crypto_key_256 0x00000002 +#define regk_crypto_null 0x00000000 +#define regk_crypto_sha1 0x00000006 +#define regk_crypto_sha256 0x00000008 +#define regk_crypto_sha384 0x0000000a +#define regk_crypto_sha512 0x0000000c + +/* DMA descriptor structures */ +struct pdma_descr_ctrl { + unsigned char short_descr : 1; + unsigned char pad1 : 1; + unsigned char eop : 1; + unsigned char intr : 1; + unsigned char short_len : 3; + unsigned char pad2 : 1; +} __packed; + +struct pdma_data_descr { + unsigned int len : 24; + unsigned int buf : 32; +} __packed; + +struct pdma_short_descr { + unsigned char data[7]; +} __packed; + +struct pdma_descr { + struct pdma_descr_ctrl ctrl; + union { + struct pdma_data_descr data; + struct pdma_short_descr shrt; + }; +}; + +struct pdma_stat_descr { + unsigned char pad1 : 1; + unsigned char pad2 : 1; + unsigned char eop : 1; + unsigned char pad3 : 5; + unsigned int len : 24; +}; + +/* Each descriptor array can hold max 64 entries */ +#define PDMA_DESCR_COUNT 64 + +#define MODULE_NAME "Artpec-6 CA" + +/* Hash modes (including HMAC variants) */ +#define ARTPEC6_CRYPTO_HASH_SHA1 1 +#define ARTPEC6_CRYPTO_HASH_SHA256 2 +#define ARTPEC6_CRYPTO_HASH_SHA384 3 +#define ARTPEC6_CRYPTO_HASH_SHA512 4 + +/* Crypto modes */ +#define ARTPEC6_CRYPTO_CIPHER_AES_ECB 1 +#define ARTPEC6_CRYPTO_CIPHER_AES_CBC 2 +#define ARTPEC6_CRYPTO_CIPHER_AES_CTR 3 +#define ARTPEC6_CRYPTO_CIPHER_AES_XTS 5 + +/* The PDMA is a DMA-engine tightly coupled with a ciphering engine. + * It operates on a descriptor array with up to 64 descriptor entries. + * The arrays must be 64 byte aligned in memory. + * + * The ciphering unit has no registers and is completely controlled by + * a 4-byte metadata that is inserted at the beginning of each dma packet. + * + * A dma packet is a sequence of descriptors terminated by setting the .eop + * field in the final descriptor of the packet. + * + * Multiple packets are used for providing context data, key data and + * the plain/ciphertext. + * + * PDMA Descriptors (Array) + * +------+------+------+~~+-------+------+---- + * | 0 | 1 | 2 |~~| 11 EOP| 12 | .... + * +--+---+--+---+----+-+~~+-------+----+-+---- + * | | | | | + * | | | | | + * __|__ +-------++-------++-------+ +----+ + * | MD | |Payload||Payload||Payload| | MD | + * +-----+ +-------++-------++-------+ +----+ + */ + +struct artpec6_crypto_bounce_buffer { + struct list_head list; + size_t length; + struct scatterlist *sg; + size_t offset; + /* buf is aligned to ARTPEC_CACHE_LINE_MAX and + * holds up to ARTPEC_CACHE_LINE_MAX bytes data. + */ + void *buf; +}; + +struct artpec6_crypto_dma_map { + dma_addr_t dma_addr; + size_t size; + enum dma_data_direction dir; +}; + +struct artpec6_crypto_dma_descriptors { + struct pdma_descr out[PDMA_DESCR_COUNT] __aligned(64); + struct pdma_descr in[PDMA_DESCR_COUNT] __aligned(64); + u32 stat[PDMA_DESCR_COUNT] __aligned(64); + struct list_head bounce_buffers; + /* Enough maps for all out/in buffers, and all three descr. arrays */ + struct artpec6_crypto_dma_map maps[PDMA_DESCR_COUNT * 2 + 2]; + dma_addr_t out_dma_addr; + dma_addr_t in_dma_addr; + dma_addr_t stat_dma_addr; + size_t out_cnt; + size_t in_cnt; + size_t map_count; +}; + +enum artpec6_crypto_variant { + ARTPEC6_CRYPTO, + ARTPEC7_CRYPTO, +}; + +struct artpec6_crypto { + void __iomem *base; + spinlock_t queue_lock; + struct list_head queue; /* waiting for pdma fifo space */ + struct list_head pending; /* submitted to pdma fifo */ + struct tasklet_struct task; + struct kmem_cache *dma_cache; + int pending_count; + struct timer_list timer; + enum artpec6_crypto_variant variant; + void *pad_buffer; /* cache-aligned block padding buffer */ + void *zero_buffer; +}; + +enum artpec6_crypto_hash_flags { + HASH_FLAG_INIT_CTX = 2, + HASH_FLAG_UPDATE = 4, + HASH_FLAG_FINALIZE = 8, + HASH_FLAG_HMAC = 16, + HASH_FLAG_UPDATE_KEY = 32, +}; + +struct artpec6_crypto_req_common { + struct list_head list; + struct list_head complete_in_progress; + struct artpec6_crypto_dma_descriptors *dma; + struct crypto_async_request *req; + void (*complete)(struct crypto_async_request *req); + gfp_t gfp_flags; +}; + +struct artpec6_hash_request_context { + char partial_buffer[SHA512_BLOCK_SIZE]; + char partial_buffer_out[SHA512_BLOCK_SIZE]; + char key_buffer[SHA512_BLOCK_SIZE]; + char pad_buffer[SHA512_BLOCK_SIZE + 32]; + unsigned char digeststate[SHA512_DIGEST_SIZE]; + size_t partial_bytes; + u64 digcnt; + u32 key_md; + u32 hash_md; + enum artpec6_crypto_hash_flags hash_flags; + struct artpec6_crypto_req_common common; +}; + +struct artpec6_hash_export_state { + char partial_buffer[SHA512_BLOCK_SIZE]; + unsigned char digeststate[SHA512_DIGEST_SIZE]; + size_t partial_bytes; + u64 digcnt; + int oper; + unsigned int hash_flags; +}; + +struct artpec6_hashalg_context { + char hmac_key[SHA512_BLOCK_SIZE]; + size_t hmac_key_length; + struct crypto_shash *child_hash; +}; + +struct artpec6_crypto_request_context { + u32 cipher_md; + bool decrypt; + struct artpec6_crypto_req_common common; +}; + +struct artpec6_cryptotfm_context { + unsigned char aes_key[2*AES_MAX_KEY_SIZE]; + size_t key_length; + u32 key_md; + int crypto_type; + struct crypto_skcipher *fallback; +}; + +struct artpec6_crypto_aead_hw_ctx { + __be64 aad_length_bits; + __be64 text_length_bits; + __u8 J0[AES_BLOCK_SIZE]; +}; + +struct artpec6_crypto_aead_req_ctx { + struct artpec6_crypto_aead_hw_ctx hw_ctx; + u32 cipher_md; + bool decrypt; + struct artpec6_crypto_req_common common; + __u8 decryption_tag[AES_BLOCK_SIZE] ____cacheline_aligned; +}; + +/* The crypto framework makes it hard to avoid this global. */ +static struct device *artpec6_crypto_dev; + +#ifdef CONFIG_FAULT_INJECTION +static DECLARE_FAULT_ATTR(artpec6_crypto_fail_status_read); +static DECLARE_FAULT_ATTR(artpec6_crypto_fail_dma_array_full); +#endif + +enum { + ARTPEC6_CRYPTO_PREPARE_HASH_NO_START, + ARTPEC6_CRYPTO_PREPARE_HASH_START, +}; + +static int artpec6_crypto_prepare_aead(struct aead_request *areq); +static int artpec6_crypto_prepare_crypto(struct skcipher_request *areq); +static int artpec6_crypto_prepare_hash(struct ahash_request *areq); + +static void +artpec6_crypto_complete_crypto(struct crypto_async_request *req); +static void +artpec6_crypto_complete_cbc_encrypt(struct crypto_async_request *req); +static void +artpec6_crypto_complete_cbc_decrypt(struct crypto_async_request *req); +static void +artpec6_crypto_complete_aead(struct crypto_async_request *req); +static void +artpec6_crypto_complete_hash(struct crypto_async_request *req); + +static int +artpec6_crypto_common_destroy(struct artpec6_crypto_req_common *common); + +static void +artpec6_crypto_start_dma(struct artpec6_crypto_req_common *common); + +struct artpec6_crypto_walk { + struct scatterlist *sg; + size_t offset; +}; + +static void artpec6_crypto_walk_init(struct artpec6_crypto_walk *awalk, + struct scatterlist *sg) +{ + awalk->sg = sg; + awalk->offset = 0; +} + +static size_t artpec6_crypto_walk_advance(struct artpec6_crypto_walk *awalk, + size_t nbytes) +{ + while (nbytes && awalk->sg) { + size_t piece; + + WARN_ON(awalk->offset > awalk->sg->length); + + piece = min(nbytes, (size_t)awalk->sg->length - awalk->offset); + nbytes -= piece; + awalk->offset += piece; + if (awalk->offset == awalk->sg->length) { + awalk->sg = sg_next(awalk->sg); + awalk->offset = 0; + } + + } + + return nbytes; +} + +static size_t +artpec6_crypto_walk_chunklen(const struct artpec6_crypto_walk *awalk) +{ + WARN_ON(awalk->sg->length == awalk->offset); + + return awalk->sg->length - awalk->offset; +} + +static dma_addr_t +artpec6_crypto_walk_chunk_phys(const struct artpec6_crypto_walk *awalk) +{ + return sg_phys(awalk->sg) + awalk->offset; +} + +static void +artpec6_crypto_copy_bounce_buffers(struct artpec6_crypto_req_common *common) +{ + struct artpec6_crypto_dma_descriptors *dma = common->dma; + struct artpec6_crypto_bounce_buffer *b; + struct artpec6_crypto_bounce_buffer *next; + + list_for_each_entry_safe(b, next, &dma->bounce_buffers, list) { + pr_debug("bounce entry %p: %zu bytes @ %zu from %p\n", + b, b->length, b->offset, b->buf); + sg_pcopy_from_buffer(b->sg, + 1, + b->buf, + b->length, + b->offset); + + list_del(&b->list); + kfree(b); + } +} + +static inline bool artpec6_crypto_busy(void) +{ + struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); + int fifo_count = ac->pending_count; + + return fifo_count > 6; +} + +static int artpec6_crypto_submit(struct artpec6_crypto_req_common *req) +{ + struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); + int ret = -EBUSY; + + spin_lock_bh(&ac->queue_lock); + + if (!artpec6_crypto_busy()) { + list_add_tail(&req->list, &ac->pending); + artpec6_crypto_start_dma(req); + ret = -EINPROGRESS; + } else if (req->req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG) { + list_add_tail(&req->list, &ac->queue); + } else { + artpec6_crypto_common_destroy(req); + } + + spin_unlock_bh(&ac->queue_lock); + + return ret; +} + +static void artpec6_crypto_start_dma(struct artpec6_crypto_req_common *common) +{ + struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); + enum artpec6_crypto_variant variant = ac->variant; + void __iomem *base = ac->base; + struct artpec6_crypto_dma_descriptors *dma = common->dma; + u32 ind, statd, outd; + + /* Make descriptor content visible to the DMA before starting it. */ + wmb(); + + ind = FIELD_PREP(PDMA_IN_DESCRQ_PUSH_LEN, dma->in_cnt - 1) | + FIELD_PREP(PDMA_IN_DESCRQ_PUSH_ADDR, dma->in_dma_addr >> 6); + + statd = FIELD_PREP(PDMA_IN_STATQ_PUSH_LEN, dma->in_cnt - 1) | + FIELD_PREP(PDMA_IN_STATQ_PUSH_ADDR, dma->stat_dma_addr >> 6); + + outd = FIELD_PREP(PDMA_OUT_DESCRQ_PUSH_LEN, dma->out_cnt - 1) | + FIELD_PREP(PDMA_OUT_DESCRQ_PUSH_ADDR, dma->out_dma_addr >> 6); + + if (variant == ARTPEC6_CRYPTO) { + writel_relaxed(ind, base + A6_PDMA_IN_DESCRQ_PUSH); + writel_relaxed(statd, base + A6_PDMA_IN_STATQ_PUSH); + writel_relaxed(PDMA_IN_CMD_START, base + A6_PDMA_IN_CMD); + } else { + writel_relaxed(ind, base + A7_PDMA_IN_DESCRQ_PUSH); + writel_relaxed(statd, base + A7_PDMA_IN_STATQ_PUSH); + writel_relaxed(PDMA_IN_CMD_START, base + A7_PDMA_IN_CMD); + } + + writel_relaxed(outd, base + PDMA_OUT_DESCRQ_PUSH); + writel_relaxed(PDMA_OUT_CMD_START, base + PDMA_OUT_CMD); + + ac->pending_count++; +} + +static void +artpec6_crypto_init_dma_operation(struct artpec6_crypto_req_common *common) +{ + struct artpec6_crypto_dma_descriptors *dma = common->dma; + + dma->out_cnt = 0; + dma->in_cnt = 0; + dma->map_count = 0; + INIT_LIST_HEAD(&dma->bounce_buffers); +} + +static bool fault_inject_dma_descr(void) +{ +#ifdef CONFIG_FAULT_INJECTION + return should_fail(&artpec6_crypto_fail_dma_array_full, 1); +#else + return false; +#endif +} + +/** artpec6_crypto_setup_out_descr_phys - Setup an out channel with a + * physical address + * + * @addr: The physical address of the data buffer + * @len: The length of the data buffer + * @eop: True if this is the last buffer in the packet + * + * @return 0 on success or -ENOSPC if there are no more descriptors available + */ +static int +artpec6_crypto_setup_out_descr_phys(struct artpec6_crypto_req_common *common, + dma_addr_t addr, size_t len, bool eop) +{ + struct artpec6_crypto_dma_descriptors *dma = common->dma; + struct pdma_descr *d; + + if (dma->out_cnt >= PDMA_DESCR_COUNT || + fault_inject_dma_descr()) { + pr_err("No free OUT DMA descriptors available!\n"); + return -ENOSPC; + } + + d = &dma->out[dma->out_cnt++]; + memset(d, 0, sizeof(*d)); + + d->ctrl.short_descr = 0; + d->ctrl.eop = eop; + d->data.len = len; + d->data.buf = addr; + return 0; +} + +/** artpec6_crypto_setup_out_descr_short - Setup a short out descriptor + * + * @dst: The virtual address of the data + * @len: The length of the data, must be between 1 to 7 bytes + * @eop: True if this is the last buffer in the packet + * + * @return 0 on success + * -ENOSPC if no more descriptors are available + * -EINVAL if the data length exceeds 7 bytes + */ +static int +artpec6_crypto_setup_out_descr_short(struct artpec6_crypto_req_common *common, + void *dst, unsigned int len, bool eop) +{ + struct artpec6_crypto_dma_descriptors *dma = common->dma; + struct pdma_descr *d; + + if (dma->out_cnt >= PDMA_DESCR_COUNT || + fault_inject_dma_descr()) { + pr_err("No free OUT DMA descriptors available!\n"); + return -ENOSPC; + } else if (len > 7 || len < 1) { + return -EINVAL; + } + d = &dma->out[dma->out_cnt++]; + memset(d, 0, sizeof(*d)); + + d->ctrl.short_descr = 1; + d->ctrl.short_len = len; + d->ctrl.eop = eop; + memcpy(d->shrt.data, dst, len); + return 0; +} + +static int artpec6_crypto_dma_map_page(struct artpec6_crypto_req_common *common, + struct page *page, size_t offset, + size_t size, + enum dma_data_direction dir, + dma_addr_t *dma_addr_out) +{ + struct artpec6_crypto_dma_descriptors *dma = common->dma; + struct device *dev = artpec6_crypto_dev; + struct artpec6_crypto_dma_map *map; + dma_addr_t dma_addr; + + *dma_addr_out = 0; + + if (dma->map_count >= ARRAY_SIZE(dma->maps)) + return -ENOMEM; + + dma_addr = dma_map_page(dev, page, offset, size, dir); + if (dma_mapping_error(dev, dma_addr)) + return -ENOMEM; + + map = &dma->maps[dma->map_count++]; + map->size = size; + map->dma_addr = dma_addr; + map->dir = dir; + + *dma_addr_out = dma_addr; + + return 0; +} + +static int +artpec6_crypto_dma_map_single(struct artpec6_crypto_req_common *common, + void *ptr, size_t size, + enum dma_data_direction dir, + dma_addr_t *dma_addr_out) +{ + struct page *page = virt_to_page(ptr); + size_t offset = (uintptr_t)ptr & ~PAGE_MASK; + + return artpec6_crypto_dma_map_page(common, page, offset, size, dir, + dma_addr_out); +} + +static int +artpec6_crypto_dma_map_descs(struct artpec6_crypto_req_common *common) +{ + struct artpec6_crypto_dma_descriptors *dma = common->dma; + int ret; + + ret = artpec6_crypto_dma_map_single(common, dma->in, + sizeof(dma->in[0]) * dma->in_cnt, + DMA_TO_DEVICE, &dma->in_dma_addr); + if (ret) + return ret; + + ret = artpec6_crypto_dma_map_single(common, dma->out, + sizeof(dma->out[0]) * dma->out_cnt, + DMA_TO_DEVICE, &dma->out_dma_addr); + if (ret) + return ret; + + /* We only read one stat descriptor */ + dma->stat[dma->in_cnt - 1] = 0; + + /* + * DMA_BIDIRECTIONAL since we need our zeroing of the stat descriptor + * to be written. + */ + return artpec6_crypto_dma_map_single(common, + dma->stat + dma->in_cnt - 1, + sizeof(dma->stat[0]), + DMA_BIDIRECTIONAL, + &dma->stat_dma_addr); +} + +static void +artpec6_crypto_dma_unmap_all(struct artpec6_crypto_req_common *common) +{ + struct artpec6_crypto_dma_descriptors *dma = common->dma; + struct device *dev = artpec6_crypto_dev; + int i; + + for (i = 0; i < dma->map_count; i++) { + struct artpec6_crypto_dma_map *map = &dma->maps[i]; + + dma_unmap_page(dev, map->dma_addr, map->size, map->dir); + } + + dma->map_count = 0; +} + +/** artpec6_crypto_setup_out_descr - Setup an out descriptor + * + * @dst: The virtual address of the data + * @len: The length of the data + * @eop: True if this is the last buffer in the packet + * @use_short: If this is true and the data length is 7 bytes or less then + * a short descriptor will be used + * + * @return 0 on success + * Any errors from artpec6_crypto_setup_out_descr_short() or + * setup_out_descr_phys() + */ +static int +artpec6_crypto_setup_out_descr(struct artpec6_crypto_req_common *common, + void *dst, unsigned int len, bool eop, + bool use_short) +{ + if (use_short && len < 7) { + return artpec6_crypto_setup_out_descr_short(common, dst, len, + eop); + } else { + int ret; + dma_addr_t dma_addr; + + ret = artpec6_crypto_dma_map_single(common, dst, len, + DMA_TO_DEVICE, + &dma_addr); + if (ret) + return ret; + + return artpec6_crypto_setup_out_descr_phys(common, dma_addr, + len, eop); + } +} + +/** artpec6_crypto_setup_in_descr_phys - Setup an in channel with a + * physical address + * + * @addr: The physical address of the data buffer + * @len: The length of the data buffer + * @intr: True if an interrupt should be fired after HW processing of this + * descriptor + * + */ +static int +artpec6_crypto_setup_in_descr_phys(struct artpec6_crypto_req_common *common, + dma_addr_t addr, unsigned int len, bool intr) +{ + struct artpec6_crypto_dma_descriptors *dma = common->dma; + struct pdma_descr *d; + + if (dma->in_cnt >= PDMA_DESCR_COUNT || + fault_inject_dma_descr()) { + pr_err("No free IN DMA descriptors available!\n"); + return -ENOSPC; + } + d = &dma->in[dma->in_cnt++]; + memset(d, 0, sizeof(*d)); + + d->ctrl.intr = intr; + d->data.len = len; + d->data.buf = addr; + return 0; +} + +/** artpec6_crypto_setup_in_descr - Setup an in channel descriptor + * + * @buffer: The virtual address to of the data buffer + * @len: The length of the data buffer + * @last: If this is the last data buffer in the request (i.e. an interrupt + * is needed + * + * Short descriptors are not used for the in channel + */ +static int +artpec6_crypto_setup_in_descr(struct artpec6_crypto_req_common *common, + void *buffer, unsigned int len, bool last) +{ + dma_addr_t dma_addr; + int ret; + + ret = artpec6_crypto_dma_map_single(common, buffer, len, + DMA_FROM_DEVICE, &dma_addr); + if (ret) + return ret; + + return artpec6_crypto_setup_in_descr_phys(common, dma_addr, len, last); +} + +static struct artpec6_crypto_bounce_buffer * +artpec6_crypto_alloc_bounce(gfp_t flags) +{ + void *base; + size_t alloc_size = sizeof(struct artpec6_crypto_bounce_buffer) + + 2 * ARTPEC_CACHE_LINE_MAX; + struct artpec6_crypto_bounce_buffer *bbuf = kzalloc(alloc_size, flags); + + if (!bbuf) + return NULL; + + base = bbuf + 1; + bbuf->buf = PTR_ALIGN(base, ARTPEC_CACHE_LINE_MAX); + return bbuf; +} + +static int setup_bounce_buffer_in(struct artpec6_crypto_req_common *common, + struct artpec6_crypto_walk *walk, size_t size) +{ + struct artpec6_crypto_bounce_buffer *bbuf; + int ret; + + bbuf = artpec6_crypto_alloc_bounce(common->gfp_flags); + if (!bbuf) + return -ENOMEM; + + bbuf->length = size; + bbuf->sg = walk->sg; + bbuf->offset = walk->offset; + + ret = artpec6_crypto_setup_in_descr(common, bbuf->buf, size, false); + if (ret) { + kfree(bbuf); + return ret; + } + + pr_debug("BOUNCE %zu offset %zu\n", size, walk->offset); + list_add_tail(&bbuf->list, &common->dma->bounce_buffers); + return 0; +} + +static int +artpec6_crypto_setup_sg_descrs_in(struct artpec6_crypto_req_common *common, + struct artpec6_crypto_walk *walk, + size_t count) +{ + size_t chunk; + int ret; + dma_addr_t addr; + + while (walk->sg && count) { + chunk = min(count, artpec6_crypto_walk_chunklen(walk)); + addr = artpec6_crypto_walk_chunk_phys(walk); + + /* When destination buffers are not aligned to the cache line + * size we need bounce buffers. The DMA-API requires that the + * entire line is owned by the DMA buffer and this holds also + * for the case when coherent DMA is used. + */ + if (!IS_ALIGNED(addr, ARTPEC_CACHE_LINE_MAX)) { + chunk = min_t(dma_addr_t, chunk, + ALIGN(addr, ARTPEC_CACHE_LINE_MAX) - + addr); + + pr_debug("CHUNK-b %pad:%zu\n", &addr, chunk); + ret = setup_bounce_buffer_in(common, walk, chunk); + } else if (chunk < ARTPEC_CACHE_LINE_MAX) { + pr_debug("CHUNK-b %pad:%zu\n", &addr, chunk); + ret = setup_bounce_buffer_in(common, walk, chunk); + } else { + dma_addr_t dma_addr; + + chunk = chunk & ~(ARTPEC_CACHE_LINE_MAX-1); + + pr_debug("CHUNK %pad:%zu\n", &addr, chunk); + + ret = artpec6_crypto_dma_map_page(common, + sg_page(walk->sg), + walk->sg->offset + + walk->offset, + chunk, + DMA_FROM_DEVICE, + &dma_addr); + if (ret) + return ret; + + ret = artpec6_crypto_setup_in_descr_phys(common, + dma_addr, + chunk, false); + } + + if (ret) + return ret; + + count = count - chunk; + artpec6_crypto_walk_advance(walk, chunk); + } + + if (count) + pr_err("EOL unexpected %zu bytes left\n", count); + + return count ? -EINVAL : 0; +} + +static int +artpec6_crypto_setup_sg_descrs_out(struct artpec6_crypto_req_common *common, + struct artpec6_crypto_walk *walk, + size_t count) +{ + size_t chunk; + int ret; + dma_addr_t addr; + + while (walk->sg && count) { + chunk = min(count, artpec6_crypto_walk_chunklen(walk)); + addr = artpec6_crypto_walk_chunk_phys(walk); + + pr_debug("OUT-CHUNK %pad:%zu\n", &addr, chunk); + + if (addr & 3) { + char buf[3]; + + chunk = min_t(size_t, chunk, (4-(addr&3))); + + sg_pcopy_to_buffer(walk->sg, 1, buf, chunk, + walk->offset); + + ret = artpec6_crypto_setup_out_descr_short(common, buf, + chunk, + false); + } else { + dma_addr_t dma_addr; + + ret = artpec6_crypto_dma_map_page(common, + sg_page(walk->sg), + walk->sg->offset + + walk->offset, + chunk, + DMA_TO_DEVICE, + &dma_addr); + if (ret) + return ret; + + ret = artpec6_crypto_setup_out_descr_phys(common, + dma_addr, + chunk, false); + } + + if (ret) + return ret; + + count = count - chunk; + artpec6_crypto_walk_advance(walk, chunk); + } + + if (count) + pr_err("EOL unexpected %zu bytes left\n", count); + + return count ? -EINVAL : 0; +} + + +/** artpec6_crypto_terminate_out_descrs - Set the EOP on the last out descriptor + * + * If the out descriptor list is non-empty, then the eop flag on the + * last used out descriptor will be set. + * + * @return 0 on success + * -EINVAL if the out descriptor is empty or has overflown + */ +static int +artpec6_crypto_terminate_out_descrs(struct artpec6_crypto_req_common *common) +{ + struct artpec6_crypto_dma_descriptors *dma = common->dma; + struct pdma_descr *d; + + if (!dma->out_cnt || dma->out_cnt > PDMA_DESCR_COUNT) { + pr_err("%s: OUT descriptor list is %s\n", + MODULE_NAME, dma->out_cnt ? "empty" : "full"); + return -EINVAL; + + } + + d = &dma->out[dma->out_cnt-1]; + d->ctrl.eop = 1; + + return 0; +} + +/** artpec6_crypto_terminate_in_descrs - Set the interrupt flag on the last + * in descriptor + * + * See artpec6_crypto_terminate_out_descrs() for return values + */ +static int +artpec6_crypto_terminate_in_descrs(struct artpec6_crypto_req_common *common) +{ + struct artpec6_crypto_dma_descriptors *dma = common->dma; + struct pdma_descr *d; + + if (!dma->in_cnt || dma->in_cnt > PDMA_DESCR_COUNT) { + pr_err("%s: IN descriptor list is %s\n", + MODULE_NAME, dma->in_cnt ? "empty" : "full"); + return -EINVAL; + } + + d = &dma->in[dma->in_cnt-1]; + d->ctrl.intr = 1; + return 0; +} + +/** create_hash_pad - Create a Secure Hash conformant pad + * + * @dst: The destination buffer to write the pad. Must be at least 64 bytes + * @dgstlen: The total length of the hash digest in bytes + * @bitcount: The total length of the digest in bits + * + * @return The total number of padding bytes written to @dst + */ +static size_t +create_hash_pad(int oper, unsigned char *dst, u64 dgstlen, u64 bitcount) +{ + unsigned int mod, target, diff, pad_bytes, size_bytes; + __be64 bits = __cpu_to_be64(bitcount); + + switch (oper) { + case regk_crypto_sha1: + case regk_crypto_sha256: + case regk_crypto_hmac_sha1: + case regk_crypto_hmac_sha256: + target = 448 / 8; + mod = 512 / 8; + size_bytes = 8; + break; + default: + target = 896 / 8; + mod = 1024 / 8; + size_bytes = 16; + break; + } + + target -= 1; + diff = dgstlen & (mod - 1); + pad_bytes = diff > target ? target + mod - diff : target - diff; + + memset(dst + 1, 0, pad_bytes); + dst[0] = 0x80; + + if (size_bytes == 16) { + memset(dst + 1 + pad_bytes, 0, 8); + memcpy(dst + 1 + pad_bytes + 8, &bits, 8); + } else { + memcpy(dst + 1 + pad_bytes, &bits, 8); + } + + return pad_bytes + size_bytes + 1; +} + +static int artpec6_crypto_common_init(struct artpec6_crypto_req_common *common, + struct crypto_async_request *parent, + void (*complete)(struct crypto_async_request *req), + struct scatterlist *dstsg, unsigned int nbytes) +{ + gfp_t flags; + struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); + + flags = (parent->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + + common->gfp_flags = flags; + common->dma = kmem_cache_alloc(ac->dma_cache, flags); + if (!common->dma) + return -ENOMEM; + + common->req = parent; + common->complete = complete; + return 0; +} + +static void +artpec6_crypto_bounce_destroy(struct artpec6_crypto_dma_descriptors *dma) +{ + struct artpec6_crypto_bounce_buffer *b; + struct artpec6_crypto_bounce_buffer *next; + + list_for_each_entry_safe(b, next, &dma->bounce_buffers, list) { + kfree(b); + } +} + +static int +artpec6_crypto_common_destroy(struct artpec6_crypto_req_common *common) +{ + struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); + + artpec6_crypto_dma_unmap_all(common); + artpec6_crypto_bounce_destroy(common->dma); + kmem_cache_free(ac->dma_cache, common->dma); + common->dma = NULL; + return 0; +} + +/* + * Ciphering functions. + */ +static int artpec6_crypto_encrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); + struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher); + struct artpec6_crypto_request_context *req_ctx = NULL; + void (*complete)(struct crypto_async_request *req); + int ret; + + req_ctx = skcipher_request_ctx(req); + + switch (ctx->crypto_type) { + case ARTPEC6_CRYPTO_CIPHER_AES_CBC: + case ARTPEC6_CRYPTO_CIPHER_AES_ECB: + case ARTPEC6_CRYPTO_CIPHER_AES_XTS: + req_ctx->decrypt = 0; + break; + default: + break; + } + + switch (ctx->crypto_type) { + case ARTPEC6_CRYPTO_CIPHER_AES_CBC: + complete = artpec6_crypto_complete_cbc_encrypt; + break; + default: + complete = artpec6_crypto_complete_crypto; + break; + } + + ret = artpec6_crypto_common_init(&req_ctx->common, + &req->base, + complete, + req->dst, req->cryptlen); + if (ret) + return ret; + + ret = artpec6_crypto_prepare_crypto(req); + if (ret) { + artpec6_crypto_common_destroy(&req_ctx->common); + return ret; + } + + return artpec6_crypto_submit(&req_ctx->common); +} + +static int artpec6_crypto_decrypt(struct skcipher_request *req) +{ + int ret; + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); + struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher); + struct artpec6_crypto_request_context *req_ctx = NULL; + void (*complete)(struct crypto_async_request *req); + + req_ctx = skcipher_request_ctx(req); + + switch (ctx->crypto_type) { + case ARTPEC6_CRYPTO_CIPHER_AES_CBC: + case ARTPEC6_CRYPTO_CIPHER_AES_ECB: + case ARTPEC6_CRYPTO_CIPHER_AES_XTS: + req_ctx->decrypt = 1; + break; + default: + break; + } + + + switch (ctx->crypto_type) { + case ARTPEC6_CRYPTO_CIPHER_AES_CBC: + complete = artpec6_crypto_complete_cbc_decrypt; + break; + default: + complete = artpec6_crypto_complete_crypto; + break; + } + + ret = artpec6_crypto_common_init(&req_ctx->common, &req->base, + complete, + req->dst, req->cryptlen); + if (ret) + return ret; + + ret = artpec6_crypto_prepare_crypto(req); + if (ret) { + artpec6_crypto_common_destroy(&req_ctx->common); + return ret; + } + + return artpec6_crypto_submit(&req_ctx->common); +} + +static int +artpec6_crypto_ctr_crypt(struct skcipher_request *req, bool encrypt) +{ + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); + struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher); + size_t iv_len = crypto_skcipher_ivsize(cipher); + unsigned int counter = be32_to_cpup((__be32 *) + (req->iv + iv_len - 4)); + unsigned int nblks = ALIGN(req->cryptlen, AES_BLOCK_SIZE) / + AES_BLOCK_SIZE; + + /* + * The hardware uses only the last 32-bits as the counter while the + * kernel tests (aes_ctr_enc_tv_template[4] for example) expect that + * the whole IV is a counter. So fallback if the counter is going to + * overlow. + */ + if (counter + nblks < counter) { + int ret; + + pr_debug("counter %x will overflow (nblks %u), falling back\n", + counter, counter + nblks); + + ret = crypto_skcipher_setkey(ctx->fallback, ctx->aes_key, + ctx->key_length); + if (ret) + return ret; + + { + SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback); + + skcipher_request_set_tfm(subreq, ctx->fallback); + skcipher_request_set_callback(subreq, req->base.flags, + NULL, NULL); + skcipher_request_set_crypt(subreq, req->src, req->dst, + req->cryptlen, req->iv); + ret = encrypt ? crypto_skcipher_encrypt(subreq) + : crypto_skcipher_decrypt(subreq); + skcipher_request_zero(subreq); + } + return ret; + } + + return encrypt ? artpec6_crypto_encrypt(req) + : artpec6_crypto_decrypt(req); +} + +static int artpec6_crypto_ctr_encrypt(struct skcipher_request *req) +{ + return artpec6_crypto_ctr_crypt(req, true); +} + +static int artpec6_crypto_ctr_decrypt(struct skcipher_request *req) +{ + return artpec6_crypto_ctr_crypt(req, false); +} + +/* + * AEAD functions + */ +static int artpec6_crypto_aead_init(struct crypto_aead *tfm) +{ + struct artpec6_cryptotfm_context *tfm_ctx = crypto_aead_ctx(tfm); + + memset(tfm_ctx, 0, sizeof(*tfm_ctx)); + + crypto_aead_set_reqsize(tfm, + sizeof(struct artpec6_crypto_aead_req_ctx)); + + return 0; +} + +static int artpec6_crypto_aead_set_key(struct crypto_aead *tfm, const u8 *key, + unsigned int len) +{ + struct artpec6_cryptotfm_context *ctx = crypto_tfm_ctx(&tfm->base); + + if (len != 16 && len != 24 && len != 32) { + crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + ctx->key_length = len; + + memcpy(ctx->aes_key, key, len); + return 0; +} + +static int artpec6_crypto_aead_encrypt(struct aead_request *req) +{ + int ret; + struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(req); + + req_ctx->decrypt = false; + ret = artpec6_crypto_common_init(&req_ctx->common, &req->base, + artpec6_crypto_complete_aead, + NULL, 0); + if (ret) + return ret; + + ret = artpec6_crypto_prepare_aead(req); + if (ret) { + artpec6_crypto_common_destroy(&req_ctx->common); + return ret; + } + + return artpec6_crypto_submit(&req_ctx->common); +} + +static int artpec6_crypto_aead_decrypt(struct aead_request *req) +{ + int ret; + struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(req); + + req_ctx->decrypt = true; + if (req->cryptlen < AES_BLOCK_SIZE) + return -EINVAL; + + ret = artpec6_crypto_common_init(&req_ctx->common, + &req->base, + artpec6_crypto_complete_aead, + NULL, 0); + if (ret) + return ret; + + ret = artpec6_crypto_prepare_aead(req); + if (ret) { + artpec6_crypto_common_destroy(&req_ctx->common); + return ret; + } + + return artpec6_crypto_submit(&req_ctx->common); +} + +static int artpec6_crypto_prepare_hash(struct ahash_request *areq) +{ + struct artpec6_hashalg_context *ctx = crypto_tfm_ctx(areq->base.tfm); + struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(areq); + size_t digestsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(areq)); + size_t contextsize = digestsize == SHA384_DIGEST_SIZE ? + SHA512_DIGEST_SIZE : digestsize; + size_t blocksize = crypto_tfm_alg_blocksize( + crypto_ahash_tfm(crypto_ahash_reqtfm(areq))); + struct artpec6_crypto_req_common *common = &req_ctx->common; + struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); + enum artpec6_crypto_variant variant = ac->variant; + u32 sel_ctx; + bool ext_ctx = false; + bool run_hw = false; + int error = 0; + + artpec6_crypto_init_dma_operation(common); + + /* Upload HMAC key, must be first the first packet */ + if (req_ctx->hash_flags & HASH_FLAG_HMAC) { + if (variant == ARTPEC6_CRYPTO) { + req_ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER, + a6_regk_crypto_dlkey); + } else { + req_ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER, + a7_regk_crypto_dlkey); + } + + /* Copy and pad up the key */ + memcpy(req_ctx->key_buffer, ctx->hmac_key, + ctx->hmac_key_length); + memset(req_ctx->key_buffer + ctx->hmac_key_length, 0, + blocksize - ctx->hmac_key_length); + + error = artpec6_crypto_setup_out_descr(common, + (void *)&req_ctx->key_md, + sizeof(req_ctx->key_md), false, false); + if (error) + return error; + + error = artpec6_crypto_setup_out_descr(common, + req_ctx->key_buffer, blocksize, + true, false); + if (error) + return error; + } + + if (!(req_ctx->hash_flags & HASH_FLAG_INIT_CTX)) { + /* Restore context */ + sel_ctx = regk_crypto_ext; + ext_ctx = true; + } else { + sel_ctx = regk_crypto_init; + } + + if (variant == ARTPEC6_CRYPTO) { + req_ctx->hash_md &= ~A6_CRY_MD_HASH_SEL_CTX; + req_ctx->hash_md |= FIELD_PREP(A6_CRY_MD_HASH_SEL_CTX, sel_ctx); + + /* If this is the final round, set the final flag */ + if (req_ctx->hash_flags & HASH_FLAG_FINALIZE) + req_ctx->hash_md |= A6_CRY_MD_HASH_HMAC_FIN; + } else { + req_ctx->hash_md &= ~A7_CRY_MD_HASH_SEL_CTX; + req_ctx->hash_md |= FIELD_PREP(A7_CRY_MD_HASH_SEL_CTX, sel_ctx); + + /* If this is the final round, set the final flag */ + if (req_ctx->hash_flags & HASH_FLAG_FINALIZE) + req_ctx->hash_md |= A7_CRY_MD_HASH_HMAC_FIN; + } + + /* Setup up metadata descriptors */ + error = artpec6_crypto_setup_out_descr(common, + (void *)&req_ctx->hash_md, + sizeof(req_ctx->hash_md), false, false); + if (error) + return error; + + error = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false); + if (error) + return error; + + if (ext_ctx) { + error = artpec6_crypto_setup_out_descr(common, + req_ctx->digeststate, + contextsize, false, false); + + if (error) + return error; + } + + if (req_ctx->hash_flags & HASH_FLAG_UPDATE) { + size_t done_bytes = 0; + size_t total_bytes = areq->nbytes + req_ctx->partial_bytes; + size_t ready_bytes = round_down(total_bytes, blocksize); + struct artpec6_crypto_walk walk; + + run_hw = ready_bytes > 0; + if (req_ctx->partial_bytes && ready_bytes) { + /* We have a partial buffer and will at least some bytes + * to the HW. Empty this partial buffer before tackling + * the SG lists + */ + memcpy(req_ctx->partial_buffer_out, + req_ctx->partial_buffer, + req_ctx->partial_bytes); + + error = artpec6_crypto_setup_out_descr(common, + req_ctx->partial_buffer_out, + req_ctx->partial_bytes, + false, true); + if (error) + return error; + + /* Reset partial buffer */ + done_bytes += req_ctx->partial_bytes; + req_ctx->partial_bytes = 0; + } + + artpec6_crypto_walk_init(&walk, areq->src); + + error = artpec6_crypto_setup_sg_descrs_out(common, &walk, + ready_bytes - + done_bytes); + if (error) + return error; + + if (walk.sg) { + size_t sg_skip = ready_bytes - done_bytes; + size_t sg_rem = areq->nbytes - sg_skip; + + sg_pcopy_to_buffer(areq->src, sg_nents(areq->src), + req_ctx->partial_buffer + + req_ctx->partial_bytes, + sg_rem, sg_skip); + + req_ctx->partial_bytes += sg_rem; + } + + req_ctx->digcnt += ready_bytes; + req_ctx->hash_flags &= ~(HASH_FLAG_UPDATE); + } + + /* Finalize */ + if (req_ctx->hash_flags & HASH_FLAG_FINALIZE) { + bool needtrim = contextsize != digestsize; + size_t hash_pad_len; + u64 digest_bits; + u32 oper; + + if (variant == ARTPEC6_CRYPTO) + oper = FIELD_GET(A6_CRY_MD_OPER, req_ctx->hash_md); + else + oper = FIELD_GET(A7_CRY_MD_OPER, req_ctx->hash_md); + + /* Write out the partial buffer if present */ + if (req_ctx->partial_bytes) { + memcpy(req_ctx->partial_buffer_out, + req_ctx->partial_buffer, + req_ctx->partial_bytes); + error = artpec6_crypto_setup_out_descr(common, + req_ctx->partial_buffer_out, + req_ctx->partial_bytes, + false, true); + if (error) + return error; + + req_ctx->digcnt += req_ctx->partial_bytes; + req_ctx->partial_bytes = 0; + } + + if (req_ctx->hash_flags & HASH_FLAG_HMAC) + digest_bits = 8 * (req_ctx->digcnt + blocksize); + else + digest_bits = 8 * req_ctx->digcnt; + + /* Add the hash pad */ + hash_pad_len = create_hash_pad(oper, req_ctx->pad_buffer, + req_ctx->digcnt, digest_bits); + error = artpec6_crypto_setup_out_descr(common, + req_ctx->pad_buffer, + hash_pad_len, false, + true); + req_ctx->digcnt = 0; + + if (error) + return error; + + /* Descriptor for the final result */ + error = artpec6_crypto_setup_in_descr(common, areq->result, + digestsize, + !needtrim); + if (error) + return error; + + if (needtrim) { + /* Discard the extra context bytes for SHA-384 */ + error = artpec6_crypto_setup_in_descr(common, + req_ctx->partial_buffer, + digestsize - contextsize, true); + if (error) + return error; + } + + } else { /* This is not the final operation for this request */ + if (!run_hw) + return ARTPEC6_CRYPTO_PREPARE_HASH_NO_START; + + /* Save the result to the context */ + error = artpec6_crypto_setup_in_descr(common, + req_ctx->digeststate, + contextsize, false); + if (error) + return error; + /* fall through */ + } + + req_ctx->hash_flags &= ~(HASH_FLAG_INIT_CTX | HASH_FLAG_UPDATE | + HASH_FLAG_FINALIZE); + + error = artpec6_crypto_terminate_in_descrs(common); + if (error) + return error; + + error = artpec6_crypto_terminate_out_descrs(common); + if (error) + return error; + + error = artpec6_crypto_dma_map_descs(common); + if (error) + return error; + + return ARTPEC6_CRYPTO_PREPARE_HASH_START; +} + + +static int artpec6_crypto_aes_ecb_init(struct crypto_skcipher *tfm) +{ + struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); + + tfm->reqsize = sizeof(struct artpec6_crypto_request_context); + ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_ECB; + + return 0; +} + +static int artpec6_crypto_aes_ctr_init(struct crypto_skcipher *tfm) +{ + struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); + + ctx->fallback = crypto_alloc_skcipher(crypto_tfm_alg_name(&tfm->base), + 0, + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->fallback)) + return PTR_ERR(ctx->fallback); + + tfm->reqsize = sizeof(struct artpec6_crypto_request_context); + ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_CTR; + + return 0; +} + +static int artpec6_crypto_aes_cbc_init(struct crypto_skcipher *tfm) +{ + struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); + + tfm->reqsize = sizeof(struct artpec6_crypto_request_context); + ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_CBC; + + return 0; +} + +static int artpec6_crypto_aes_xts_init(struct crypto_skcipher *tfm) +{ + struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); + + tfm->reqsize = sizeof(struct artpec6_crypto_request_context); + ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_XTS; + + return 0; +} + +static void artpec6_crypto_aes_exit(struct crypto_skcipher *tfm) +{ + struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); + + memset(ctx, 0, sizeof(*ctx)); +} + +static void artpec6_crypto_aes_ctr_exit(struct crypto_skcipher *tfm) +{ + struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); + + crypto_free_skcipher(ctx->fallback); + artpec6_crypto_aes_exit(tfm); +} + +static int +artpec6_crypto_cipher_set_key(struct crypto_skcipher *cipher, const u8 *key, + unsigned int keylen) +{ + struct artpec6_cryptotfm_context *ctx = + crypto_skcipher_ctx(cipher); + + switch (keylen) { + case 16: + case 24: + case 32: + break; + default: + crypto_skcipher_set_flags(cipher, + CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + memcpy(ctx->aes_key, key, keylen); + ctx->key_length = keylen; + return 0; +} + +static int +artpec6_crypto_xts_set_key(struct crypto_skcipher *cipher, const u8 *key, + unsigned int keylen) +{ + struct artpec6_cryptotfm_context *ctx = + crypto_skcipher_ctx(cipher); + int ret; + + ret = xts_check_key(&cipher->base, key, keylen); + if (ret) + return ret; + + switch (keylen) { + case 32: + case 48: + case 64: + break; + default: + crypto_skcipher_set_flags(cipher, + CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + memcpy(ctx->aes_key, key, keylen); + ctx->key_length = keylen; + return 0; +} + +/** artpec6_crypto_process_crypto - Prepare an async block cipher crypto request + * + * @req: The asynch request to process + * + * @return 0 if the dma job was successfully prepared + * <0 on error + * + * This function sets up the PDMA descriptors for a block cipher request. + * + * The required padding is added for AES-CTR using a statically defined + * buffer. + * + * The PDMA descriptor list will be as follows: + * + * OUT: [KEY_MD][KEY][EOP]<CIPHER_MD>[IV]<data_0>...[data_n][AES-CTR_pad]<eop> + * IN: <CIPHER_MD><data_0>...[data_n]<intr> + * + */ +static int artpec6_crypto_prepare_crypto(struct skcipher_request *areq) +{ + int ret; + struct artpec6_crypto_walk walk; + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); + struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher); + struct artpec6_crypto_request_context *req_ctx = NULL; + size_t iv_len = crypto_skcipher_ivsize(cipher); + struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); + enum artpec6_crypto_variant variant = ac->variant; + struct artpec6_crypto_req_common *common; + bool cipher_decr = false; + size_t cipher_klen; + u32 cipher_len = 0; /* Same as regk_crypto_key_128 for NULL crypto */ + u32 oper; + + req_ctx = skcipher_request_ctx(areq); + common = &req_ctx->common; + + artpec6_crypto_init_dma_operation(common); + + if (variant == ARTPEC6_CRYPTO) + ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER, a6_regk_crypto_dlkey); + else + ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER, a7_regk_crypto_dlkey); + + ret = artpec6_crypto_setup_out_descr(common, (void *)&ctx->key_md, + sizeof(ctx->key_md), false, false); + if (ret) + return ret; + + ret = artpec6_crypto_setup_out_descr(common, ctx->aes_key, + ctx->key_length, true, false); + if (ret) + return ret; + + req_ctx->cipher_md = 0; + + if (ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_XTS) + cipher_klen = ctx->key_length/2; + else + cipher_klen = ctx->key_length; + + /* Metadata */ + switch (cipher_klen) { + case 16: + cipher_len = regk_crypto_key_128; + break; + case 24: + cipher_len = regk_crypto_key_192; + break; + case 32: + cipher_len = regk_crypto_key_256; + break; + default: + pr_err("%s: Invalid key length %d!\n", + MODULE_NAME, ctx->key_length); + return -EINVAL; + } + + switch (ctx->crypto_type) { + case ARTPEC6_CRYPTO_CIPHER_AES_ECB: + oper = regk_crypto_aes_ecb; + cipher_decr = req_ctx->decrypt; + break; + + case ARTPEC6_CRYPTO_CIPHER_AES_CBC: + oper = regk_crypto_aes_cbc; + cipher_decr = req_ctx->decrypt; + break; + + case ARTPEC6_CRYPTO_CIPHER_AES_CTR: + oper = regk_crypto_aes_ctr; + cipher_decr = false; + break; + + case ARTPEC6_CRYPTO_CIPHER_AES_XTS: + oper = regk_crypto_aes_xts; + cipher_decr = req_ctx->decrypt; + + if (variant == ARTPEC6_CRYPTO) + req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DSEQ; + else + req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DSEQ; + break; + + default: + pr_err("%s: Invalid cipher mode %d!\n", + MODULE_NAME, ctx->crypto_type); + return -EINVAL; + } + + if (variant == ARTPEC6_CRYPTO) { + req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_OPER, oper); + req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_CIPHER_LEN, + cipher_len); + if (cipher_decr) + req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DECR; + } else { + req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_OPER, oper); + req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_CIPHER_LEN, + cipher_len); + if (cipher_decr) + req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DECR; + } + + ret = artpec6_crypto_setup_out_descr(common, + &req_ctx->cipher_md, + sizeof(req_ctx->cipher_md), + false, false); + if (ret) + return ret; + + ret = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false); + if (ret) + return ret; + + if (iv_len) { + ret = artpec6_crypto_setup_out_descr(common, areq->iv, iv_len, + false, false); + if (ret) + return ret; + } + /* Data out */ + artpec6_crypto_walk_init(&walk, areq->src); + ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, areq->cryptlen); + if (ret) + return ret; + + /* Data in */ + artpec6_crypto_walk_init(&walk, areq->dst); + ret = artpec6_crypto_setup_sg_descrs_in(common, &walk, areq->cryptlen); + if (ret) + return ret; + + /* CTR-mode padding required by the HW. */ + if (ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_CTR || + ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_XTS) { + size_t pad = ALIGN(areq->cryptlen, AES_BLOCK_SIZE) - + areq->cryptlen; + + if (pad) { + ret = artpec6_crypto_setup_out_descr(common, + ac->pad_buffer, + pad, false, false); + if (ret) + return ret; + + ret = artpec6_crypto_setup_in_descr(common, + ac->pad_buffer, pad, + false); + if (ret) + return ret; + } + } + + ret = artpec6_crypto_terminate_out_descrs(common); + if (ret) + return ret; + + ret = artpec6_crypto_terminate_in_descrs(common); + if (ret) + return ret; + + return artpec6_crypto_dma_map_descs(common); +} + +static int artpec6_crypto_prepare_aead(struct aead_request *areq) +{ + size_t count; + int ret; + size_t input_length; + struct artpec6_cryptotfm_context *ctx = crypto_tfm_ctx(areq->base.tfm); + struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(areq); + struct crypto_aead *cipher = crypto_aead_reqtfm(areq); + struct artpec6_crypto_req_common *common = &req_ctx->common; + struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); + enum artpec6_crypto_variant variant = ac->variant; + u32 md_cipher_len; + + artpec6_crypto_init_dma_operation(common); + + /* Key */ + if (variant == ARTPEC6_CRYPTO) { + ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER, + a6_regk_crypto_dlkey); + } else { + ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER, + a7_regk_crypto_dlkey); + } + ret = artpec6_crypto_setup_out_descr(common, (void *)&ctx->key_md, + sizeof(ctx->key_md), false, false); + if (ret) + return ret; + + ret = artpec6_crypto_setup_out_descr(common, ctx->aes_key, + ctx->key_length, true, false); + if (ret) + return ret; + + req_ctx->cipher_md = 0; + + switch (ctx->key_length) { + case 16: + md_cipher_len = regk_crypto_key_128; + break; + case 24: + md_cipher_len = regk_crypto_key_192; + break; + case 32: + md_cipher_len = regk_crypto_key_256; + break; + default: + return -EINVAL; + } + + if (variant == ARTPEC6_CRYPTO) { + req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_OPER, + regk_crypto_aes_gcm); + req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_CIPHER_LEN, + md_cipher_len); + if (req_ctx->decrypt) + req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DECR; + } else { + req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_OPER, + regk_crypto_aes_gcm); + req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_CIPHER_LEN, + md_cipher_len); + if (req_ctx->decrypt) + req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DECR; + } + + ret = artpec6_crypto_setup_out_descr(common, + (void *) &req_ctx->cipher_md, + sizeof(req_ctx->cipher_md), false, + false); + if (ret) + return ret; + + ret = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false); + if (ret) + return ret; + + /* For the decryption, cryptlen includes the tag. */ + input_length = areq->cryptlen; + if (req_ctx->decrypt) + input_length -= AES_BLOCK_SIZE; + + /* Prepare the context buffer */ + req_ctx->hw_ctx.aad_length_bits = + __cpu_to_be64(8*areq->assoclen); + + req_ctx->hw_ctx.text_length_bits = + __cpu_to_be64(8*input_length); + + memcpy(req_ctx->hw_ctx.J0, areq->iv, crypto_aead_ivsize(cipher)); + // The HW omits the initial increment of the counter field. + memcpy(req_ctx->hw_ctx.J0 + GCM_AES_IV_SIZE, "\x00\x00\x00\x01", 4); + + ret = artpec6_crypto_setup_out_descr(common, &req_ctx->hw_ctx, + sizeof(struct artpec6_crypto_aead_hw_ctx), false, false); + if (ret) + return ret; + + { + struct artpec6_crypto_walk walk; + + artpec6_crypto_walk_init(&walk, areq->src); + + /* Associated data */ + count = areq->assoclen; + ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, count); + if (ret) + return ret; + + if (!IS_ALIGNED(areq->assoclen, 16)) { + size_t assoc_pad = 16 - (areq->assoclen % 16); + /* The HW mandates zero padding here */ + ret = artpec6_crypto_setup_out_descr(common, + ac->zero_buffer, + assoc_pad, false, + false); + if (ret) + return ret; + } + + /* Data to crypto */ + count = input_length; + ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, count); + if (ret) + return ret; + + if (!IS_ALIGNED(input_length, 16)) { + size_t crypto_pad = 16 - (input_length % 16); + /* The HW mandates zero padding here */ + ret = artpec6_crypto_setup_out_descr(common, + ac->zero_buffer, + crypto_pad, + false, + false); + if (ret) + return ret; + } + } + + /* Data from crypto */ + { + struct artpec6_crypto_walk walk; + size_t output_len = areq->cryptlen; + + if (req_ctx->decrypt) + output_len -= AES_BLOCK_SIZE; + + artpec6_crypto_walk_init(&walk, areq->dst); + + /* skip associated data in the output */ + count = artpec6_crypto_walk_advance(&walk, areq->assoclen); + if (count) + return -EINVAL; + + count = output_len; + ret = artpec6_crypto_setup_sg_descrs_in(common, &walk, count); + if (ret) + return ret; + + /* Put padding between the cryptotext and the auth tag */ + if (!IS_ALIGNED(output_len, 16)) { + size_t crypto_pad = 16 - (output_len % 16); + + ret = artpec6_crypto_setup_in_descr(common, + ac->pad_buffer, + crypto_pad, false); + if (ret) + return ret; + } + + /* The authentication tag shall follow immediately after + * the output ciphertext. For decryption it is put in a context + * buffer for later compare against the input tag. + */ + count = AES_BLOCK_SIZE; + + if (req_ctx->decrypt) { + ret = artpec6_crypto_setup_in_descr(common, + req_ctx->decryption_tag, count, false); + if (ret) + return ret; + + } else { + ret = artpec6_crypto_setup_sg_descrs_in(common, &walk, + count); + if (ret) + return ret; + } + + } + + ret = artpec6_crypto_terminate_in_descrs(common); + if (ret) + return ret; + + ret = artpec6_crypto_terminate_out_descrs(common); + if (ret) + return ret; + + return artpec6_crypto_dma_map_descs(common); +} + +static void artpec6_crypto_process_queue(struct artpec6_crypto *ac, + struct list_head *completions) +{ + struct artpec6_crypto_req_common *req; + + while (!list_empty(&ac->queue) && !artpec6_crypto_busy()) { + req = list_first_entry(&ac->queue, + struct artpec6_crypto_req_common, + list); + list_move_tail(&req->list, &ac->pending); + artpec6_crypto_start_dma(req); + + list_add_tail(&req->complete_in_progress, completions); + } + + /* + * In some cases, the hardware can raise an in_eop_flush interrupt + * before actually updating the status, so we have an timer which will + * recheck the status on timeout. Since the cases are expected to be + * very rare, we use a relatively large timeout value. There should be + * no noticeable negative effect if we timeout spuriously. + */ + if (ac->pending_count) + mod_timer(&ac->timer, jiffies + msecs_to_jiffies(100)); + else + del_timer(&ac->timer); +} + +static void artpec6_crypto_timeout(struct timer_list *t) +{ + struct artpec6_crypto *ac = from_timer(ac, t, timer); + + dev_info_ratelimited(artpec6_crypto_dev, "timeout\n"); + + tasklet_schedule(&ac->task); +} + +static void artpec6_crypto_task(unsigned long data) +{ + struct artpec6_crypto *ac = (struct artpec6_crypto *)data; + struct artpec6_crypto_req_common *req; + struct artpec6_crypto_req_common *n; + struct list_head complete_done; + struct list_head complete_in_progress; + + INIT_LIST_HEAD(&complete_done); + INIT_LIST_HEAD(&complete_in_progress); + + if (list_empty(&ac->pending)) { + pr_debug("Spurious IRQ\n"); + return; + } + + spin_lock_bh(&ac->queue_lock); + + list_for_each_entry_safe(req, n, &ac->pending, list) { + struct artpec6_crypto_dma_descriptors *dma = req->dma; + u32 stat; + + dma_sync_single_for_cpu(artpec6_crypto_dev, dma->stat_dma_addr, + sizeof(dma->stat[0]), + DMA_BIDIRECTIONAL); + + stat = req->dma->stat[req->dma->in_cnt-1]; + + /* A non-zero final status descriptor indicates + * this job has finished. + */ + pr_debug("Request %p status is %X\n", req, stat); + if (!stat) + break; + + /* Allow testing of timeout handling with fault injection */ +#ifdef CONFIG_FAULT_INJECTION + if (should_fail(&artpec6_crypto_fail_status_read, 1)) + continue; +#endif + + pr_debug("Completing request %p\n", req); + + list_move_tail(&req->list, &complete_done); + + artpec6_crypto_dma_unmap_all(req); + artpec6_crypto_copy_bounce_buffers(req); + + ac->pending_count--; + artpec6_crypto_common_destroy(req); + } + + artpec6_crypto_process_queue(ac, &complete_in_progress); + + spin_unlock_bh(&ac->queue_lock); + + /* Perform the completion callbacks without holding the queue lock + * to allow new request submissions from the callbacks. + */ + list_for_each_entry_safe(req, n, &complete_done, list) { + req->complete(req->req); + } + + list_for_each_entry_safe(req, n, &complete_in_progress, + complete_in_progress) { + req->req->complete(req->req, -EINPROGRESS); + } +} + +static void artpec6_crypto_complete_crypto(struct crypto_async_request *req) +{ + req->complete(req, 0); +} + +static void +artpec6_crypto_complete_cbc_decrypt(struct crypto_async_request *req) +{ + struct skcipher_request *cipher_req = container_of(req, + struct skcipher_request, base); + + scatterwalk_map_and_copy(cipher_req->iv, cipher_req->src, + cipher_req->cryptlen - AES_BLOCK_SIZE, + AES_BLOCK_SIZE, 0); + req->complete(req, 0); +} + +static void +artpec6_crypto_complete_cbc_encrypt(struct crypto_async_request *req) +{ + struct skcipher_request *cipher_req = container_of(req, + struct skcipher_request, base); + + scatterwalk_map_and_copy(cipher_req->iv, cipher_req->dst, + cipher_req->cryptlen - AES_BLOCK_SIZE, + AES_BLOCK_SIZE, 0); + req->complete(req, 0); +} + +static void artpec6_crypto_complete_aead(struct crypto_async_request *req) +{ + int result = 0; + + /* Verify GCM hashtag. */ + struct aead_request *areq = container_of(req, + struct aead_request, base); + struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(areq); + + if (req_ctx->decrypt) { + u8 input_tag[AES_BLOCK_SIZE]; + + sg_pcopy_to_buffer(areq->src, + sg_nents(areq->src), + input_tag, + AES_BLOCK_SIZE, + areq->assoclen + areq->cryptlen - + AES_BLOCK_SIZE); + + if (memcmp(req_ctx->decryption_tag, + input_tag, + AES_BLOCK_SIZE)) { + pr_debug("***EBADMSG:\n"); + print_hex_dump_debug("ref:", DUMP_PREFIX_ADDRESS, 32, 1, + input_tag, AES_BLOCK_SIZE, true); + print_hex_dump_debug("out:", DUMP_PREFIX_ADDRESS, 32, 1, + req_ctx->decryption_tag, + AES_BLOCK_SIZE, true); + + result = -EBADMSG; + } + } + + req->complete(req, result); +} + +static void artpec6_crypto_complete_hash(struct crypto_async_request *req) +{ + req->complete(req, 0); +} + + +/*------------------- Hash functions -----------------------------------------*/ +static int +artpec6_crypto_hash_set_key(struct crypto_ahash *tfm, + const u8 *key, unsigned int keylen) +{ + struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(&tfm->base); + size_t blocksize; + int ret; + + if (!keylen) { + pr_err("Invalid length (%d) of HMAC key\n", + keylen); + return -EINVAL; + } + + memset(tfm_ctx->hmac_key, 0, sizeof(tfm_ctx->hmac_key)); + + blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + + if (keylen > blocksize) { + SHASH_DESC_ON_STACK(hdesc, tfm_ctx->child_hash); + + hdesc->tfm = tfm_ctx->child_hash; + hdesc->flags = crypto_ahash_get_flags(tfm) & + CRYPTO_TFM_REQ_MAY_SLEEP; + + tfm_ctx->hmac_key_length = blocksize; + ret = crypto_shash_digest(hdesc, key, keylen, + tfm_ctx->hmac_key); + if (ret) + return ret; + + } else { + memcpy(tfm_ctx->hmac_key, key, keylen); + tfm_ctx->hmac_key_length = keylen; + } + + return 0; +} + +static int +artpec6_crypto_init_hash(struct ahash_request *req, u8 type, int hmac) +{ + struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); + enum artpec6_crypto_variant variant = ac->variant; + struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); + u32 oper; + + memset(req_ctx, 0, sizeof(*req_ctx)); + + req_ctx->hash_flags = HASH_FLAG_INIT_CTX; + if (hmac) + req_ctx->hash_flags |= (HASH_FLAG_HMAC | HASH_FLAG_UPDATE_KEY); + + switch (type) { + case ARTPEC6_CRYPTO_HASH_SHA1: + oper = hmac ? regk_crypto_hmac_sha1 : regk_crypto_sha1; + break; + case ARTPEC6_CRYPTO_HASH_SHA256: + oper = hmac ? regk_crypto_hmac_sha256 : regk_crypto_sha256; + break; + case ARTPEC6_CRYPTO_HASH_SHA384: + oper = hmac ? regk_crypto_hmac_sha384 : regk_crypto_sha384; + break; + case ARTPEC6_CRYPTO_HASH_SHA512: + oper = hmac ? regk_crypto_hmac_sha512 : regk_crypto_sha512; + break; + + default: + pr_err("%s: Unsupported hash type 0x%x\n", MODULE_NAME, type); + return -EINVAL; + } + + if (variant == ARTPEC6_CRYPTO) + req_ctx->hash_md = FIELD_PREP(A6_CRY_MD_OPER, oper); + else + req_ctx->hash_md = FIELD_PREP(A7_CRY_MD_OPER, oper); + + return 0; +} + +static int artpec6_crypto_prepare_submit_hash(struct ahash_request *req) +{ + struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); + int ret; + + if (!req_ctx->common.dma) { + ret = artpec6_crypto_common_init(&req_ctx->common, + &req->base, + artpec6_crypto_complete_hash, + NULL, 0); + + if (ret) + return ret; + } + + ret = artpec6_crypto_prepare_hash(req); + switch (ret) { + case ARTPEC6_CRYPTO_PREPARE_HASH_START: + ret = artpec6_crypto_submit(&req_ctx->common); + break; + + case ARTPEC6_CRYPTO_PREPARE_HASH_NO_START: + ret = 0; + /* Fallthrough */ + + default: + artpec6_crypto_common_destroy(&req_ctx->common); + break; + } + + return ret; +} + +static int artpec6_crypto_hash_final(struct ahash_request *req) +{ + struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); + + req_ctx->hash_flags |= HASH_FLAG_FINALIZE; + + return artpec6_crypto_prepare_submit_hash(req); +} + +static int artpec6_crypto_hash_update(struct ahash_request *req) +{ + struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); + + req_ctx->hash_flags |= HASH_FLAG_UPDATE; + + return artpec6_crypto_prepare_submit_hash(req); +} + +static int artpec6_crypto_sha1_init(struct ahash_request *req) +{ + return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA1, 0); +} + +static int artpec6_crypto_sha1_digest(struct ahash_request *req) +{ + struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); + + artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA1, 0); + + req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; + + return artpec6_crypto_prepare_submit_hash(req); +} + +static int artpec6_crypto_sha256_init(struct ahash_request *req) +{ + return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 0); +} + +static int artpec6_crypto_sha256_digest(struct ahash_request *req) +{ + struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); + + artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 0); + req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; + + return artpec6_crypto_prepare_submit_hash(req); +} + +static int __maybe_unused artpec6_crypto_sha384_init(struct ahash_request *req) +{ + return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA384, 0); +} + +static int __maybe_unused +artpec6_crypto_sha384_digest(struct ahash_request *req) +{ + struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); + + artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA384, 0); + req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; + + return artpec6_crypto_prepare_submit_hash(req); +} + +static int artpec6_crypto_sha512_init(struct ahash_request *req) +{ + return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA512, 0); +} + +static int artpec6_crypto_sha512_digest(struct ahash_request *req) +{ + struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); + + artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA512, 0); + req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; + + return artpec6_crypto_prepare_submit_hash(req); +} + +static int artpec6_crypto_hmac_sha256_init(struct ahash_request *req) +{ + return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 1); +} + +static int __maybe_unused +artpec6_crypto_hmac_sha384_init(struct ahash_request *req) +{ + return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA384, 1); +} + +static int artpec6_crypto_hmac_sha512_init(struct ahash_request *req) +{ + return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA512, 1); +} + +static int artpec6_crypto_hmac_sha256_digest(struct ahash_request *req) +{ + struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); + + artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 1); + req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; + + return artpec6_crypto_prepare_submit_hash(req); +} + +static int __maybe_unused +artpec6_crypto_hmac_sha384_digest(struct ahash_request *req) +{ + struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); + + artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA384, 1); + req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; + + return artpec6_crypto_prepare_submit_hash(req); +} + +static int artpec6_crypto_hmac_sha512_digest(struct ahash_request *req) +{ + struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); + + artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA512, 1); + req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; + + return artpec6_crypto_prepare_submit_hash(req); +} + +static int artpec6_crypto_ahash_init_common(struct crypto_tfm *tfm, + const char *base_hash_name) +{ + struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(tfm); + + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct artpec6_hash_request_context)); + memset(tfm_ctx, 0, sizeof(*tfm_ctx)); + + if (base_hash_name) { + struct crypto_shash *child; + + child = crypto_alloc_shash(base_hash_name, 0, + CRYPTO_ALG_NEED_FALLBACK); + + if (IS_ERR(child)) + return PTR_ERR(child); + + tfm_ctx->child_hash = child; + } + + return 0; +} + +static int artpec6_crypto_ahash_init(struct crypto_tfm *tfm) +{ + return artpec6_crypto_ahash_init_common(tfm, NULL); +} + +static int artpec6_crypto_ahash_init_hmac_sha256(struct crypto_tfm *tfm) +{ + return artpec6_crypto_ahash_init_common(tfm, "sha256"); +} + +static int __maybe_unused +artpec6_crypto_ahash_init_hmac_sha384(struct crypto_tfm *tfm) +{ + return artpec6_crypto_ahash_init_common(tfm, "sha384"); +} + +static int artpec6_crypto_ahash_init_hmac_sha512(struct crypto_tfm *tfm) +{ + return artpec6_crypto_ahash_init_common(tfm, "sha512"); +} + +static void artpec6_crypto_ahash_exit(struct crypto_tfm *tfm) +{ + struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(tfm); + + if (tfm_ctx->child_hash) + crypto_free_shash(tfm_ctx->child_hash); + + memset(tfm_ctx->hmac_key, 0, sizeof(tfm_ctx->hmac_key)); + tfm_ctx->hmac_key_length = 0; +} + +static int artpec6_crypto_hash_export(struct ahash_request *req, void *out) +{ + const struct artpec6_hash_request_context *ctx = ahash_request_ctx(req); + struct artpec6_hash_export_state *state = out; + struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); + enum artpec6_crypto_variant variant = ac->variant; + + BUILD_BUG_ON(sizeof(state->partial_buffer) != + sizeof(ctx->partial_buffer)); + BUILD_BUG_ON(sizeof(state->digeststate) != sizeof(ctx->digeststate)); + + state->digcnt = ctx->digcnt; + state->partial_bytes = ctx->partial_bytes; + state->hash_flags = ctx->hash_flags; + + if (variant == ARTPEC6_CRYPTO) + state->oper = FIELD_GET(A6_CRY_MD_OPER, ctx->hash_md); + else + state->oper = FIELD_GET(A7_CRY_MD_OPER, ctx->hash_md); + + memcpy(state->partial_buffer, ctx->partial_buffer, + sizeof(state->partial_buffer)); + memcpy(state->digeststate, ctx->digeststate, + sizeof(state->digeststate)); + + return 0; +} + +static int artpec6_crypto_hash_import(struct ahash_request *req, const void *in) +{ + struct artpec6_hash_request_context *ctx = ahash_request_ctx(req); + const struct artpec6_hash_export_state *state = in; + struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); + enum artpec6_crypto_variant variant = ac->variant; + + memset(ctx, 0, sizeof(*ctx)); + + ctx->digcnt = state->digcnt; + ctx->partial_bytes = state->partial_bytes; + ctx->hash_flags = state->hash_flags; + + if (variant == ARTPEC6_CRYPTO) + ctx->hash_md = FIELD_PREP(A6_CRY_MD_OPER, state->oper); + else + ctx->hash_md = FIELD_PREP(A7_CRY_MD_OPER, state->oper); + + memcpy(ctx->partial_buffer, state->partial_buffer, + sizeof(state->partial_buffer)); + memcpy(ctx->digeststate, state->digeststate, + sizeof(state->digeststate)); + + return 0; +} + +static int init_crypto_hw(struct artpec6_crypto *ac) +{ + enum artpec6_crypto_variant variant = ac->variant; + void __iomem *base = ac->base; + u32 out_descr_buf_size; + u32 out_data_buf_size; + u32 in_data_buf_size; + u32 in_descr_buf_size; + u32 in_stat_buf_size; + u32 in, out; + + /* + * The PDMA unit contains 1984 bytes of internal memory for the OUT + * channels and 1024 bytes for the IN channel. This is an elastic + * memory used to internally store the descriptors and data. The values + * ares specified in 64 byte incremements. Trustzone buffers are not + * used at this stage. + */ + out_data_buf_size = 16; /* 1024 bytes for data */ + out_descr_buf_size = 15; /* 960 bytes for descriptors */ + in_data_buf_size = 8; /* 512 bytes for data */ + in_descr_buf_size = 4; /* 256 bytes for descriptors */ + in_stat_buf_size = 4; /* 256 bytes for stat descrs */ + + BUILD_BUG_ON_MSG((out_data_buf_size + + out_descr_buf_size) * 64 > 1984, + "Invalid OUT configuration"); + + BUILD_BUG_ON_MSG((in_data_buf_size + + in_descr_buf_size + + in_stat_buf_size) * 64 > 1024, + "Invalid IN configuration"); + + in = FIELD_PREP(PDMA_IN_BUF_CFG_DATA_BUF_SIZE, in_data_buf_size) | + FIELD_PREP(PDMA_IN_BUF_CFG_DESCR_BUF_SIZE, in_descr_buf_size) | + FIELD_PREP(PDMA_IN_BUF_CFG_STAT_BUF_SIZE, in_stat_buf_size); + + out = FIELD_PREP(PDMA_OUT_BUF_CFG_DATA_BUF_SIZE, out_data_buf_size) | + FIELD_PREP(PDMA_OUT_BUF_CFG_DESCR_BUF_SIZE, out_descr_buf_size); + + writel_relaxed(out, base + PDMA_OUT_BUF_CFG); + writel_relaxed(PDMA_OUT_CFG_EN, base + PDMA_OUT_CFG); + + if (variant == ARTPEC6_CRYPTO) { + writel_relaxed(in, base + A6_PDMA_IN_BUF_CFG); + writel_relaxed(PDMA_IN_CFG_EN, base + A6_PDMA_IN_CFG); + writel_relaxed(A6_PDMA_INTR_MASK_IN_DATA | + A6_PDMA_INTR_MASK_IN_EOP_FLUSH, + base + A6_PDMA_INTR_MASK); + } else { + writel_relaxed(in, base + A7_PDMA_IN_BUF_CFG); + writel_relaxed(PDMA_IN_CFG_EN, base + A7_PDMA_IN_CFG); + writel_relaxed(A7_PDMA_INTR_MASK_IN_DATA | + A7_PDMA_INTR_MASK_IN_EOP_FLUSH, + base + A7_PDMA_INTR_MASK); + } + + return 0; +} + +static void artpec6_crypto_disable_hw(struct artpec6_crypto *ac) +{ + enum artpec6_crypto_variant variant = ac->variant; + void __iomem *base = ac->base; + + if (variant == ARTPEC6_CRYPTO) { + writel_relaxed(A6_PDMA_IN_CMD_STOP, base + A6_PDMA_IN_CMD); + writel_relaxed(0, base + A6_PDMA_IN_CFG); + writel_relaxed(A6_PDMA_OUT_CMD_STOP, base + PDMA_OUT_CMD); + } else { + writel_relaxed(A7_PDMA_IN_CMD_STOP, base + A7_PDMA_IN_CMD); + writel_relaxed(0, base + A7_PDMA_IN_CFG); + writel_relaxed(A7_PDMA_OUT_CMD_STOP, base + PDMA_OUT_CMD); + } + + writel_relaxed(0, base + PDMA_OUT_CFG); + +} + +static irqreturn_t artpec6_crypto_irq(int irq, void *dev_id) +{ + struct artpec6_crypto *ac = dev_id; + enum artpec6_crypto_variant variant = ac->variant; + void __iomem *base = ac->base; + u32 mask_in_data, mask_in_eop_flush; + u32 in_cmd_flush_stat, in_cmd_reg; + u32 ack_intr_reg; + u32 ack = 0; + u32 intr; + + if (variant == ARTPEC6_CRYPTO) { + intr = readl_relaxed(base + A6_PDMA_MASKED_INTR); + mask_in_data = A6_PDMA_INTR_MASK_IN_DATA; + mask_in_eop_flush = A6_PDMA_INTR_MASK_IN_EOP_FLUSH; + in_cmd_flush_stat = A6_PDMA_IN_CMD_FLUSH_STAT; + in_cmd_reg = A6_PDMA_IN_CMD; + ack_intr_reg = A6_PDMA_ACK_INTR; + } else { + intr = readl_relaxed(base + A7_PDMA_MASKED_INTR); + mask_in_data = A7_PDMA_INTR_MASK_IN_DATA; + mask_in_eop_flush = A7_PDMA_INTR_MASK_IN_EOP_FLUSH; + in_cmd_flush_stat = A7_PDMA_IN_CMD_FLUSH_STAT; + in_cmd_reg = A7_PDMA_IN_CMD; + ack_intr_reg = A7_PDMA_ACK_INTR; + } + + /* We get two interrupt notifications from each job. + * The in_data means all data was sent to memory and then + * we request a status flush command to write the per-job + * status to its status vector. This ensures that the + * tasklet can detect exactly how many submitted jobs + * that have finished. + */ + if (intr & mask_in_data) + ack |= mask_in_data; + + if (intr & mask_in_eop_flush) + ack |= mask_in_eop_flush; + else + writel_relaxed(in_cmd_flush_stat, base + in_cmd_reg); + + writel_relaxed(ack, base + ack_intr_reg); + + if (intr & mask_in_eop_flush) + tasklet_schedule(&ac->task); + + return IRQ_HANDLED; +} + +/*------------------- Algorithm definitions ----------------------------------*/ + +/* Hashes */ +static struct ahash_alg hash_algos[] = { + /* SHA-1 */ + { + .init = artpec6_crypto_sha1_init, + .update = artpec6_crypto_hash_update, + .final = artpec6_crypto_hash_final, + .digest = artpec6_crypto_sha1_digest, + .import = artpec6_crypto_hash_import, + .export = artpec6_crypto_hash_export, + .halg.digestsize = SHA1_DIGEST_SIZE, + .halg.statesize = sizeof(struct artpec6_hash_export_state), + .halg.base = { + .cra_name = "sha1", + .cra_driver_name = "artpec-sha1", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct artpec6_hashalg_context), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + .cra_init = artpec6_crypto_ahash_init, + .cra_exit = artpec6_crypto_ahash_exit, + } + }, + /* SHA-256 */ + { + .init = artpec6_crypto_sha256_init, + .update = artpec6_crypto_hash_update, + .final = artpec6_crypto_hash_final, + .digest = artpec6_crypto_sha256_digest, + .import = artpec6_crypto_hash_import, + .export = artpec6_crypto_hash_export, + .halg.digestsize = SHA256_DIGEST_SIZE, + .halg.statesize = sizeof(struct artpec6_hash_export_state), + .halg.base = { + .cra_name = "sha256", + .cra_driver_name = "artpec-sha256", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct artpec6_hashalg_context), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + .cra_init = artpec6_crypto_ahash_init, + .cra_exit = artpec6_crypto_ahash_exit, + } + }, + /* HMAC SHA-256 */ + { + .init = artpec6_crypto_hmac_sha256_init, + .update = artpec6_crypto_hash_update, + .final = artpec6_crypto_hash_final, + .digest = artpec6_crypto_hmac_sha256_digest, + .import = artpec6_crypto_hash_import, + .export = artpec6_crypto_hash_export, + .setkey = artpec6_crypto_hash_set_key, + .halg.digestsize = SHA256_DIGEST_SIZE, + .halg.statesize = sizeof(struct artpec6_hash_export_state), + .halg.base = { + .cra_name = "hmac(sha256)", + .cra_driver_name = "artpec-hmac-sha256", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct artpec6_hashalg_context), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + .cra_init = artpec6_crypto_ahash_init_hmac_sha256, + .cra_exit = artpec6_crypto_ahash_exit, + } + }, +}; + +static struct ahash_alg artpec7_hash_algos[] = { + /* SHA-384 */ + { + .init = artpec6_crypto_sha384_init, + .update = artpec6_crypto_hash_update, + .final = artpec6_crypto_hash_final, + .digest = artpec6_crypto_sha384_digest, + .import = artpec6_crypto_hash_import, + .export = artpec6_crypto_hash_export, + .halg.digestsize = SHA384_DIGEST_SIZE, + .halg.statesize = sizeof(struct artpec6_hash_export_state), + .halg.base = { + .cra_name = "sha384", + .cra_driver_name = "artpec-sha384", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct artpec6_hashalg_context), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + .cra_init = artpec6_crypto_ahash_init, + .cra_exit = artpec6_crypto_ahash_exit, + } + }, + /* HMAC SHA-384 */ + { + .init = artpec6_crypto_hmac_sha384_init, + .update = artpec6_crypto_hash_update, + .final = artpec6_crypto_hash_final, + .digest = artpec6_crypto_hmac_sha384_digest, + .import = artpec6_crypto_hash_import, + .export = artpec6_crypto_hash_export, + .setkey = artpec6_crypto_hash_set_key, + .halg.digestsize = SHA384_DIGEST_SIZE, + .halg.statesize = sizeof(struct artpec6_hash_export_state), + .halg.base = { + .cra_name = "hmac(sha384)", + .cra_driver_name = "artpec-hmac-sha384", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct artpec6_hashalg_context), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + .cra_init = artpec6_crypto_ahash_init_hmac_sha384, + .cra_exit = artpec6_crypto_ahash_exit, + } + }, + /* SHA-512 */ + { + .init = artpec6_crypto_sha512_init, + .update = artpec6_crypto_hash_update, + .final = artpec6_crypto_hash_final, + .digest = artpec6_crypto_sha512_digest, + .import = artpec6_crypto_hash_import, + .export = artpec6_crypto_hash_export, + .halg.digestsize = SHA512_DIGEST_SIZE, + .halg.statesize = sizeof(struct artpec6_hash_export_state), + .halg.base = { + .cra_name = "sha512", + .cra_driver_name = "artpec-sha512", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct artpec6_hashalg_context), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + .cra_init = artpec6_crypto_ahash_init, + .cra_exit = artpec6_crypto_ahash_exit, + } + }, + /* HMAC SHA-512 */ + { + .init = artpec6_crypto_hmac_sha512_init, + .update = artpec6_crypto_hash_update, + .final = artpec6_crypto_hash_final, + .digest = artpec6_crypto_hmac_sha512_digest, + .import = artpec6_crypto_hash_import, + .export = artpec6_crypto_hash_export, + .setkey = artpec6_crypto_hash_set_key, + .halg.digestsize = SHA512_DIGEST_SIZE, + .halg.statesize = sizeof(struct artpec6_hash_export_state), + .halg.base = { + .cra_name = "hmac(sha512)", + .cra_driver_name = "artpec-hmac-sha512", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct artpec6_hashalg_context), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + .cra_init = artpec6_crypto_ahash_init_hmac_sha512, + .cra_exit = artpec6_crypto_ahash_exit, + } + }, +}; + +/* Crypto */ +static struct skcipher_alg crypto_algos[] = { + /* AES - ECB */ + { + .base = { + .cra_name = "ecb(aes)", + .cra_driver_name = "artpec6-ecb-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = artpec6_crypto_cipher_set_key, + .encrypt = artpec6_crypto_encrypt, + .decrypt = artpec6_crypto_decrypt, + .init = artpec6_crypto_aes_ecb_init, + .exit = artpec6_crypto_aes_exit, + }, + /* AES - CTR */ + { + .base = { + .cra_name = "ctr(aes)", + .cra_driver_name = "artpec6-ctr-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = artpec6_crypto_cipher_set_key, + .encrypt = artpec6_crypto_ctr_encrypt, + .decrypt = artpec6_crypto_ctr_decrypt, + .init = artpec6_crypto_aes_ctr_init, + .exit = artpec6_crypto_aes_ctr_exit, + }, + /* AES - CBC */ + { + .base = { + .cra_name = "cbc(aes)", + .cra_driver_name = "artpec6-cbc-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = artpec6_crypto_cipher_set_key, + .encrypt = artpec6_crypto_encrypt, + .decrypt = artpec6_crypto_decrypt, + .init = artpec6_crypto_aes_cbc_init, + .exit = artpec6_crypto_aes_exit + }, + /* AES - XTS */ + { + .base = { + .cra_name = "xts(aes)", + .cra_driver_name = "artpec6-xts-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + }, + .min_keysize = 2*AES_MIN_KEY_SIZE, + .max_keysize = 2*AES_MAX_KEY_SIZE, + .ivsize = 16, + .setkey = artpec6_crypto_xts_set_key, + .encrypt = artpec6_crypto_encrypt, + .decrypt = artpec6_crypto_decrypt, + .init = artpec6_crypto_aes_xts_init, + .exit = artpec6_crypto_aes_exit, + }, +}; + +static struct aead_alg aead_algos[] = { + { + .init = artpec6_crypto_aead_init, + .setkey = artpec6_crypto_aead_set_key, + .encrypt = artpec6_crypto_aead_encrypt, + .decrypt = artpec6_crypto_aead_decrypt, + .ivsize = GCM_AES_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + + .base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "artpec-gcm-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + }, + } +}; + +#ifdef CONFIG_DEBUG_FS + +struct dbgfs_u32 { + char *name; + mode_t mode; + u32 *flag; + char *desc; +}; + +static struct dentry *dbgfs_root; + +static void artpec6_crypto_init_debugfs(void) +{ + dbgfs_root = debugfs_create_dir("artpec6_crypto", NULL); + + if (!dbgfs_root || IS_ERR(dbgfs_root)) { + dbgfs_root = NULL; + pr_err("%s: Could not initialise debugfs!\n", MODULE_NAME); + return; + } + +#ifdef CONFIG_FAULT_INJECTION + fault_create_debugfs_attr("fail_status_read", dbgfs_root, + &artpec6_crypto_fail_status_read); + + fault_create_debugfs_attr("fail_dma_array_full", dbgfs_root, + &artpec6_crypto_fail_dma_array_full); +#endif +} + +static void artpec6_crypto_free_debugfs(void) +{ + if (!dbgfs_root) + return; + + debugfs_remove_recursive(dbgfs_root); + dbgfs_root = NULL; +} +#endif + +static const struct of_device_id artpec6_crypto_of_match[] = { + { .compatible = "axis,artpec6-crypto", .data = (void *)ARTPEC6_CRYPTO }, + { .compatible = "axis,artpec7-crypto", .data = (void *)ARTPEC7_CRYPTO }, + {} +}; +MODULE_DEVICE_TABLE(of, artpec6_crypto_of_match); + +static int artpec6_crypto_probe(struct platform_device *pdev) +{ + const struct of_device_id *match; + enum artpec6_crypto_variant variant; + struct artpec6_crypto *ac; + struct device *dev = &pdev->dev; + void __iomem *base; + struct resource *res; + int irq; + int err; + + if (artpec6_crypto_dev) + return -ENODEV; + + match = of_match_node(artpec6_crypto_of_match, dev->of_node); + if (!match) + return -EINVAL; + + variant = (enum artpec6_crypto_variant)match->data; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(base)) + return PTR_ERR(base); + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return -ENODEV; + + ac = devm_kzalloc(&pdev->dev, sizeof(struct artpec6_crypto), + GFP_KERNEL); + if (!ac) + return -ENOMEM; + + platform_set_drvdata(pdev, ac); + ac->variant = variant; + + spin_lock_init(&ac->queue_lock); + INIT_LIST_HEAD(&ac->queue); + INIT_LIST_HEAD(&ac->pending); + timer_setup(&ac->timer, artpec6_crypto_timeout, 0); + + ac->base = base; + + ac->dma_cache = kmem_cache_create("artpec6_crypto_dma", + sizeof(struct artpec6_crypto_dma_descriptors), + 64, + 0, + NULL); + if (!ac->dma_cache) + return -ENOMEM; + +#ifdef CONFIG_DEBUG_FS + artpec6_crypto_init_debugfs(); +#endif + + tasklet_init(&ac->task, artpec6_crypto_task, + (unsigned long)ac); + + ac->pad_buffer = devm_kzalloc(&pdev->dev, 2 * ARTPEC_CACHE_LINE_MAX, + GFP_KERNEL); + if (!ac->pad_buffer) + return -ENOMEM; + ac->pad_buffer = PTR_ALIGN(ac->pad_buffer, ARTPEC_CACHE_LINE_MAX); + + ac->zero_buffer = devm_kzalloc(&pdev->dev, 2 * ARTPEC_CACHE_LINE_MAX, + GFP_KERNEL); + if (!ac->zero_buffer) + return -ENOMEM; + ac->zero_buffer = PTR_ALIGN(ac->zero_buffer, ARTPEC_CACHE_LINE_MAX); + + err = init_crypto_hw(ac); + if (err) + goto free_cache; + + err = devm_request_irq(&pdev->dev, irq, artpec6_crypto_irq, 0, + "artpec6-crypto", ac); + if (err) + goto disable_hw; + + artpec6_crypto_dev = &pdev->dev; + + err = crypto_register_ahashes(hash_algos, ARRAY_SIZE(hash_algos)); + if (err) { + dev_err(dev, "Failed to register ahashes\n"); + goto disable_hw; + } + + if (variant != ARTPEC6_CRYPTO) { + err = crypto_register_ahashes(artpec7_hash_algos, + ARRAY_SIZE(artpec7_hash_algos)); + if (err) { + dev_err(dev, "Failed to register ahashes\n"); + goto unregister_ahashes; + } + } + + err = crypto_register_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos)); + if (err) { + dev_err(dev, "Failed to register ciphers\n"); + goto unregister_a7_ahashes; + } + + err = crypto_register_aeads(aead_algos, ARRAY_SIZE(aead_algos)); + if (err) { + dev_err(dev, "Failed to register aeads\n"); + goto unregister_algs; + } + + return 0; + +unregister_algs: + crypto_unregister_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos)); +unregister_a7_ahashes: + if (variant != ARTPEC6_CRYPTO) + crypto_unregister_ahashes(artpec7_hash_algos, + ARRAY_SIZE(artpec7_hash_algos)); +unregister_ahashes: + crypto_unregister_ahashes(hash_algos, ARRAY_SIZE(hash_algos)); +disable_hw: + artpec6_crypto_disable_hw(ac); +free_cache: + kmem_cache_destroy(ac->dma_cache); + return err; +} + +static int artpec6_crypto_remove(struct platform_device *pdev) +{ + struct artpec6_crypto *ac = platform_get_drvdata(pdev); + int irq = platform_get_irq(pdev, 0); + + crypto_unregister_ahashes(hash_algos, ARRAY_SIZE(hash_algos)); + if (ac->variant != ARTPEC6_CRYPTO) + crypto_unregister_ahashes(artpec7_hash_algos, + ARRAY_SIZE(artpec7_hash_algos)); + crypto_unregister_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos)); + crypto_unregister_aeads(aead_algos, ARRAY_SIZE(aead_algos)); + + tasklet_disable(&ac->task); + devm_free_irq(&pdev->dev, irq, ac); + tasklet_kill(&ac->task); + del_timer_sync(&ac->timer); + + artpec6_crypto_disable_hw(ac); + + kmem_cache_destroy(ac->dma_cache); +#ifdef CONFIG_DEBUG_FS + artpec6_crypto_free_debugfs(); +#endif + return 0; +} + +static struct platform_driver artpec6_crypto_driver = { + .probe = artpec6_crypto_probe, + .remove = artpec6_crypto_remove, + .driver = { + .name = "artpec6-crypto", + .owner = THIS_MODULE, + .of_match_table = artpec6_crypto_of_match, + }, +}; + +module_platform_driver(artpec6_crypto_driver); + +MODULE_AUTHOR("Axis Communications AB"); +MODULE_DESCRIPTION("ARTPEC-6 Crypto driver"); +MODULE_LICENSE("GPL"); |