diff options
Diffstat (limited to '')
| -rw-r--r-- | drivers/crypto/ux500/Kconfig | 32 | ||||
| -rw-r--r-- | drivers/crypto/ux500/Makefile | 8 | ||||
| -rw-r--r-- | drivers/crypto/ux500/cryp/Makefile | 10 | ||||
| -rw-r--r-- | drivers/crypto/ux500/cryp/cryp.c | 393 | ||||
| -rw-r--r-- | drivers/crypto/ux500/cryp/cryp.h | 313 | ||||
| -rw-r--r-- | drivers/crypto/ux500/cryp/cryp_core.c | 1612 | ||||
| -rw-r--r-- | drivers/crypto/ux500/cryp/cryp_irq.c | 45 | ||||
| -rw-r--r-- | drivers/crypto/ux500/cryp/cryp_irq.h | 31 | ||||
| -rw-r--r-- | drivers/crypto/ux500/cryp/cryp_irqp.h | 125 | ||||
| -rw-r--r-- | drivers/crypto/ux500/cryp/cryp_p.h | 123 | ||||
| -rw-r--r-- | drivers/crypto/ux500/hash/Makefile | 11 | ||||
| -rw-r--r-- | drivers/crypto/ux500/hash/hash_alg.h | 398 | ||||
| -rw-r--r-- | drivers/crypto/ux500/hash/hash_core.c | 1973 |
13 files changed, 5074 insertions, 0 deletions
diff --git a/drivers/crypto/ux500/Kconfig b/drivers/crypto/ux500/Kconfig new file mode 100644 index 000000000..f56d65c56 --- /dev/null +++ b/drivers/crypto/ux500/Kconfig @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Copyright (C) ST-Ericsson SA 2010 +# Author: Shujuan Chen (shujuan.chen@stericsson.com) +# + +config CRYPTO_DEV_UX500_CRYP + tristate "UX500 crypto driver for CRYP block" + depends on CRYPTO_DEV_UX500 + select CRYPTO_ALGAPI + select CRYPTO_SKCIPHER + select CRYPTO_LIB_DES + help + This selects the crypto driver for the UX500_CRYP hardware. It supports + AES-ECB, CBC and CTR with keys sizes of 128, 192 and 256 bit sizes. + +config CRYPTO_DEV_UX500_HASH + tristate "UX500 crypto driver for HASH block" + depends on CRYPTO_DEV_UX500 + select CRYPTO_HASH + select CRYPTO_SHA1 + select CRYPTO_SHA256 + help + This selects the hash driver for the UX500_HASH hardware. + Depends on UX500/STM DMA if running in DMA mode. + +config CRYPTO_DEV_UX500_DEBUG + bool "Activate ux500 platform debug-mode for crypto and hash block" + depends on CRYPTO_DEV_UX500_CRYP || CRYPTO_DEV_UX500_HASH + help + Say Y if you want to add debug prints to ux500_hash and + ux500_cryp devices. diff --git a/drivers/crypto/ux500/Makefile b/drivers/crypto/ux500/Makefile new file mode 100644 index 000000000..f014eb017 --- /dev/null +++ b/drivers/crypto/ux500/Makefile @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Copyright (C) ST-Ericsson SA 2010 +# Author: Shujuan Chen (shujuan.chen@stericsson.com) +# + +obj-$(CONFIG_CRYPTO_DEV_UX500_HASH) += hash/ +obj-$(CONFIG_CRYPTO_DEV_UX500_CRYP) += cryp/ diff --git a/drivers/crypto/ux500/cryp/Makefile b/drivers/crypto/ux500/cryp/Makefile new file mode 100644 index 000000000..3e67531f4 --- /dev/null +++ b/drivers/crypto/ux500/cryp/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0-only +#/* +# * Copyright (C) ST-Ericsson SA 2010 +# * Author: shujuan.chen@stericsson.com for ST-Ericsson. +# */ + +ccflags-$(CONFIG_CRYPTO_DEV_UX500_DEBUG) += -DDEBUG + +obj-$(CONFIG_CRYPTO_DEV_UX500_CRYP) += ux500_cryp.o +ux500_cryp-objs := cryp.o cryp_irq.o cryp_core.o diff --git a/drivers/crypto/ux500/cryp/cryp.c b/drivers/crypto/ux500/cryp/cryp.c new file mode 100644 index 000000000..9866c2a5e --- /dev/null +++ b/drivers/crypto/ux500/cryp/cryp.c @@ -0,0 +1,393 @@ +// SPDX-License-Identifier: GPL-2.0-only +/** + * Copyright (C) ST-Ericsson SA 2010 + * Author: Shujuan Chen <shujuan.chen@stericsson.com> for ST-Ericsson. + * Author: Jonas Linde <jonas.linde@stericsson.com> for ST-Ericsson. + * Author: Niklas Hernaeus <niklas.hernaeus@stericsson.com> for ST-Ericsson. + * Author: Joakim Bech <joakim.xx.bech@stericsson.com> for ST-Ericsson. + * Author: Berne Hebark <berne.herbark@stericsson.com> for ST-Ericsson. + */ + +#include <linux/errno.h> +#include <linux/kernel.h> +#include <linux/types.h> + +#include "cryp_p.h" +#include "cryp.h" + +/** + * cryp_wait_until_done - wait until the device logic is not busy + */ +void cryp_wait_until_done(struct cryp_device_data *device_data) +{ + while (cryp_is_logic_busy(device_data)) + cpu_relax(); +} + +/** + * cryp_check - This routine checks Peripheral and PCell Id + * @device_data: Pointer to the device data struct for base address. + */ +int cryp_check(struct cryp_device_data *device_data) +{ + int peripheralid2 = 0; + + if (NULL == device_data) + return -EINVAL; + + peripheralid2 = readl_relaxed(&device_data->base->periphId2); + + if (peripheralid2 != CRYP_PERIPHERAL_ID2_DB8500) + return -EPERM; + + /* Check Peripheral and Pcell Id Register for CRYP */ + if ((CRYP_PERIPHERAL_ID0 == + readl_relaxed(&device_data->base->periphId0)) + && (CRYP_PERIPHERAL_ID1 == + readl_relaxed(&device_data->base->periphId1)) + && (CRYP_PERIPHERAL_ID3 == + readl_relaxed(&device_data->base->periphId3)) + && (CRYP_PCELL_ID0 == + readl_relaxed(&device_data->base->pcellId0)) + && (CRYP_PCELL_ID1 == + readl_relaxed(&device_data->base->pcellId1)) + && (CRYP_PCELL_ID2 == + readl_relaxed(&device_data->base->pcellId2)) + && (CRYP_PCELL_ID3 == + readl_relaxed(&device_data->base->pcellId3))) { + return 0; + } + + return -EPERM; +} + +/** + * cryp_activity - This routine enables/disable the cryptography function. + * @device_data: Pointer to the device data struct for base address. + * @cryp_crypen: Enable/Disable functionality + */ +void cryp_activity(struct cryp_device_data *device_data, + enum cryp_crypen cryp_crypen) +{ + CRYP_PUT_BITS(&device_data->base->cr, + cryp_crypen, + CRYP_CR_CRYPEN_POS, + CRYP_CR_CRYPEN_MASK); +} + +/** + * cryp_flush_inoutfifo - Resets both the input and the output FIFOs + * @device_data: Pointer to the device data struct for base address. + */ +void cryp_flush_inoutfifo(struct cryp_device_data *device_data) +{ + /* + * We always need to disable the hardware before trying to flush the + * FIFO. This is something that isn't written in the design + * specification, but we have been informed by the hardware designers + * that this must be done. + */ + cryp_activity(device_data, CRYP_CRYPEN_DISABLE); + cryp_wait_until_done(device_data); + + CRYP_SET_BITS(&device_data->base->cr, CRYP_CR_FFLUSH_MASK); + /* + * CRYP_SR_INFIFO_READY_MASK is the expected value on the status + * register when starting a new calculation, which means Input FIFO is + * not full and input FIFO is empty. + */ + while (readl_relaxed(&device_data->base->sr) != + CRYP_SR_INFIFO_READY_MASK) + cpu_relax(); +} + +/** + * cryp_set_configuration - This routine set the cr CRYP IP + * @device_data: Pointer to the device data struct for base address. + * @cryp_config: Pointer to the configuration parameter + * @control_register: The control register to be written later on. + */ +int cryp_set_configuration(struct cryp_device_data *device_data, + struct cryp_config *cryp_config, + u32 *control_register) +{ + u32 cr_for_kse; + + if (NULL == device_data || NULL == cryp_config) + return -EINVAL; + + *control_register |= (cryp_config->keysize << CRYP_CR_KEYSIZE_POS); + + /* Prepare key for decryption in AES_ECB and AES_CBC mode. */ + if ((CRYP_ALGORITHM_DECRYPT == cryp_config->algodir) && + ((CRYP_ALGO_AES_ECB == cryp_config->algomode) || + (CRYP_ALGO_AES_CBC == cryp_config->algomode))) { + cr_for_kse = *control_register; + /* + * This seems a bit odd, but it is indeed needed to set this to + * encrypt even though it is a decryption that we are doing. It + * also mentioned in the design spec that you need to do this. + * After the keyprepartion for decrypting is done you should set + * algodir back to decryption, which is done outside this if + * statement. + * + * According to design specification we should set mode ECB + * during key preparation even though we might be running CBC + * when enter this function. + * + * Writing to KSE_ENABLED will drop CRYPEN when key preparation + * is done. Therefore we need to set CRYPEN again outside this + * if statement when running decryption. + */ + cr_for_kse |= ((CRYP_ALGORITHM_ENCRYPT << CRYP_CR_ALGODIR_POS) | + (CRYP_ALGO_AES_ECB << CRYP_CR_ALGOMODE_POS) | + (CRYP_CRYPEN_ENABLE << CRYP_CR_CRYPEN_POS) | + (KSE_ENABLED << CRYP_CR_KSE_POS)); + + writel_relaxed(cr_for_kse, &device_data->base->cr); + cryp_wait_until_done(device_data); + } + + *control_register |= + ((cryp_config->algomode << CRYP_CR_ALGOMODE_POS) | + (cryp_config->algodir << CRYP_CR_ALGODIR_POS)); + + return 0; +} + +/** + * cryp_configure_protection - set the protection bits in the CRYP logic. + * @device_data: Pointer to the device data struct for base address. + * @p_protect_config: Pointer to the protection mode and + * secure mode configuration + */ +int cryp_configure_protection(struct cryp_device_data *device_data, + struct cryp_protection_config *p_protect_config) +{ + if (NULL == p_protect_config) + return -EINVAL; + + CRYP_WRITE_BIT(&device_data->base->cr, + (u32) p_protect_config->secure_access, + CRYP_CR_SECURE_MASK); + CRYP_PUT_BITS(&device_data->base->cr, + p_protect_config->privilege_access, + CRYP_CR_PRLG_POS, + CRYP_CR_PRLG_MASK); + + return 0; +} + +/** + * cryp_is_logic_busy - returns the busy status of the CRYP logic + * @device_data: Pointer to the device data struct for base address. + */ +int cryp_is_logic_busy(struct cryp_device_data *device_data) +{ + return CRYP_TEST_BITS(&device_data->base->sr, + CRYP_SR_BUSY_MASK); +} + +/** + * cryp_configure_for_dma - configures the CRYP IP for DMA operation + * @device_data: Pointer to the device data struct for base address. + * @dma_req: Specifies the DMA request type value. + */ +void cryp_configure_for_dma(struct cryp_device_data *device_data, + enum cryp_dma_req_type dma_req) +{ + CRYP_SET_BITS(&device_data->base->dmacr, + (u32) dma_req); +} + +/** + * cryp_configure_key_values - configures the key values for CRYP operations + * @device_data: Pointer to the device data struct for base address. + * @key_reg_index: Key value index register + * @key_value: The key value struct + */ +int cryp_configure_key_values(struct cryp_device_data *device_data, + enum cryp_key_reg_index key_reg_index, + struct cryp_key_value key_value) +{ + while (cryp_is_logic_busy(device_data)) + cpu_relax(); + + switch (key_reg_index) { + case CRYP_KEY_REG_1: + writel_relaxed(key_value.key_value_left, + &device_data->base->key_1_l); + writel_relaxed(key_value.key_value_right, + &device_data->base->key_1_r); + break; + case CRYP_KEY_REG_2: + writel_relaxed(key_value.key_value_left, + &device_data->base->key_2_l); + writel_relaxed(key_value.key_value_right, + &device_data->base->key_2_r); + break; + case CRYP_KEY_REG_3: + writel_relaxed(key_value.key_value_left, + &device_data->base->key_3_l); + writel_relaxed(key_value.key_value_right, + &device_data->base->key_3_r); + break; + case CRYP_KEY_REG_4: + writel_relaxed(key_value.key_value_left, + &device_data->base->key_4_l); + writel_relaxed(key_value.key_value_right, + &device_data->base->key_4_r); + break; + default: + return -EINVAL; + } + + return 0; +} + +/** + * cryp_configure_init_vector - configures the initialization vector register + * @device_data: Pointer to the device data struct for base address. + * @init_vector_index: Specifies the index of the init vector. + * @init_vector_value: Specifies the value for the init vector. + */ +int cryp_configure_init_vector(struct cryp_device_data *device_data, + enum cryp_init_vector_index + init_vector_index, + struct cryp_init_vector_value + init_vector_value) +{ + while (cryp_is_logic_busy(device_data)) + cpu_relax(); + + switch (init_vector_index) { + case CRYP_INIT_VECTOR_INDEX_0: + writel_relaxed(init_vector_value.init_value_left, + &device_data->base->init_vect_0_l); + writel_relaxed(init_vector_value.init_value_right, + &device_data->base->init_vect_0_r); + break; + case CRYP_INIT_VECTOR_INDEX_1: + writel_relaxed(init_vector_value.init_value_left, + &device_data->base->init_vect_1_l); + writel_relaxed(init_vector_value.init_value_right, + &device_data->base->init_vect_1_r); + break; + default: + return -EINVAL; + } + + return 0; +} + +/** + * cryp_save_device_context - Store hardware registers and + * other device context parameter + * @device_data: Pointer to the device data struct for base address. + * @ctx: Crypto device context + */ +void cryp_save_device_context(struct cryp_device_data *device_data, + struct cryp_device_context *ctx, + int cryp_mode) +{ + enum cryp_algo_mode algomode; + struct cryp_register __iomem *src_reg = device_data->base; + struct cryp_config *config = + (struct cryp_config *)device_data->current_ctx; + + /* + * Always start by disable the hardware and wait for it to finish the + * ongoing calculations before trying to reprogram it. + */ + cryp_activity(device_data, CRYP_CRYPEN_DISABLE); + cryp_wait_until_done(device_data); + + if (cryp_mode == CRYP_MODE_DMA) + cryp_configure_for_dma(device_data, CRYP_DMA_DISABLE_BOTH); + + if (CRYP_TEST_BITS(&src_reg->sr, CRYP_SR_IFEM_MASK) == 0) + ctx->din = readl_relaxed(&src_reg->din); + + ctx->cr = readl_relaxed(&src_reg->cr) & CRYP_CR_CONTEXT_SAVE_MASK; + + switch (config->keysize) { + case CRYP_KEY_SIZE_256: + ctx->key_4_l = readl_relaxed(&src_reg->key_4_l); + ctx->key_4_r = readl_relaxed(&src_reg->key_4_r); + fallthrough; + + case CRYP_KEY_SIZE_192: + ctx->key_3_l = readl_relaxed(&src_reg->key_3_l); + ctx->key_3_r = readl_relaxed(&src_reg->key_3_r); + fallthrough; + + case CRYP_KEY_SIZE_128: + ctx->key_2_l = readl_relaxed(&src_reg->key_2_l); + ctx->key_2_r = readl_relaxed(&src_reg->key_2_r); + fallthrough; + + default: + ctx->key_1_l = readl_relaxed(&src_reg->key_1_l); + ctx->key_1_r = readl_relaxed(&src_reg->key_1_r); + } + + /* Save IV for CBC mode for both AES and DES. */ + algomode = ((ctx->cr & CRYP_CR_ALGOMODE_MASK) >> CRYP_CR_ALGOMODE_POS); + if (algomode == CRYP_ALGO_TDES_CBC || + algomode == CRYP_ALGO_DES_CBC || + algomode == CRYP_ALGO_AES_CBC) { + ctx->init_vect_0_l = readl_relaxed(&src_reg->init_vect_0_l); + ctx->init_vect_0_r = readl_relaxed(&src_reg->init_vect_0_r); + ctx->init_vect_1_l = readl_relaxed(&src_reg->init_vect_1_l); + ctx->init_vect_1_r = readl_relaxed(&src_reg->init_vect_1_r); + } +} + +/** + * cryp_restore_device_context - Restore hardware registers and + * other device context parameter + * @device_data: Pointer to the device data struct for base address. + * @ctx: Crypto device context + */ +void cryp_restore_device_context(struct cryp_device_data *device_data, + struct cryp_device_context *ctx) +{ + struct cryp_register __iomem *reg = device_data->base; + struct cryp_config *config = + (struct cryp_config *)device_data->current_ctx; + + /* + * Fall through for all items in switch statement. DES is captured in + * the default. + */ + switch (config->keysize) { + case CRYP_KEY_SIZE_256: + writel_relaxed(ctx->key_4_l, ®->key_4_l); + writel_relaxed(ctx->key_4_r, ®->key_4_r); + fallthrough; + + case CRYP_KEY_SIZE_192: + writel_relaxed(ctx->key_3_l, ®->key_3_l); + writel_relaxed(ctx->key_3_r, ®->key_3_r); + fallthrough; + + case CRYP_KEY_SIZE_128: + writel_relaxed(ctx->key_2_l, ®->key_2_l); + writel_relaxed(ctx->key_2_r, ®->key_2_r); + fallthrough; + + default: + writel_relaxed(ctx->key_1_l, ®->key_1_l); + writel_relaxed(ctx->key_1_r, ®->key_1_r); + } + + /* Restore IV for CBC mode for AES and DES. */ + if (config->algomode == CRYP_ALGO_TDES_CBC || + config->algomode == CRYP_ALGO_DES_CBC || + config->algomode == CRYP_ALGO_AES_CBC) { + writel_relaxed(ctx->init_vect_0_l, ®->init_vect_0_l); + writel_relaxed(ctx->init_vect_0_r, ®->init_vect_0_r); + writel_relaxed(ctx->init_vect_1_l, ®->init_vect_1_l); + writel_relaxed(ctx->init_vect_1_r, ®->init_vect_1_r); + } +} diff --git a/drivers/crypto/ux500/cryp/cryp.h b/drivers/crypto/ux500/cryp/cryp.h new file mode 100644 index 000000000..8da7f87b3 --- /dev/null +++ b/drivers/crypto/ux500/cryp/cryp.h @@ -0,0 +1,313 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/** + * Copyright (C) ST-Ericsson SA 2010 + * Author: Shujuan Chen <shujuan.chen@stericsson.com> for ST-Ericsson. + * Author: Jonas Linde <jonas.linde@stericsson.com> for ST-Ericsson. + * Author: Joakim Bech <joakim.xx.bech@stericsson.com> for ST-Ericsson. + * Author: Berne Hebark <berne.herbark@stericsson.com> for ST-Ericsson. + * Author: Niklas Hernaeus <niklas.hernaeus@stericsson.com> for ST-Ericsson. + */ + +#ifndef _CRYP_H_ +#define _CRYP_H_ + +#include <linux/completion.h> +#include <linux/dmaengine.h> +#include <linux/klist.h> +#include <linux/mutex.h> + +#define DEV_DBG_NAME "crypX crypX:" + +/* CRYP enable/disable */ +enum cryp_crypen { + CRYP_CRYPEN_DISABLE = 0, + CRYP_CRYPEN_ENABLE = 1 +}; + +/* CRYP Start Computation enable/disable */ +enum cryp_start { + CRYP_START_DISABLE = 0, + CRYP_START_ENABLE = 1 +}; + +/* CRYP Init Signal enable/disable */ +enum cryp_init { + CRYP_INIT_DISABLE = 0, + CRYP_INIT_ENABLE = 1 +}; + +/* Cryp State enable/disable */ +enum cryp_state { + CRYP_STATE_DISABLE = 0, + CRYP_STATE_ENABLE = 1 +}; + +/* Key preparation bit enable */ +enum cryp_key_prep { + KSE_DISABLED = 0, + KSE_ENABLED = 1 +}; + +/* Key size for AES */ +#define CRYP_KEY_SIZE_128 (0) +#define CRYP_KEY_SIZE_192 (1) +#define CRYP_KEY_SIZE_256 (2) + +/* AES modes */ +enum cryp_algo_mode { + CRYP_ALGO_TDES_ECB, + CRYP_ALGO_TDES_CBC, + CRYP_ALGO_DES_ECB, + CRYP_ALGO_DES_CBC, + CRYP_ALGO_AES_ECB, + CRYP_ALGO_AES_CBC, + CRYP_ALGO_AES_CTR, + CRYP_ALGO_AES_XTS +}; + +/* Cryp Encryption or Decryption */ +enum cryp_algorithm_dir { + CRYP_ALGORITHM_ENCRYPT, + CRYP_ALGORITHM_DECRYPT +}; + +/* Hardware access method */ +enum cryp_mode { + CRYP_MODE_POLLING, + CRYP_MODE_INTERRUPT, + CRYP_MODE_DMA +}; + +/** + * struct cryp_config - + * @keysize: Key size for AES + * @algomode: AES modes + * @algodir: Cryp Encryption or Decryption + * + * CRYP configuration structure to be passed to set configuration + */ +struct cryp_config { + int keysize; + enum cryp_algo_mode algomode; + enum cryp_algorithm_dir algodir; +}; + +/** + * struct cryp_protection_config - + * @privilege_access: Privileged cryp state enable/disable + * @secure_access: Secure cryp state enable/disable + * + * Protection configuration structure for setting privilage access + */ +struct cryp_protection_config { + enum cryp_state privilege_access; + enum cryp_state secure_access; +}; + +/* Cryp status */ +enum cryp_status_id { + CRYP_STATUS_BUSY = 0x10, + CRYP_STATUS_OUTPUT_FIFO_FULL = 0x08, + CRYP_STATUS_OUTPUT_FIFO_NOT_EMPTY = 0x04, + CRYP_STATUS_INPUT_FIFO_NOT_FULL = 0x02, + CRYP_STATUS_INPUT_FIFO_EMPTY = 0x01 +}; + +/* Cryp DMA interface */ +#define CRYP_DMA_TX_FIFO 0x08 +#define CRYP_DMA_RX_FIFO 0x10 + +enum cryp_dma_req_type { + CRYP_DMA_DISABLE_BOTH, + CRYP_DMA_ENABLE_IN_DATA, + CRYP_DMA_ENABLE_OUT_DATA, + CRYP_DMA_ENABLE_BOTH_DIRECTIONS +}; + +enum cryp_dma_channel { + CRYP_DMA_RX = 0, + CRYP_DMA_TX +}; + +/* Key registers */ +enum cryp_key_reg_index { + CRYP_KEY_REG_1, + CRYP_KEY_REG_2, + CRYP_KEY_REG_3, + CRYP_KEY_REG_4 +}; + +/* Key register left and right */ +struct cryp_key_value { + u32 key_value_left; + u32 key_value_right; +}; + +/* Cryp Initialization structure */ +enum cryp_init_vector_index { + CRYP_INIT_VECTOR_INDEX_0, + CRYP_INIT_VECTOR_INDEX_1 +}; + +/* struct cryp_init_vector_value - + * @init_value_left + * @init_value_right + * */ +struct cryp_init_vector_value { + u32 init_value_left; + u32 init_value_right; +}; + +/** + * struct cryp_device_context - structure for a cryp context. + * @cr: control register + * @dmacr: DMA control register + * @imsc: Interrupt mask set/clear register + * @key_1_l: Key 1l register + * @key_1_r: Key 1r register + * @key_2_l: Key 2l register + * @key_2_r: Key 2r register + * @key_3_l: Key 3l register + * @key_3_r: Key 3r register + * @key_4_l: Key 4l register + * @key_4_r: Key 4r register + * @init_vect_0_l: Initialization vector 0l register + * @init_vect_0_r: Initialization vector 0r register + * @init_vect_1_l: Initialization vector 1l register + * @init_vect_1_r: Initialization vector 0r register + * @din: Data in register + * @dout: Data out register + * + * CRYP power management specifc structure. + */ +struct cryp_device_context { + u32 cr; + u32 dmacr; + u32 imsc; + + u32 key_1_l; + u32 key_1_r; + u32 key_2_l; + u32 key_2_r; + u32 key_3_l; + u32 key_3_r; + u32 key_4_l; + u32 key_4_r; + + u32 init_vect_0_l; + u32 init_vect_0_r; + u32 init_vect_1_l; + u32 init_vect_1_r; + + u32 din; + u32 dout; +}; + +struct cryp_dma { + dma_cap_mask_t mask; + struct completion cryp_dma_complete; + struct dma_chan *chan_cryp2mem; + struct dma_chan *chan_mem2cryp; + struct stedma40_chan_cfg *cfg_cryp2mem; + struct stedma40_chan_cfg *cfg_mem2cryp; + int sg_src_len; + int sg_dst_len; + struct scatterlist *sg_src; + struct scatterlist *sg_dst; + int nents_src; + int nents_dst; +}; + +/** + * struct cryp_device_data - structure for a cryp device. + * @base: Pointer to virtual base address of the cryp device. + * @phybase: Pointer to physical memory location of the cryp device. + * @dev: Pointer to the devices dev structure. + * @clk: Pointer to the device's clock control. + * @pwr_regulator: Pointer to the device's power control. + * @power_status: Current status of the power. + * @ctx_lock: Lock for current_ctx. + * @current_ctx: Pointer to the currently allocated context. + * @list_node: For inclusion into a klist. + * @dma: The dma structure holding channel configuration. + * @power_state: TRUE = power state on, FALSE = power state off. + * @power_state_spinlock: Spinlock for power_state. + * @restore_dev_ctx: TRUE = saved ctx, FALSE = no saved ctx. + */ +struct cryp_device_data { + struct cryp_register __iomem *base; + phys_addr_t phybase; + struct device *dev; + struct clk *clk; + struct regulator *pwr_regulator; + int power_status; + spinlock_t ctx_lock; + struct cryp_ctx *current_ctx; + struct klist_node list_node; + struct cryp_dma dma; + bool power_state; + spinlock_t power_state_spinlock; + bool restore_dev_ctx; +}; + +void cryp_wait_until_done(struct cryp_device_data *device_data); + +/* Initialization functions */ + +int cryp_check(struct cryp_device_data *device_data); + +void cryp_activity(struct cryp_device_data *device_data, + enum cryp_crypen cryp_crypen); + +void cryp_flush_inoutfifo(struct cryp_device_data *device_data); + +int cryp_set_configuration(struct cryp_device_data *device_data, + struct cryp_config *cryp_config, + u32 *control_register); + +void cryp_configure_for_dma(struct cryp_device_data *device_data, + enum cryp_dma_req_type dma_req); + +int cryp_configure_key_values(struct cryp_device_data *device_data, + enum cryp_key_reg_index key_reg_index, + struct cryp_key_value key_value); + +int cryp_configure_init_vector(struct cryp_device_data *device_data, + enum cryp_init_vector_index + init_vector_index, + struct cryp_init_vector_value + init_vector_value); + +int cryp_configure_protection(struct cryp_device_data *device_data, + struct cryp_protection_config *p_protect_config); + +/* Power management funtions */ +void cryp_save_device_context(struct cryp_device_data *device_data, + struct cryp_device_context *ctx, + int cryp_mode); + +void cryp_restore_device_context(struct cryp_device_data *device_data, + struct cryp_device_context *ctx); + +/* Data transfer and status bits. */ +int cryp_is_logic_busy(struct cryp_device_data *device_data); + +int cryp_get_status(struct cryp_device_data *device_data); + +/** + * cryp_write_indata - This routine writes 32 bit data into the data input + * register of the cryptography IP. + * @device_data: Pointer to the device data struct for base address. + * @write_data: Data to write. + */ +int cryp_write_indata(struct cryp_device_data *device_data, u32 write_data); + +/** + * cryp_read_outdata - This routine reads the data from the data output + * register of the CRYP logic + * @device_data: Pointer to the device data struct for base address. + * @read_data: Read the data from the output FIFO. + */ +int cryp_read_outdata(struct cryp_device_data *device_data, u32 *read_data); + +#endif /* _CRYP_H_ */ diff --git a/drivers/crypto/ux500/cryp/cryp_core.c b/drivers/crypto/ux500/cryp/cryp_core.c new file mode 100644 index 000000000..c3adeb2e5 --- /dev/null +++ b/drivers/crypto/ux500/cryp/cryp_core.c @@ -0,0 +1,1612 @@ +// SPDX-License-Identifier: GPL-2.0-only +/** + * Copyright (C) ST-Ericsson SA 2010 + * Author: Shujuan Chen <shujuan.chen@stericsson.com> for ST-Ericsson. + * Author: Joakim Bech <joakim.xx.bech@stericsson.com> for ST-Ericsson. + * Author: Berne Hebark <berne.herbark@stericsson.com> for ST-Ericsson. + * Author: Niklas Hernaeus <niklas.hernaeus@stericsson.com> for ST-Ericsson. + * Author: Jonas Linde <jonas.linde@stericsson.com> for ST-Ericsson. + * Author: Andreas Westin <andreas.westin@stericsson.com> for ST-Ericsson. + */ + +#include <linux/clk.h> +#include <linux/completion.h> +#include <linux/device.h> +#include <linux/dma-mapping.h> +#include <linux/dmaengine.h> +#include <linux/err.h> +#include <linux/errno.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/irqreturn.h> +#include <linux/kernel.h> +#include <linux/klist.h> +#include <linux/module.h> +#include <linux/mod_devicetable.h> +#include <linux/platform_device.h> +#include <linux/regulator/consumer.h> +#include <linux/semaphore.h> +#include <linux/platform_data/dma-ste-dma40.h> + +#include <crypto/aes.h> +#include <crypto/ctr.h> +#include <crypto/internal/des.h> +#include <crypto/internal/skcipher.h> +#include <crypto/scatterwalk.h> + +#include <linux/platform_data/crypto-ux500.h> + +#include "cryp_p.h" +#include "cryp.h" + +#define CRYP_MAX_KEY_SIZE 32 +#define BYTES_PER_WORD 4 + +static int cryp_mode; +static atomic_t session_id; + +static struct stedma40_chan_cfg *mem_to_engine; +static struct stedma40_chan_cfg *engine_to_mem; + +/** + * struct cryp_driver_data - data specific to the driver. + * + * @device_list: A list of registered devices to choose from. + * @device_allocation: A semaphore initialized with number of devices. + */ +struct cryp_driver_data { + struct klist device_list; + struct semaphore device_allocation; +}; + +/** + * struct cryp_ctx - Crypto context + * @config: Crypto mode. + * @key[CRYP_MAX_KEY_SIZE]: Key. + * @keylen: Length of key. + * @iv: Pointer to initialization vector. + * @indata: Pointer to indata. + * @outdata: Pointer to outdata. + * @datalen: Length of indata. + * @outlen: Length of outdata. + * @blocksize: Size of blocks. + * @updated: Updated flag. + * @dev_ctx: Device dependent context. + * @device: Pointer to the device. + */ +struct cryp_ctx { + struct cryp_config config; + u8 key[CRYP_MAX_KEY_SIZE]; + u32 keylen; + u8 *iv; + const u8 *indata; + u8 *outdata; + u32 datalen; + u32 outlen; + u32 blocksize; + u8 updated; + struct cryp_device_context dev_ctx; + struct cryp_device_data *device; + u32 session_id; +}; + +static struct cryp_driver_data driver_data; + +/** + * swap_bits_in_byte - mirror the bits in a byte + * @b: the byte to be mirrored + * + * The bits are swapped the following way: + * Byte b include bits 0-7, nibble 1 (n1) include bits 0-3 and + * nibble 2 (n2) bits 4-7. + * + * Nibble 1 (n1): + * (The "old" (moved) bit is replaced with a zero) + * 1. Move bit 6 and 7, 4 positions to the left. + * 2. Move bit 3 and 5, 2 positions to the left. + * 3. Move bit 1-4, 1 position to the left. + * + * Nibble 2 (n2): + * 1. Move bit 0 and 1, 4 positions to the right. + * 2. Move bit 2 and 4, 2 positions to the right. + * 3. Move bit 3-6, 1 position to the right. + * + * Combine the two nibbles to a complete and swapped byte. + */ + +static inline u8 swap_bits_in_byte(u8 b) +{ +#define R_SHIFT_4_MASK 0xc0 /* Bits 6 and 7, right shift 4 */ +#define R_SHIFT_2_MASK 0x28 /* (After right shift 4) Bits 3 and 5, + right shift 2 */ +#define R_SHIFT_1_MASK 0x1e /* (After right shift 2) Bits 1-4, + right shift 1 */ +#define L_SHIFT_4_MASK 0x03 /* Bits 0 and 1, left shift 4 */ +#define L_SHIFT_2_MASK 0x14 /* (After left shift 4) Bits 2 and 4, + left shift 2 */ +#define L_SHIFT_1_MASK 0x78 /* (After left shift 1) Bits 3-6, + left shift 1 */ + + u8 n1; + u8 n2; + + /* Swap most significant nibble */ + /* Right shift 4, bits 6 and 7 */ + n1 = ((b & R_SHIFT_4_MASK) >> 4) | (b & ~(R_SHIFT_4_MASK >> 4)); + /* Right shift 2, bits 3 and 5 */ + n1 = ((n1 & R_SHIFT_2_MASK) >> 2) | (n1 & ~(R_SHIFT_2_MASK >> 2)); + /* Right shift 1, bits 1-4 */ + n1 = (n1 & R_SHIFT_1_MASK) >> 1; + + /* Swap least significant nibble */ + /* Left shift 4, bits 0 and 1 */ + n2 = ((b & L_SHIFT_4_MASK) << 4) | (b & ~(L_SHIFT_4_MASK << 4)); + /* Left shift 2, bits 2 and 4 */ + n2 = ((n2 & L_SHIFT_2_MASK) << 2) | (n2 & ~(L_SHIFT_2_MASK << 2)); + /* Left shift 1, bits 3-6 */ + n2 = (n2 & L_SHIFT_1_MASK) << 1; + + return n1 | n2; +} + +static inline void swap_words_in_key_and_bits_in_byte(const u8 *in, + u8 *out, u32 len) +{ + unsigned int i = 0; + int j; + int index = 0; + + j = len - BYTES_PER_WORD; + while (j >= 0) { + for (i = 0; i < BYTES_PER_WORD; i++) { + index = len - j - BYTES_PER_WORD + i; + out[j + i] = + swap_bits_in_byte(in[index]); + } + j -= BYTES_PER_WORD; + } +} + +static void add_session_id(struct cryp_ctx *ctx) +{ + /* + * We never want 0 to be a valid value, since this is the default value + * for the software context. + */ + if (unlikely(atomic_inc_and_test(&session_id))) + atomic_inc(&session_id); + + ctx->session_id = atomic_read(&session_id); +} + +static irqreturn_t cryp_interrupt_handler(int irq, void *param) +{ + struct cryp_ctx *ctx; + int count; + struct cryp_device_data *device_data; + + if (param == NULL) { + BUG_ON(!param); + return IRQ_HANDLED; + } + + /* The device is coming from the one found in hw_crypt_noxts. */ + device_data = (struct cryp_device_data *)param; + + ctx = device_data->current_ctx; + + if (ctx == NULL) { + BUG_ON(!ctx); + return IRQ_HANDLED; + } + + dev_dbg(ctx->device->dev, "[%s] (len: %d) %s, ", __func__, ctx->outlen, + cryp_pending_irq_src(device_data, CRYP_IRQ_SRC_OUTPUT_FIFO) ? + "out" : "in"); + + if (cryp_pending_irq_src(device_data, + CRYP_IRQ_SRC_OUTPUT_FIFO)) { + if (ctx->outlen / ctx->blocksize > 0) { + count = ctx->blocksize / 4; + + readsl(&device_data->base->dout, ctx->outdata, count); + ctx->outdata += count; + ctx->outlen -= count; + + if (ctx->outlen == 0) { + cryp_disable_irq_src(device_data, + CRYP_IRQ_SRC_OUTPUT_FIFO); + } + } + } else if (cryp_pending_irq_src(device_data, + CRYP_IRQ_SRC_INPUT_FIFO)) { + if (ctx->datalen / ctx->blocksize > 0) { + count = ctx->blocksize / 4; + + writesl(&device_data->base->din, ctx->indata, count); + + ctx->indata += count; + ctx->datalen -= count; + + if (ctx->datalen == 0) + cryp_disable_irq_src(device_data, + CRYP_IRQ_SRC_INPUT_FIFO); + + if (ctx->config.algomode == CRYP_ALGO_AES_XTS) { + CRYP_PUT_BITS(&device_data->base->cr, + CRYP_START_ENABLE, + CRYP_CR_START_POS, + CRYP_CR_START_MASK); + + cryp_wait_until_done(device_data); + } + } + } + + return IRQ_HANDLED; +} + +static int mode_is_aes(enum cryp_algo_mode mode) +{ + return CRYP_ALGO_AES_ECB == mode || + CRYP_ALGO_AES_CBC == mode || + CRYP_ALGO_AES_CTR == mode || + CRYP_ALGO_AES_XTS == mode; +} + +static int cfg_iv(struct cryp_device_data *device_data, u32 left, u32 right, + enum cryp_init_vector_index index) +{ + struct cryp_init_vector_value vector_value; + + dev_dbg(device_data->dev, "[%s]", __func__); + + vector_value.init_value_left = left; + vector_value.init_value_right = right; + + return cryp_configure_init_vector(device_data, + index, + vector_value); +} + +static int cfg_ivs(struct cryp_device_data *device_data, struct cryp_ctx *ctx) +{ + int i; + int status = 0; + int num_of_regs = ctx->blocksize / 8; + __be32 *civ = (__be32 *)ctx->iv; + u32 iv[AES_BLOCK_SIZE / 4]; + + dev_dbg(device_data->dev, "[%s]", __func__); + + /* + * Since we loop on num_of_regs we need to have a check in case + * someone provides an incorrect blocksize which would force calling + * cfg_iv with i greater than 2 which is an error. + */ + if (num_of_regs > 2) { + dev_err(device_data->dev, "[%s] Incorrect blocksize %d", + __func__, ctx->blocksize); + return -EINVAL; + } + + for (i = 0; i < ctx->blocksize / 4; i++) + iv[i] = be32_to_cpup(civ + i); + + for (i = 0; i < num_of_regs; i++) { + status = cfg_iv(device_data, iv[i*2], iv[i*2+1], + (enum cryp_init_vector_index) i); + if (status != 0) + return status; + } + return status; +} + +static int set_key(struct cryp_device_data *device_data, + u32 left_key, + u32 right_key, + enum cryp_key_reg_index index) +{ + struct cryp_key_value key_value; + int cryp_error; + + dev_dbg(device_data->dev, "[%s]", __func__); + + key_value.key_value_left = left_key; + key_value.key_value_right = right_key; + + cryp_error = cryp_configure_key_values(device_data, + index, + key_value); + if (cryp_error != 0) + dev_err(device_data->dev, "[%s]: " + "cryp_configure_key_values() failed!", __func__); + + return cryp_error; +} + +static int cfg_keys(struct cryp_ctx *ctx) +{ + int i; + int num_of_regs = ctx->keylen / 8; + u32 swapped_key[CRYP_MAX_KEY_SIZE / 4]; + __be32 *ckey = (__be32 *)ctx->key; + int cryp_error = 0; + + dev_dbg(ctx->device->dev, "[%s]", __func__); + + if (mode_is_aes(ctx->config.algomode)) { + swap_words_in_key_and_bits_in_byte((u8 *)ckey, + (u8 *)swapped_key, + ctx->keylen); + } else { + for (i = 0; i < ctx->keylen / 4; i++) + swapped_key[i] = be32_to_cpup(ckey + i); + } + + for (i = 0; i < num_of_regs; i++) { + cryp_error = set_key(ctx->device, + swapped_key[i * 2], + swapped_key[i * 2 + 1], + (enum cryp_key_reg_index) i); + + if (cryp_error != 0) { + dev_err(ctx->device->dev, "[%s]: set_key() failed!", + __func__); + return cryp_error; + } + } + return cryp_error; +} + +static int cryp_setup_context(struct cryp_ctx *ctx, + struct cryp_device_data *device_data) +{ + u32 control_register = CRYP_CR_DEFAULT; + + switch (cryp_mode) { + case CRYP_MODE_INTERRUPT: + writel_relaxed(CRYP_IMSC_DEFAULT, &device_data->base->imsc); + break; + + case CRYP_MODE_DMA: + writel_relaxed(CRYP_DMACR_DEFAULT, &device_data->base->dmacr); + break; + + default: + break; + } + + if (ctx->updated == 0) { + cryp_flush_inoutfifo(device_data); + if (cfg_keys(ctx) != 0) { + dev_err(ctx->device->dev, "[%s]: cfg_keys failed!", + __func__); + return -EINVAL; + } + + if (ctx->iv && + CRYP_ALGO_AES_ECB != ctx->config.algomode && + CRYP_ALGO_DES_ECB != ctx->config.algomode && + CRYP_ALGO_TDES_ECB != ctx->config.algomode) { + if (cfg_ivs(device_data, ctx) != 0) + return -EPERM; + } + + cryp_set_configuration(device_data, &ctx->config, + &control_register); + add_session_id(ctx); + } else if (ctx->updated == 1 && + ctx->session_id != atomic_read(&session_id)) { + cryp_flush_inoutfifo(device_data); + cryp_restore_device_context(device_data, &ctx->dev_ctx); + + add_session_id(ctx); + control_register = ctx->dev_ctx.cr; + } else + control_register = ctx->dev_ctx.cr; + + writel(control_register | + (CRYP_CRYPEN_ENABLE << CRYP_CR_CRYPEN_POS), + &device_data->base->cr); + + return 0; +} + +static int cryp_get_device_data(struct cryp_ctx *ctx, + struct cryp_device_data **device_data) +{ + int ret; + struct klist_iter device_iterator; + struct klist_node *device_node; + struct cryp_device_data *local_device_data = NULL; + pr_debug(DEV_DBG_NAME " [%s]", __func__); + + /* Wait until a device is available */ + ret = down_interruptible(&driver_data.device_allocation); + if (ret) + return ret; /* Interrupted */ + + /* Select a device */ + klist_iter_init(&driver_data.device_list, &device_iterator); + + device_node = klist_next(&device_iterator); + while (device_node) { + local_device_data = container_of(device_node, + struct cryp_device_data, list_node); + spin_lock(&local_device_data->ctx_lock); + /* current_ctx allocates a device, NULL = unallocated */ + if (local_device_data->current_ctx) { + device_node = klist_next(&device_iterator); + } else { + local_device_data->current_ctx = ctx; + ctx->device = local_device_data; + spin_unlock(&local_device_data->ctx_lock); + break; + } + spin_unlock(&local_device_data->ctx_lock); + } + klist_iter_exit(&device_iterator); + + if (!device_node) { + /** + * No free device found. + * Since we allocated a device with down_interruptible, this + * should not be able to happen. + * Number of available devices, which are contained in + * device_allocation, is therefore decremented by not doing + * an up(device_allocation). + */ + return -EBUSY; + } + + *device_data = local_device_data; + + return 0; +} + +static void cryp_dma_setup_channel(struct cryp_device_data *device_data, + struct device *dev) +{ + struct dma_slave_config mem2cryp = { + .direction = DMA_MEM_TO_DEV, + .dst_addr = device_data->phybase + CRYP_DMA_TX_FIFO, + .dst_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES, + .dst_maxburst = 4, + }; + struct dma_slave_config cryp2mem = { + .direction = DMA_DEV_TO_MEM, + .src_addr = device_data->phybase + CRYP_DMA_RX_FIFO, + .src_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES, + .src_maxburst = 4, + }; + + dma_cap_zero(device_data->dma.mask); + dma_cap_set(DMA_SLAVE, device_data->dma.mask); + + device_data->dma.cfg_mem2cryp = mem_to_engine; + device_data->dma.chan_mem2cryp = + dma_request_channel(device_data->dma.mask, + stedma40_filter, + device_data->dma.cfg_mem2cryp); + + device_data->dma.cfg_cryp2mem = engine_to_mem; + device_data->dma.chan_cryp2mem = + dma_request_channel(device_data->dma.mask, + stedma40_filter, + device_data->dma.cfg_cryp2mem); + + dmaengine_slave_config(device_data->dma.chan_mem2cryp, &mem2cryp); + dmaengine_slave_config(device_data->dma.chan_cryp2mem, &cryp2mem); + + init_completion(&device_data->dma.cryp_dma_complete); +} + +static void cryp_dma_out_callback(void *data) +{ + struct cryp_ctx *ctx = (struct cryp_ctx *) data; + dev_dbg(ctx->device->dev, "[%s]: ", __func__); + + complete(&ctx->device->dma.cryp_dma_complete); +} + +static int cryp_set_dma_transfer(struct cryp_ctx *ctx, + struct scatterlist *sg, + int len, + enum dma_data_direction direction) +{ + struct dma_async_tx_descriptor *desc; + struct dma_chan *channel = NULL; + dma_cookie_t cookie; + + dev_dbg(ctx->device->dev, "[%s]: ", __func__); + + if (unlikely(!IS_ALIGNED((unsigned long)sg, 4))) { + dev_err(ctx->device->dev, "[%s]: Data in sg list isn't " + "aligned! Addr: 0x%08lx", __func__, (unsigned long)sg); + return -EFAULT; + } + + switch (direction) { + case DMA_TO_DEVICE: + channel = ctx->device->dma.chan_mem2cryp; + ctx->device->dma.sg_src = sg; + ctx->device->dma.sg_src_len = dma_map_sg(channel->device->dev, + ctx->device->dma.sg_src, + ctx->device->dma.nents_src, + direction); + + if (!ctx->device->dma.sg_src_len) { + dev_dbg(ctx->device->dev, + "[%s]: Could not map the sg list (TO_DEVICE)", + __func__); + return -EFAULT; + } + + dev_dbg(ctx->device->dev, "[%s]: Setting up DMA for buffer " + "(TO_DEVICE)", __func__); + + desc = dmaengine_prep_slave_sg(channel, + ctx->device->dma.sg_src, + ctx->device->dma.sg_src_len, + DMA_MEM_TO_DEV, DMA_CTRL_ACK); + break; + + case DMA_FROM_DEVICE: + channel = ctx->device->dma.chan_cryp2mem; + ctx->device->dma.sg_dst = sg; + ctx->device->dma.sg_dst_len = dma_map_sg(channel->device->dev, + ctx->device->dma.sg_dst, + ctx->device->dma.nents_dst, + direction); + + if (!ctx->device->dma.sg_dst_len) { + dev_dbg(ctx->device->dev, + "[%s]: Could not map the sg list (FROM_DEVICE)", + __func__); + return -EFAULT; + } + + dev_dbg(ctx->device->dev, "[%s]: Setting up DMA for buffer " + "(FROM_DEVICE)", __func__); + + desc = dmaengine_prep_slave_sg(channel, + ctx->device->dma.sg_dst, + ctx->device->dma.sg_dst_len, + DMA_DEV_TO_MEM, + DMA_CTRL_ACK | + DMA_PREP_INTERRUPT); + + desc->callback = cryp_dma_out_callback; + desc->callback_param = ctx; + break; + + default: + dev_dbg(ctx->device->dev, "[%s]: Invalid DMA direction", + __func__); + return -EFAULT; + } + + cookie = dmaengine_submit(desc); + if (dma_submit_error(cookie)) { + dev_dbg(ctx->device->dev, "[%s]: DMA submission failed\n", + __func__); + return cookie; + } + + dma_async_issue_pending(channel); + + return 0; +} + +static void cryp_dma_done(struct cryp_ctx *ctx) +{ + struct dma_chan *chan; + + dev_dbg(ctx->device->dev, "[%s]: ", __func__); + + chan = ctx->device->dma.chan_mem2cryp; + dmaengine_terminate_all(chan); + dma_unmap_sg(chan->device->dev, ctx->device->dma.sg_src, + ctx->device->dma.sg_src_len, DMA_TO_DEVICE); + + chan = ctx->device->dma.chan_cryp2mem; + dmaengine_terminate_all(chan); + dma_unmap_sg(chan->device->dev, ctx->device->dma.sg_dst, + ctx->device->dma.sg_dst_len, DMA_FROM_DEVICE); +} + +static int cryp_dma_write(struct cryp_ctx *ctx, struct scatterlist *sg, + int len) +{ + int error = cryp_set_dma_transfer(ctx, sg, len, DMA_TO_DEVICE); + dev_dbg(ctx->device->dev, "[%s]: ", __func__); + + if (error) { + dev_dbg(ctx->device->dev, "[%s]: cryp_set_dma_transfer() " + "failed", __func__); + return error; + } + + return len; +} + +static int cryp_dma_read(struct cryp_ctx *ctx, struct scatterlist *sg, int len) +{ + int error = cryp_set_dma_transfer(ctx, sg, len, DMA_FROM_DEVICE); + if (error) { + dev_dbg(ctx->device->dev, "[%s]: cryp_set_dma_transfer() " + "failed", __func__); + return error; + } + + return len; +} + +static void cryp_polling_mode(struct cryp_ctx *ctx, + struct cryp_device_data *device_data) +{ + int len = ctx->blocksize / BYTES_PER_WORD; + int remaining_length = ctx->datalen; + u32 *indata = (u32 *)ctx->indata; + u32 *outdata = (u32 *)ctx->outdata; + + while (remaining_length > 0) { + writesl(&device_data->base->din, indata, len); + indata += len; + remaining_length -= (len * BYTES_PER_WORD); + cryp_wait_until_done(device_data); + + readsl(&device_data->base->dout, outdata, len); + outdata += len; + cryp_wait_until_done(device_data); + } +} + +static int cryp_disable_power(struct device *dev, + struct cryp_device_data *device_data, + bool save_device_context) +{ + int ret = 0; + + dev_dbg(dev, "[%s]", __func__); + + spin_lock(&device_data->power_state_spinlock); + if (!device_data->power_state) + goto out; + + spin_lock(&device_data->ctx_lock); + if (save_device_context && device_data->current_ctx) { + cryp_save_device_context(device_data, + &device_data->current_ctx->dev_ctx, + cryp_mode); + device_data->restore_dev_ctx = true; + } + spin_unlock(&device_data->ctx_lock); + + clk_disable(device_data->clk); + ret = regulator_disable(device_data->pwr_regulator); + if (ret) + dev_err(dev, "[%s]: " + "regulator_disable() failed!", + __func__); + + device_data->power_state = false; + +out: + spin_unlock(&device_data->power_state_spinlock); + + return ret; +} + +static int cryp_enable_power( + struct device *dev, + struct cryp_device_data *device_data, + bool restore_device_context) +{ + int ret = 0; + + dev_dbg(dev, "[%s]", __func__); + + spin_lock(&device_data->power_state_spinlock); + if (!device_data->power_state) { + ret = regulator_enable(device_data->pwr_regulator); + if (ret) { + dev_err(dev, "[%s]: regulator_enable() failed!", + __func__); + goto out; + } + + ret = clk_enable(device_data->clk); + if (ret) { + dev_err(dev, "[%s]: clk_enable() failed!", + __func__); + regulator_disable(device_data->pwr_regulator); + goto out; + } + device_data->power_state = true; + } + + if (device_data->restore_dev_ctx) { + spin_lock(&device_data->ctx_lock); + if (restore_device_context && device_data->current_ctx) { + device_data->restore_dev_ctx = false; + cryp_restore_device_context(device_data, + &device_data->current_ctx->dev_ctx); + } + spin_unlock(&device_data->ctx_lock); + } +out: + spin_unlock(&device_data->power_state_spinlock); + + return ret; +} + +static int hw_crypt_noxts(struct cryp_ctx *ctx, + struct cryp_device_data *device_data) +{ + int ret = 0; + + const u8 *indata = ctx->indata; + u8 *outdata = ctx->outdata; + u32 datalen = ctx->datalen; + u32 outlen = datalen; + + pr_debug(DEV_DBG_NAME " [%s]", __func__); + + ctx->outlen = ctx->datalen; + + if (unlikely(!IS_ALIGNED((unsigned long)indata, 4))) { + pr_debug(DEV_DBG_NAME " [%s]: Data isn't aligned! Addr: " + "0x%08lx", __func__, (unsigned long)indata); + return -EINVAL; + } + + ret = cryp_setup_context(ctx, device_data); + + if (ret) + goto out; + + if (cryp_mode == CRYP_MODE_INTERRUPT) { + cryp_enable_irq_src(device_data, CRYP_IRQ_SRC_INPUT_FIFO | + CRYP_IRQ_SRC_OUTPUT_FIFO); + + /* + * ctx->outlen is decremented in the cryp_interrupt_handler + * function. We had to add cpu_relax() (barrier) to make sure + * that gcc didn't optimze away this variable. + */ + while (ctx->outlen > 0) + cpu_relax(); + } else if (cryp_mode == CRYP_MODE_POLLING || + cryp_mode == CRYP_MODE_DMA) { + /* + * The reason for having DMA in this if case is that if we are + * running cryp_mode = 2, then we separate DMA routines for + * handling cipher/plaintext > blocksize, except when + * running the normal CRYPTO_ALG_TYPE_CIPHER, then we still use + * the polling mode. Overhead of doing DMA setup eats up the + * benefits using it. + */ + cryp_polling_mode(ctx, device_data); + } else { + dev_err(ctx->device->dev, "[%s]: Invalid operation mode!", + __func__); + ret = -EPERM; + goto out; + } + + cryp_save_device_context(device_data, &ctx->dev_ctx, cryp_mode); + ctx->updated = 1; + +out: + ctx->indata = indata; + ctx->outdata = outdata; + ctx->datalen = datalen; + ctx->outlen = outlen; + + return ret; +} + +static int get_nents(struct scatterlist *sg, int nbytes) +{ + int nents = 0; + + while (nbytes > 0) { + nbytes -= sg->length; + sg = sg_next(sg); + nents++; + } + + return nents; +} + +static int ablk_dma_crypt(struct skcipher_request *areq) +{ + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); + struct cryp_ctx *ctx = crypto_skcipher_ctx(cipher); + struct cryp_device_data *device_data; + + int bytes_written = 0; + int bytes_read = 0; + int ret; + + pr_debug(DEV_DBG_NAME " [%s]", __func__); + + ctx->datalen = areq->cryptlen; + ctx->outlen = areq->cryptlen; + + ret = cryp_get_device_data(ctx, &device_data); + if (ret) + return ret; + + ret = cryp_setup_context(ctx, device_data); + if (ret) + goto out; + + /* We have the device now, so store the nents in the dma struct. */ + ctx->device->dma.nents_src = get_nents(areq->src, ctx->datalen); + ctx->device->dma.nents_dst = get_nents(areq->dst, ctx->outlen); + + /* Enable DMA in- and output. */ + cryp_configure_for_dma(device_data, CRYP_DMA_ENABLE_BOTH_DIRECTIONS); + + bytes_written = cryp_dma_write(ctx, areq->src, ctx->datalen); + bytes_read = cryp_dma_read(ctx, areq->dst, bytes_written); + + wait_for_completion(&ctx->device->dma.cryp_dma_complete); + cryp_dma_done(ctx); + + cryp_save_device_context(device_data, &ctx->dev_ctx, cryp_mode); + ctx->updated = 1; + +out: + spin_lock(&device_data->ctx_lock); + device_data->current_ctx = NULL; + ctx->device = NULL; + spin_unlock(&device_data->ctx_lock); + + /* + * The down_interruptible part for this semaphore is called in + * cryp_get_device_data. + */ + up(&driver_data.device_allocation); + + if (unlikely(bytes_written != bytes_read)) + return -EPERM; + + return 0; +} + +static int ablk_crypt(struct skcipher_request *areq) +{ + struct skcipher_walk walk; + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); + struct cryp_ctx *ctx = crypto_skcipher_ctx(cipher); + struct cryp_device_data *device_data; + unsigned long src_paddr; + unsigned long dst_paddr; + int ret; + int nbytes; + + pr_debug(DEV_DBG_NAME " [%s]", __func__); + + ret = cryp_get_device_data(ctx, &device_data); + if (ret) + goto out; + + ret = skcipher_walk_async(&walk, areq); + + if (ret) { + pr_err(DEV_DBG_NAME "[%s]: skcipher_walk_async() failed!", + __func__); + goto out; + } + + while ((nbytes = walk.nbytes) > 0) { + ctx->iv = walk.iv; + src_paddr = (page_to_phys(walk.src.phys.page) + walk.src.phys.offset); + ctx->indata = phys_to_virt(src_paddr); + + dst_paddr = (page_to_phys(walk.dst.phys.page) + walk.dst.phys.offset); + ctx->outdata = phys_to_virt(dst_paddr); + + ctx->datalen = nbytes - (nbytes % ctx->blocksize); + + ret = hw_crypt_noxts(ctx, device_data); + if (ret) + goto out; + + nbytes -= ctx->datalen; + ret = skcipher_walk_done(&walk, nbytes); + if (ret) + goto out; + } + +out: + /* Release the device */ + spin_lock(&device_data->ctx_lock); + device_data->current_ctx = NULL; + ctx->device = NULL; + spin_unlock(&device_data->ctx_lock); + + /* + * The down_interruptible part for this semaphore is called in + * cryp_get_device_data. + */ + up(&driver_data.device_allocation); + + return ret; +} + +static int aes_skcipher_setkey(struct crypto_skcipher *cipher, + const u8 *key, unsigned int keylen) +{ + struct cryp_ctx *ctx = crypto_skcipher_ctx(cipher); + + pr_debug(DEV_DBG_NAME " [%s]", __func__); + + switch (keylen) { + case AES_KEYSIZE_128: + ctx->config.keysize = CRYP_KEY_SIZE_128; + break; + + case AES_KEYSIZE_192: + ctx->config.keysize = CRYP_KEY_SIZE_192; + break; + + case AES_KEYSIZE_256: + ctx->config.keysize = CRYP_KEY_SIZE_256; + break; + + default: + pr_err(DEV_DBG_NAME "[%s]: Unknown keylen!", __func__); + return -EINVAL; + } + + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + + ctx->updated = 0; + + return 0; +} + +static int des_skcipher_setkey(struct crypto_skcipher *cipher, + const u8 *key, unsigned int keylen) +{ + struct cryp_ctx *ctx = crypto_skcipher_ctx(cipher); + int err; + + pr_debug(DEV_DBG_NAME " [%s]", __func__); + + err = verify_skcipher_des_key(cipher, key); + if (err) + return err; + + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + + ctx->updated = 0; + return 0; +} + +static int des3_skcipher_setkey(struct crypto_skcipher *cipher, + const u8 *key, unsigned int keylen) +{ + struct cryp_ctx *ctx = crypto_skcipher_ctx(cipher); + int err; + + pr_debug(DEV_DBG_NAME " [%s]", __func__); + + err = verify_skcipher_des3_key(cipher, key); + if (err) + return err; + + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + + ctx->updated = 0; + return 0; +} + +static int cryp_blk_encrypt(struct skcipher_request *areq) +{ + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); + struct cryp_ctx *ctx = crypto_skcipher_ctx(cipher); + + pr_debug(DEV_DBG_NAME " [%s]", __func__); + + ctx->config.algodir = CRYP_ALGORITHM_ENCRYPT; + + /* + * DMA does not work for DES due to a hw bug */ + if (cryp_mode == CRYP_MODE_DMA && mode_is_aes(ctx->config.algomode)) + return ablk_dma_crypt(areq); + + /* For everything except DMA, we run the non DMA version. */ + return ablk_crypt(areq); +} + +static int cryp_blk_decrypt(struct skcipher_request *areq) +{ + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); + struct cryp_ctx *ctx = crypto_skcipher_ctx(cipher); + + pr_debug(DEV_DBG_NAME " [%s]", __func__); + + ctx->config.algodir = CRYP_ALGORITHM_DECRYPT; + + /* DMA does not work for DES due to a hw bug */ + if (cryp_mode == CRYP_MODE_DMA && mode_is_aes(ctx->config.algomode)) + return ablk_dma_crypt(areq); + + /* For everything except DMA, we run the non DMA version. */ + return ablk_crypt(areq); +} + +struct cryp_algo_template { + enum cryp_algo_mode algomode; + struct skcipher_alg skcipher; +}; + +static int cryp_init_tfm(struct crypto_skcipher *tfm) +{ + struct cryp_ctx *ctx = crypto_skcipher_ctx(tfm); + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + struct cryp_algo_template *cryp_alg = container_of(alg, + struct cryp_algo_template, + skcipher); + + ctx->config.algomode = cryp_alg->algomode; + ctx->blocksize = crypto_skcipher_blocksize(tfm); + + return 0; +} + +static struct cryp_algo_template cryp_algs[] = { + { + .algomode = CRYP_ALGO_AES_ECB, + .skcipher = { + .base.cra_name = "ecb(aes)", + .base.cra_driver_name = "ecb-aes-ux500", + .base.cra_priority = 300, + .base.cra_flags = CRYPTO_ALG_ASYNC, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct cryp_ctx), + .base.cra_alignmask = 3, + .base.cra_module = THIS_MODULE, + + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = aes_skcipher_setkey, + .encrypt = cryp_blk_encrypt, + .decrypt = cryp_blk_decrypt, + .init = cryp_init_tfm, + } + }, + { + .algomode = CRYP_ALGO_AES_CBC, + .skcipher = { + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "cbc-aes-ux500", + .base.cra_priority = 300, + .base.cra_flags = CRYPTO_ALG_ASYNC, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct cryp_ctx), + .base.cra_alignmask = 3, + .base.cra_module = THIS_MODULE, + + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = aes_skcipher_setkey, + .encrypt = cryp_blk_encrypt, + .decrypt = cryp_blk_decrypt, + .init = cryp_init_tfm, + .ivsize = AES_BLOCK_SIZE, + } + }, + { + .algomode = CRYP_ALGO_AES_CTR, + .skcipher = { + .base.cra_name = "ctr(aes)", + .base.cra_driver_name = "ctr-aes-ux500", + .base.cra_priority = 300, + .base.cra_flags = CRYPTO_ALG_ASYNC, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct cryp_ctx), + .base.cra_alignmask = 3, + .base.cra_module = THIS_MODULE, + + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = aes_skcipher_setkey, + .encrypt = cryp_blk_encrypt, + .decrypt = cryp_blk_decrypt, + .init = cryp_init_tfm, + .ivsize = AES_BLOCK_SIZE, + .chunksize = AES_BLOCK_SIZE, + } + }, + { + .algomode = CRYP_ALGO_DES_ECB, + .skcipher = { + .base.cra_name = "ecb(des)", + .base.cra_driver_name = "ecb-des-ux500", + .base.cra_priority = 300, + .base.cra_flags = CRYPTO_ALG_ASYNC, + .base.cra_blocksize = DES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct cryp_ctx), + .base.cra_alignmask = 3, + .base.cra_module = THIS_MODULE, + + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .setkey = des_skcipher_setkey, + .encrypt = cryp_blk_encrypt, + .decrypt = cryp_blk_decrypt, + .init = cryp_init_tfm, + } + }, + { + .algomode = CRYP_ALGO_TDES_ECB, + .skcipher = { + .base.cra_name = "ecb(des3_ede)", + .base.cra_driver_name = "ecb-des3_ede-ux500", + .base.cra_priority = 300, + .base.cra_flags = CRYPTO_ALG_ASYNC, + .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct cryp_ctx), + .base.cra_alignmask = 3, + .base.cra_module = THIS_MODULE, + + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .setkey = des3_skcipher_setkey, + .encrypt = cryp_blk_encrypt, + .decrypt = cryp_blk_decrypt, + .init = cryp_init_tfm, + } + }, + { + .algomode = CRYP_ALGO_DES_CBC, + .skcipher = { + .base.cra_name = "cbc(des)", + .base.cra_driver_name = "cbc-des-ux500", + .base.cra_priority = 300, + .base.cra_flags = CRYPTO_ALG_ASYNC, + .base.cra_blocksize = DES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct cryp_ctx), + .base.cra_alignmask = 3, + .base.cra_module = THIS_MODULE, + + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .setkey = des_skcipher_setkey, + .encrypt = cryp_blk_encrypt, + .decrypt = cryp_blk_decrypt, + .ivsize = DES_BLOCK_SIZE, + .init = cryp_init_tfm, + } + }, + { + .algomode = CRYP_ALGO_TDES_CBC, + .skcipher = { + .base.cra_name = "cbc(des3_ede)", + .base.cra_driver_name = "cbc-des3_ede-ux500", + .base.cra_priority = 300, + .base.cra_flags = CRYPTO_ALG_ASYNC, + .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct cryp_ctx), + .base.cra_alignmask = 3, + .base.cra_module = THIS_MODULE, + + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .setkey = des3_skcipher_setkey, + .encrypt = cryp_blk_encrypt, + .decrypt = cryp_blk_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .init = cryp_init_tfm, + } + } +}; + +/** + * cryp_algs_register_all - + */ +static int cryp_algs_register_all(void) +{ + int ret; + int i; + int count; + + pr_debug("[%s]", __func__); + + for (i = 0; i < ARRAY_SIZE(cryp_algs); i++) { + ret = crypto_register_skcipher(&cryp_algs[i].skcipher); + if (ret) { + count = i; + pr_err("[%s] alg registration failed", + cryp_algs[i].skcipher.base.cra_driver_name); + goto unreg; + } + } + return 0; +unreg: + for (i = 0; i < count; i++) + crypto_unregister_skcipher(&cryp_algs[i].skcipher); + return ret; +} + +/** + * cryp_algs_unregister_all - + */ +static void cryp_algs_unregister_all(void) +{ + int i; + + pr_debug(DEV_DBG_NAME " [%s]", __func__); + + for (i = 0; i < ARRAY_SIZE(cryp_algs); i++) + crypto_unregister_skcipher(&cryp_algs[i].skcipher); +} + +static int ux500_cryp_probe(struct platform_device *pdev) +{ + int ret; + struct resource *res; + struct resource *res_irq; + struct cryp_device_data *device_data; + struct cryp_protection_config prot = { + .privilege_access = CRYP_STATE_ENABLE + }; + struct device *dev = &pdev->dev; + + dev_dbg(dev, "[%s]", __func__); + device_data = devm_kzalloc(dev, sizeof(*device_data), GFP_ATOMIC); + if (!device_data) { + ret = -ENOMEM; + goto out; + } + + device_data->dev = dev; + device_data->current_ctx = NULL; + + /* Grab the DMA configuration from platform data. */ + mem_to_engine = &((struct cryp_platform_data *) + dev->platform_data)->mem_to_engine; + engine_to_mem = &((struct cryp_platform_data *) + dev->platform_data)->engine_to_mem; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) { + dev_err(dev, "[%s]: platform_get_resource() failed", + __func__); + ret = -ENODEV; + goto out; + } + + device_data->phybase = res->start; + device_data->base = devm_ioremap_resource(dev, res); + if (IS_ERR(device_data->base)) { + dev_err(dev, "[%s]: ioremap failed!", __func__); + ret = PTR_ERR(device_data->base); + goto out; + } + + spin_lock_init(&device_data->ctx_lock); + spin_lock_init(&device_data->power_state_spinlock); + + /* Enable power for CRYP hardware block */ + device_data->pwr_regulator = regulator_get(&pdev->dev, "v-ape"); + if (IS_ERR(device_data->pwr_regulator)) { + dev_err(dev, "[%s]: could not get cryp regulator", __func__); + ret = PTR_ERR(device_data->pwr_regulator); + device_data->pwr_regulator = NULL; + goto out; + } + + /* Enable the clk for CRYP hardware block */ + device_data->clk = devm_clk_get(&pdev->dev, NULL); + if (IS_ERR(device_data->clk)) { + dev_err(dev, "[%s]: clk_get() failed!", __func__); + ret = PTR_ERR(device_data->clk); + goto out_regulator; + } + + ret = clk_prepare(device_data->clk); + if (ret) { + dev_err(dev, "[%s]: clk_prepare() failed!", __func__); + goto out_regulator; + } + + /* Enable device power (and clock) */ + ret = cryp_enable_power(device_data->dev, device_data, false); + if (ret) { + dev_err(dev, "[%s]: cryp_enable_power() failed!", __func__); + goto out_clk_unprepare; + } + + if (cryp_check(device_data)) { + dev_err(dev, "[%s]: cryp_check() failed!", __func__); + ret = -EINVAL; + goto out_power; + } + + if (cryp_configure_protection(device_data, &prot)) { + dev_err(dev, "[%s]: cryp_configure_protection() failed!", + __func__); + ret = -EINVAL; + goto out_power; + } + + res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0); + if (!res_irq) { + dev_err(dev, "[%s]: IORESOURCE_IRQ unavailable", + __func__); + ret = -ENODEV; + goto out_power; + } + + ret = devm_request_irq(&pdev->dev, res_irq->start, + cryp_interrupt_handler, 0, "cryp1", device_data); + if (ret) { + dev_err(dev, "[%s]: Unable to request IRQ", __func__); + goto out_power; + } + + if (cryp_mode == CRYP_MODE_DMA) + cryp_dma_setup_channel(device_data, dev); + + platform_set_drvdata(pdev, device_data); + + /* Put the new device into the device list... */ + klist_add_tail(&device_data->list_node, &driver_data.device_list); + + /* ... and signal that a new device is available. */ + up(&driver_data.device_allocation); + + atomic_set(&session_id, 1); + + ret = cryp_algs_register_all(); + if (ret) { + dev_err(dev, "[%s]: cryp_algs_register_all() failed!", + __func__); + goto out_power; + } + + dev_info(dev, "successfully registered\n"); + + return 0; + +out_power: + cryp_disable_power(device_data->dev, device_data, false); + +out_clk_unprepare: + clk_unprepare(device_data->clk); + +out_regulator: + regulator_put(device_data->pwr_regulator); + +out: + return ret; +} + +static int ux500_cryp_remove(struct platform_device *pdev) +{ + struct cryp_device_data *device_data; + + dev_dbg(&pdev->dev, "[%s]", __func__); + device_data = platform_get_drvdata(pdev); + if (!device_data) { + dev_err(&pdev->dev, "[%s]: platform_get_drvdata() failed!", + __func__); + return -ENOMEM; + } + + /* Try to decrease the number of available devices. */ + if (down_trylock(&driver_data.device_allocation)) + return -EBUSY; + + /* Check that the device is free */ + spin_lock(&device_data->ctx_lock); + /* current_ctx allocates a device, NULL = unallocated */ + if (device_data->current_ctx) { + /* The device is busy */ + spin_unlock(&device_data->ctx_lock); + /* Return the device to the pool. */ + up(&driver_data.device_allocation); + return -EBUSY; + } + + spin_unlock(&device_data->ctx_lock); + + /* Remove the device from the list */ + if (klist_node_attached(&device_data->list_node)) + klist_remove(&device_data->list_node); + + /* If this was the last device, remove the services */ + if (list_empty(&driver_data.device_list.k_list)) + cryp_algs_unregister_all(); + + if (cryp_disable_power(&pdev->dev, device_data, false)) + dev_err(&pdev->dev, "[%s]: cryp_disable_power() failed", + __func__); + + clk_unprepare(device_data->clk); + regulator_put(device_data->pwr_regulator); + + return 0; +} + +static void ux500_cryp_shutdown(struct platform_device *pdev) +{ + struct cryp_device_data *device_data; + + dev_dbg(&pdev->dev, "[%s]", __func__); + + device_data = platform_get_drvdata(pdev); + if (!device_data) { + dev_err(&pdev->dev, "[%s]: platform_get_drvdata() failed!", + __func__); + return; + } + + /* Check that the device is free */ + spin_lock(&device_data->ctx_lock); + /* current_ctx allocates a device, NULL = unallocated */ + if (!device_data->current_ctx) { + if (down_trylock(&driver_data.device_allocation)) + dev_dbg(&pdev->dev, "[%s]: Cryp still in use!" + "Shutting down anyway...", __func__); + /** + * (Allocate the device) + * Need to set this to non-null (dummy) value, + * to avoid usage if context switching. + */ + device_data->current_ctx++; + } + spin_unlock(&device_data->ctx_lock); + + /* Remove the device from the list */ + if (klist_node_attached(&device_data->list_node)) + klist_remove(&device_data->list_node); + + /* If this was the last device, remove the services */ + if (list_empty(&driver_data.device_list.k_list)) + cryp_algs_unregister_all(); + + if (cryp_disable_power(&pdev->dev, device_data, false)) + dev_err(&pdev->dev, "[%s]: cryp_disable_power() failed", + __func__); + +} + +#ifdef CONFIG_PM_SLEEP +static int ux500_cryp_suspend(struct device *dev) +{ + int ret; + struct platform_device *pdev = to_platform_device(dev); + struct cryp_device_data *device_data; + struct resource *res_irq; + struct cryp_ctx *temp_ctx = NULL; + + dev_dbg(dev, "[%s]", __func__); + + /* Handle state? */ + device_data = platform_get_drvdata(pdev); + if (!device_data) { + dev_err(dev, "[%s]: platform_get_drvdata() failed!", __func__); + return -ENOMEM; + } + + res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0); + if (!res_irq) + dev_err(dev, "[%s]: IORESOURCE_IRQ, unavailable", __func__); + else + disable_irq(res_irq->start); + + spin_lock(&device_data->ctx_lock); + if (!device_data->current_ctx) + device_data->current_ctx++; + spin_unlock(&device_data->ctx_lock); + + if (device_data->current_ctx == ++temp_ctx) { + if (down_interruptible(&driver_data.device_allocation)) + dev_dbg(dev, "[%s]: down_interruptible() failed", + __func__); + ret = cryp_disable_power(dev, device_data, false); + + } else + ret = cryp_disable_power(dev, device_data, true); + + if (ret) + dev_err(dev, "[%s]: cryp_disable_power()", __func__); + + return ret; +} + +static int ux500_cryp_resume(struct device *dev) +{ + int ret = 0; + struct platform_device *pdev = to_platform_device(dev); + struct cryp_device_data *device_data; + struct resource *res_irq; + struct cryp_ctx *temp_ctx = NULL; + + dev_dbg(dev, "[%s]", __func__); + + device_data = platform_get_drvdata(pdev); + if (!device_data) { + dev_err(dev, "[%s]: platform_get_drvdata() failed!", __func__); + return -ENOMEM; + } + + spin_lock(&device_data->ctx_lock); + if (device_data->current_ctx == ++temp_ctx) + device_data->current_ctx = NULL; + spin_unlock(&device_data->ctx_lock); + + + if (!device_data->current_ctx) + up(&driver_data.device_allocation); + else + ret = cryp_enable_power(dev, device_data, true); + + if (ret) + dev_err(dev, "[%s]: cryp_enable_power() failed!", __func__); + else { + res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0); + if (res_irq) + enable_irq(res_irq->start); + } + + return ret; +} +#endif + +static SIMPLE_DEV_PM_OPS(ux500_cryp_pm, ux500_cryp_suspend, ux500_cryp_resume); + +static const struct of_device_id ux500_cryp_match[] = { + { .compatible = "stericsson,ux500-cryp" }, + { }, +}; +MODULE_DEVICE_TABLE(of, ux500_cryp_match); + +static struct platform_driver cryp_driver = { + .probe = ux500_cryp_probe, + .remove = ux500_cryp_remove, + .shutdown = ux500_cryp_shutdown, + .driver = { + .name = "cryp1", + .of_match_table = ux500_cryp_match, + .pm = &ux500_cryp_pm, + } +}; + +static int __init ux500_cryp_mod_init(void) +{ + pr_debug("[%s] is called!", __func__); + klist_init(&driver_data.device_list, NULL, NULL); + /* Initialize the semaphore to 0 devices (locked state) */ + sema_init(&driver_data.device_allocation, 0); + return platform_driver_register(&cryp_driver); +} + +static void __exit ux500_cryp_mod_fini(void) +{ + pr_debug("[%s] is called!", __func__); + platform_driver_unregister(&cryp_driver); +} + +module_init(ux500_cryp_mod_init); +module_exit(ux500_cryp_mod_fini); + +module_param(cryp_mode, int, 0); + +MODULE_DESCRIPTION("Driver for ST-Ericsson UX500 CRYP crypto engine."); +MODULE_ALIAS_CRYPTO("aes-all"); +MODULE_ALIAS_CRYPTO("des-all"); + +MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/ux500/cryp/cryp_irq.c b/drivers/crypto/ux500/cryp/cryp_irq.c new file mode 100644 index 000000000..7ebde69e8 --- /dev/null +++ b/drivers/crypto/ux500/cryp/cryp_irq.c @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0-only +/** + * Copyright (C) ST-Ericsson SA 2010 + * Author: Shujuan Chen <shujuan.chen@stericsson.com> for ST-Ericsson. + * Author: Jonas Linde <jonas.linde@stericsson.com> for ST-Ericsson. + * Author: Joakim Bech <joakim.xx.bech@stericsson.com> for ST-Ericsson. + * Author: Berne Hebark <berne.herbark@stericsson.com> for ST-Ericsson. + * Author: Niklas Hernaeus <niklas.hernaeus@stericsson.com> for ST-Ericsson. + */ + +#include <linux/kernel.h> +#include <linux/bitmap.h> +#include <linux/device.h> + +#include "cryp.h" +#include "cryp_p.h" +#include "cryp_irq.h" +#include "cryp_irqp.h" + +void cryp_enable_irq_src(struct cryp_device_data *device_data, u32 irq_src) +{ + u32 i; + + dev_dbg(device_data->dev, "[%s]", __func__); + + i = readl_relaxed(&device_data->base->imsc); + i = i | irq_src; + writel_relaxed(i, &device_data->base->imsc); +} + +void cryp_disable_irq_src(struct cryp_device_data *device_data, u32 irq_src) +{ + u32 i; + + dev_dbg(device_data->dev, "[%s]", __func__); + + i = readl_relaxed(&device_data->base->imsc); + i = i & ~irq_src; + writel_relaxed(i, &device_data->base->imsc); +} + +bool cryp_pending_irq_src(struct cryp_device_data *device_data, u32 irq_src) +{ + return (readl_relaxed(&device_data->base->mis) & irq_src) > 0; +} diff --git a/drivers/crypto/ux500/cryp/cryp_irq.h b/drivers/crypto/ux500/cryp/cryp_irq.h new file mode 100644 index 000000000..1984f3010 --- /dev/null +++ b/drivers/crypto/ux500/cryp/cryp_irq.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/** + * Copyright (C) ST-Ericsson SA 2010 + * Author: Shujuan Chen <shujuan.chen@stericsson.com> for ST-Ericsson. + * Author: Jonas Linde <jonas.linde@stericsson.com> for ST-Ericsson. + * Author: Joakim Bech <joakim.xx.bech@stericsson.com> for ST-Ericsson. + * Author: Berne Hebark <berne.herbark@stericsson.com> for ST-Ericsson. + * Author: Niklas Hernaeus <niklas.hernaeus@stericsson.com> for ST-Ericsson. + */ + +#ifndef _CRYP_IRQ_H_ +#define _CRYP_IRQ_H_ + +#include "cryp.h" + +enum cryp_irq_src_id { + CRYP_IRQ_SRC_INPUT_FIFO = 0x1, + CRYP_IRQ_SRC_OUTPUT_FIFO = 0x2, + CRYP_IRQ_SRC_ALL = 0x3 +}; + +/** + * M0 Funtions + */ +void cryp_enable_irq_src(struct cryp_device_data *device_data, u32 irq_src); + +void cryp_disable_irq_src(struct cryp_device_data *device_data, u32 irq_src); + +bool cryp_pending_irq_src(struct cryp_device_data *device_data, u32 irq_src); + +#endif /* _CRYP_IRQ_H_ */ diff --git a/drivers/crypto/ux500/cryp/cryp_irqp.h b/drivers/crypto/ux500/cryp/cryp_irqp.h new file mode 100644 index 000000000..879ed68a1 --- /dev/null +++ b/drivers/crypto/ux500/cryp/cryp_irqp.h @@ -0,0 +1,125 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/** + * Copyright (C) ST-Ericsson SA 2010 + * Author: Shujuan Chen <shujuan.chen@stericsson.com> for ST-Ericsson. + * Author: Jonas Linde <jonas.linde@stericsson.com> for ST-Ericsson. + * Author: Joakim Bech <joakim.xx.bech@stericsson.com> for ST-Ericsson. + * Author: Berne Hebark <berne.herbark@stericsson.com> for ST-Ericsson. + * Author: Niklas Hernaeus <niklas.hernaeus@stericsson.com> for ST-Ericsson. + */ + +#ifndef __CRYP_IRQP_H_ +#define __CRYP_IRQP_H_ + +#include "cryp_irq.h" + +/** + * + * CRYP Registers - Offset mapping + * +-----------------+ + * 00h | CRYP_CR | Configuration register + * +-----------------+ + * 04h | CRYP_SR | Status register + * +-----------------+ + * 08h | CRYP_DIN | Data In register + * +-----------------+ + * 0ch | CRYP_DOUT | Data out register + * +-----------------+ + * 10h | CRYP_DMACR | DMA control register + * +-----------------+ + * 14h | CRYP_IMSC | IMSC + * +-----------------+ + * 18h | CRYP_RIS | Raw interrupt status + * +-----------------+ + * 1ch | CRYP_MIS | Masked interrupt status. + * +-----------------+ + * Key registers + * IVR registers + * Peripheral + * Cell IDs + * + * Refer data structure for other register map + */ + +/** + * struct cryp_register + * @cr - Configuration register + * @status - Status register + * @din - Data input register + * @din_size - Data input size register + * @dout - Data output register + * @dout_size - Data output size register + * @dmacr - Dma control register + * @imsc - Interrupt mask set/clear register + * @ris - Raw interrupt status + * @mis - Masked interrupt statu register + * @key_1_l - Key register 1 L + * @key_1_r - Key register 1 R + * @key_2_l - Key register 2 L + * @key_2_r - Key register 2 R + * @key_3_l - Key register 3 L + * @key_3_r - Key register 3 R + * @key_4_l - Key register 4 L + * @key_4_r - Key register 4 R + * @init_vect_0_l - init vector 0 L + * @init_vect_0_r - init vector 0 R + * @init_vect_1_l - init vector 1 L + * @init_vect_1_r - init vector 1 R + * @cryp_unused1 - unused registers + * @itcr - Integration test control register + * @itip - Integration test input register + * @itop - Integration test output register + * @cryp_unused2 - unused registers + * @periphId0 - FE0 CRYP Peripheral Identication Register + * @periphId1 - FE4 + * @periphId2 - FE8 + * @periphId3 - FEC + * @pcellId0 - FF0 CRYP PCell Identication Register + * @pcellId1 - FF4 + * @pcellId2 - FF8 + * @pcellId3 - FFC + */ +struct cryp_register { + u32 cr; /* Configuration register */ + u32 sr; /* Status register */ + u32 din; /* Data input register */ + u32 din_size; /* Data input size register */ + u32 dout; /* Data output register */ + u32 dout_size; /* Data output size register */ + u32 dmacr; /* Dma control register */ + u32 imsc; /* Interrupt mask set/clear register */ + u32 ris; /* Raw interrupt status */ + u32 mis; /* Masked interrupt statu register */ + + u32 key_1_l; /*Key register 1 L */ + u32 key_1_r; /*Key register 1 R */ + u32 key_2_l; /*Key register 2 L */ + u32 key_2_r; /*Key register 2 R */ + u32 key_3_l; /*Key register 3 L */ + u32 key_3_r; /*Key register 3 R */ + u32 key_4_l; /*Key register 4 L */ + u32 key_4_r; /*Key register 4 R */ + + u32 init_vect_0_l; /*init vector 0 L */ + u32 init_vect_0_r; /*init vector 0 R */ + u32 init_vect_1_l; /*init vector 1 L */ + u32 init_vect_1_r; /*init vector 1 R */ + + u32 cryp_unused1[(0x80 - 0x58) / sizeof(u32)]; /* unused registers */ + u32 itcr; /*Integration test control register */ + u32 itip; /*Integration test input register */ + u32 itop; /*Integration test output register */ + u32 cryp_unused2[(0xFE0 - 0x8C) / sizeof(u32)]; /* unused registers */ + + u32 periphId0; /* FE0 CRYP Peripheral Identication Register */ + u32 periphId1; /* FE4 */ + u32 periphId2; /* FE8 */ + u32 periphId3; /* FEC */ + + u32 pcellId0; /* FF0 CRYP PCell Identication Register */ + u32 pcellId1; /* FF4 */ + u32 pcellId2; /* FF8 */ + u32 pcellId3; /* FFC */ +}; + +#endif diff --git a/drivers/crypto/ux500/cryp/cryp_p.h b/drivers/crypto/ux500/cryp/cryp_p.h new file mode 100644 index 000000000..0df84eaa8 --- /dev/null +++ b/drivers/crypto/ux500/cryp/cryp_p.h @@ -0,0 +1,123 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/** + * Copyright (C) ST-Ericsson SA 2010 + * Author: Shujuan Chen <shujuan.chen@stericsson.com> for ST-Ericsson. + * Author: Jonas Linde <jonas.linde@stericsson.com> for ST-Ericsson. + * Author: Joakim Bech <joakim.xx.bech@stericsson.com> for ST-Ericsson. + * Author: Berne Hebark <berne.herbark@stericsson.com> for ST-Ericsson. + * Author: Niklas Hernaeus <niklas.hernaeus@stericsson.com> for ST-Ericsson. + */ + +#ifndef _CRYP_P_H_ +#define _CRYP_P_H_ + +#include <linux/io.h> +#include <linux/bitops.h> + +#include "cryp.h" +#include "cryp_irqp.h" + +/** + * Generic Macros + */ +#define CRYP_SET_BITS(reg_name, mask) \ + writel_relaxed((readl_relaxed(reg_name) | mask), reg_name) + +#define CRYP_WRITE_BIT(reg_name, val, mask) \ + writel_relaxed(((readl_relaxed(reg_name) & ~(mask)) |\ + ((val) & (mask))), reg_name) + +#define CRYP_TEST_BITS(reg_name, val) \ + (readl_relaxed(reg_name) & (val)) + +#define CRYP_PUT_BITS(reg, val, shift, mask) \ + writel_relaxed(((readl_relaxed(reg) & ~(mask)) | \ + (((u32)val << shift) & (mask))), reg) + +/** + * CRYP specific Macros + */ +#define CRYP_PERIPHERAL_ID0 0xE3 +#define CRYP_PERIPHERAL_ID1 0x05 + +#define CRYP_PERIPHERAL_ID2_DB8500 0x28 +#define CRYP_PERIPHERAL_ID3 0x00 + +#define CRYP_PCELL_ID0 0x0D +#define CRYP_PCELL_ID1 0xF0 +#define CRYP_PCELL_ID2 0x05 +#define CRYP_PCELL_ID3 0xB1 + +/** + * CRYP register default values + */ +#define MAX_DEVICE_SUPPORT 2 + +/* Priv set, keyrden set and datatype 8bits swapped set as default. */ +#define CRYP_CR_DEFAULT 0x0482 +#define CRYP_DMACR_DEFAULT 0x0 +#define CRYP_IMSC_DEFAULT 0x0 +#define CRYP_DIN_DEFAULT 0x0 +#define CRYP_DOUT_DEFAULT 0x0 +#define CRYP_KEY_DEFAULT 0x0 +#define CRYP_INIT_VECT_DEFAULT 0x0 + +/** + * CRYP Control register specific mask + */ +#define CRYP_CR_SECURE_MASK BIT(0) +#define CRYP_CR_PRLG_MASK BIT(1) +#define CRYP_CR_ALGODIR_MASK BIT(2) +#define CRYP_CR_ALGOMODE_MASK (BIT(5) | BIT(4) | BIT(3)) +#define CRYP_CR_DATATYPE_MASK (BIT(7) | BIT(6)) +#define CRYP_CR_KEYSIZE_MASK (BIT(9) | BIT(8)) +#define CRYP_CR_KEYRDEN_MASK BIT(10) +#define CRYP_CR_KSE_MASK BIT(11) +#define CRYP_CR_START_MASK BIT(12) +#define CRYP_CR_INIT_MASK BIT(13) +#define CRYP_CR_FFLUSH_MASK BIT(14) +#define CRYP_CR_CRYPEN_MASK BIT(15) +#define CRYP_CR_CONTEXT_SAVE_MASK (CRYP_CR_SECURE_MASK |\ + CRYP_CR_PRLG_MASK |\ + CRYP_CR_ALGODIR_MASK |\ + CRYP_CR_ALGOMODE_MASK |\ + CRYP_CR_DATATYPE_MASK |\ + CRYP_CR_KEYSIZE_MASK |\ + CRYP_CR_KEYRDEN_MASK |\ + CRYP_CR_DATATYPE_MASK) + + +#define CRYP_SR_INFIFO_READY_MASK (BIT(0) | BIT(1)) +#define CRYP_SR_IFEM_MASK BIT(0) +#define CRYP_SR_BUSY_MASK BIT(4) + +/** + * Bit position used while setting bits in register + */ +#define CRYP_CR_PRLG_POS 1 +#define CRYP_CR_ALGODIR_POS 2 +#define CRYP_CR_ALGOMODE_POS 3 +#define CRYP_CR_DATATYPE_POS 6 +#define CRYP_CR_KEYSIZE_POS 8 +#define CRYP_CR_KEYRDEN_POS 10 +#define CRYP_CR_KSE_POS 11 +#define CRYP_CR_START_POS 12 +#define CRYP_CR_INIT_POS 13 +#define CRYP_CR_CRYPEN_POS 15 + +#define CRYP_SR_BUSY_POS 4 + +/** + * CRYP PCRs------PC_NAND control register + * BIT_MASK + */ +#define CRYP_DMA_REQ_MASK (BIT(1) | BIT(0)) +#define CRYP_DMA_REQ_MASK_POS 0 + + +struct cryp_system_context { + /* CRYP Register structure */ + struct cryp_register *p_cryp_reg[MAX_DEVICE_SUPPORT]; +}; + +#endif diff --git a/drivers/crypto/ux500/hash/Makefile b/drivers/crypto/ux500/hash/Makefile new file mode 100644 index 000000000..a8f088724 --- /dev/null +++ b/drivers/crypto/ux500/hash/Makefile @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Copyright (C) ST-Ericsson SA 2010 +# Author: Shujuan Chen (shujuan.chen@stericsson.com) +# +ifdef CONFIG_CRYPTO_DEV_UX500_DEBUG +CFLAGS_hash_core.o := -DDEBUG +endif + +obj-$(CONFIG_CRYPTO_DEV_UX500_HASH) += ux500_hash.o +ux500_hash-objs := hash_core.o diff --git a/drivers/crypto/ux500/hash/hash_alg.h b/drivers/crypto/ux500/hash/hash_alg.h new file mode 100644 index 000000000..7c9bcc151 --- /dev/null +++ b/drivers/crypto/ux500/hash/hash_alg.h @@ -0,0 +1,398 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) ST-Ericsson SA 2010 + * Author: Shujuan Chen (shujuan.chen@stericsson.com) + * Author: Joakim Bech (joakim.xx.bech@stericsson.com) + * Author: Berne Hebark (berne.hebark@stericsson.com)) + */ +#ifndef _HASH_ALG_H +#define _HASH_ALG_H + +#include <linux/bitops.h> + +#define HASH_BLOCK_SIZE 64 +#define HASH_DMA_FIFO 4 +#define HASH_DMA_ALIGN_SIZE 4 +#define HASH_DMA_PERFORMANCE_MIN_SIZE 1024 +#define HASH_BYTES_PER_WORD 4 + +/* Maximum value of the length's high word */ +#define HASH_HIGH_WORD_MAX_VAL 0xFFFFFFFFUL + +/* Power on Reset values HASH registers */ +#define HASH_RESET_CR_VALUE 0x0 +#define HASH_RESET_STR_VALUE 0x0 + +/* Number of context swap registers */ +#define HASH_CSR_COUNT 52 + +#define HASH_RESET_CSRX_REG_VALUE 0x0 +#define HASH_RESET_CSFULL_REG_VALUE 0x0 +#define HASH_RESET_CSDATAIN_REG_VALUE 0x0 + +#define HASH_RESET_INDEX_VAL 0x0 +#define HASH_RESET_BIT_INDEX_VAL 0x0 +#define HASH_RESET_BUFFER_VAL 0x0 +#define HASH_RESET_LEN_HIGH_VAL 0x0 +#define HASH_RESET_LEN_LOW_VAL 0x0 + +/* Control register bitfields */ +#define HASH_CR_RESUME_MASK 0x11FCF + +#define HASH_CR_SWITCHON_POS 31 +#define HASH_CR_SWITCHON_MASK BIT(31) + +#define HASH_CR_EMPTYMSG_POS 20 +#define HASH_CR_EMPTYMSG_MASK BIT(20) + +#define HASH_CR_DINF_POS 12 +#define HASH_CR_DINF_MASK BIT(12) + +#define HASH_CR_NBW_POS 8 +#define HASH_CR_NBW_MASK 0x00000F00UL + +#define HASH_CR_LKEY_POS 16 +#define HASH_CR_LKEY_MASK BIT(16) + +#define HASH_CR_ALGO_POS 7 +#define HASH_CR_ALGO_MASK BIT(7) + +#define HASH_CR_MODE_POS 6 +#define HASH_CR_MODE_MASK BIT(6) + +#define HASH_CR_DATAFORM_POS 4 +#define HASH_CR_DATAFORM_MASK (BIT(4) | BIT(5)) + +#define HASH_CR_DMAE_POS 3 +#define HASH_CR_DMAE_MASK BIT(3) + +#define HASH_CR_INIT_POS 2 +#define HASH_CR_INIT_MASK BIT(2) + +#define HASH_CR_PRIVN_POS 1 +#define HASH_CR_PRIVN_MASK BIT(1) + +#define HASH_CR_SECN_POS 0 +#define HASH_CR_SECN_MASK BIT(0) + +/* Start register bitfields */ +#define HASH_STR_DCAL_POS 8 +#define HASH_STR_DCAL_MASK BIT(8) +#define HASH_STR_DEFAULT 0x0 + +#define HASH_STR_NBLW_POS 0 +#define HASH_STR_NBLW_MASK 0x0000001FUL + +#define HASH_NBLW_MAX_VAL 0x1F + +/* PrimeCell IDs */ +#define HASH_P_ID0 0xE0 +#define HASH_P_ID1 0x05 +#define HASH_P_ID2 0x38 +#define HASH_P_ID3 0x00 +#define HASH_CELL_ID0 0x0D +#define HASH_CELL_ID1 0xF0 +#define HASH_CELL_ID2 0x05 +#define HASH_CELL_ID3 0xB1 + +#define HASH_SET_BITS(reg_name, mask) \ + writel_relaxed((readl_relaxed(reg_name) | mask), reg_name) + +#define HASH_CLEAR_BITS(reg_name, mask) \ + writel_relaxed((readl_relaxed(reg_name) & ~mask), reg_name) + +#define HASH_PUT_BITS(reg, val, shift, mask) \ + writel_relaxed(((readl(reg) & ~(mask)) | \ + (((u32)val << shift) & (mask))), reg) + +#define HASH_SET_DIN(val, len) writesl(&device_data->base->din, (val), (len)) + +#define HASH_INITIALIZE \ + HASH_PUT_BITS( \ + &device_data->base->cr, \ + 0x01, HASH_CR_INIT_POS, \ + HASH_CR_INIT_MASK) + +#define HASH_SET_DATA_FORMAT(data_format) \ + HASH_PUT_BITS( \ + &device_data->base->cr, \ + (u32) (data_format), HASH_CR_DATAFORM_POS, \ + HASH_CR_DATAFORM_MASK) +#define HASH_SET_NBLW(val) \ + HASH_PUT_BITS( \ + &device_data->base->str, \ + (u32) (val), HASH_STR_NBLW_POS, \ + HASH_STR_NBLW_MASK) +#define HASH_SET_DCAL \ + HASH_PUT_BITS( \ + &device_data->base->str, \ + 0x01, HASH_STR_DCAL_POS, \ + HASH_STR_DCAL_MASK) + +/* Hardware access method */ +enum hash_mode { + HASH_MODE_CPU, + HASH_MODE_DMA +}; + +/** + * struct uint64 - Structure to handle 64 bits integers. + * @high_word: Most significant bits. + * @low_word: Least significant bits. + * + * Used to handle 64 bits integers. + */ +struct uint64 { + u32 high_word; + u32 low_word; +}; + +/** + * struct hash_register - Contains all registers in ux500 hash hardware. + * @cr: HASH control register (0x000). + * @din: HASH data input register (0x004). + * @str: HASH start register (0x008). + * @hx: HASH digest register 0..7 (0x00c-0x01C). + * @padding0: Reserved (0x02C). + * @itcr: Integration test control register (0x080). + * @itip: Integration test input register (0x084). + * @itop: Integration test output register (0x088). + * @padding1: Reserved (0x08C). + * @csfull: HASH context full register (0x0F8). + * @csdatain: HASH context swap data input register (0x0FC). + * @csrx: HASH context swap register 0..51 (0x100-0x1CC). + * @padding2: Reserved (0x1D0). + * @periphid0: HASH peripheral identification register 0 (0xFE0). + * @periphid1: HASH peripheral identification register 1 (0xFE4). + * @periphid2: HASH peripheral identification register 2 (0xFE8). + * @periphid3: HASH peripheral identification register 3 (0xFEC). + * @cellid0: HASH PCell identification register 0 (0xFF0). + * @cellid1: HASH PCell identification register 1 (0xFF4). + * @cellid2: HASH PCell identification register 2 (0xFF8). + * @cellid3: HASH PCell identification register 3 (0xFFC). + * + * The device communicates to the HASH via 32-bit-wide control registers + * accessible via the 32-bit width AMBA rev. 2.0 AHB Bus. Below is a structure + * with the registers used. + */ +struct hash_register { + u32 cr; + u32 din; + u32 str; + u32 hx[8]; + + u32 padding0[(0x080 - 0x02C) / sizeof(u32)]; + + u32 itcr; + u32 itip; + u32 itop; + + u32 padding1[(0x0F8 - 0x08C) / sizeof(u32)]; + + u32 csfull; + u32 csdatain; + u32 csrx[HASH_CSR_COUNT]; + + u32 padding2[(0xFE0 - 0x1D0) / sizeof(u32)]; + + u32 periphid0; + u32 periphid1; + u32 periphid2; + u32 periphid3; + + u32 cellid0; + u32 cellid1; + u32 cellid2; + u32 cellid3; +}; + +/** + * struct hash_state - Hash context state. + * @temp_cr: Temporary HASH Control Register. + * @str_reg: HASH Start Register. + * @din_reg: HASH Data Input Register. + * @csr[52]: HASH Context Swap Registers 0-39. + * @csfull: HASH Context Swap Registers 40 ie Status flags. + * @csdatain: HASH Context Swap Registers 41 ie Input data. + * @buffer: Working buffer for messages going to the hardware. + * @length: Length of the part of message hashed so far (floor(N/64) * 64). + * @index: Valid number of bytes in buffer (N % 64). + * @bit_index: Valid number of bits in buffer (N % 8). + * + * This structure is used between context switches, i.e. when ongoing jobs are + * interupted with new jobs. When this happens we need to store intermediate + * results in software. + * + * WARNING: "index" is the member of the structure, to be sure that "buffer" + * is aligned on a 4-bytes boundary. This is highly implementation dependent + * and MUST be checked whenever this code is ported on new platforms. + */ +struct hash_state { + u32 temp_cr; + u32 str_reg; + u32 din_reg; + u32 csr[52]; + u32 csfull; + u32 csdatain; + u32 buffer[HASH_BLOCK_SIZE / sizeof(u32)]; + struct uint64 length; + u8 index; + u8 bit_index; +}; + +/** + * enum hash_device_id - HASH device ID. + * @HASH_DEVICE_ID_0: Hash hardware with ID 0 + * @HASH_DEVICE_ID_1: Hash hardware with ID 1 + */ +enum hash_device_id { + HASH_DEVICE_ID_0 = 0, + HASH_DEVICE_ID_1 = 1 +}; + +/** + * enum hash_data_format - HASH data format. + * @HASH_DATA_32_BITS: 32 bits data format + * @HASH_DATA_16_BITS: 16 bits data format + * @HASH_DATA_8_BITS: 8 bits data format. + * @HASH_DATA_1_BITS: 1 bit data format. + */ +enum hash_data_format { + HASH_DATA_32_BITS = 0x0, + HASH_DATA_16_BITS = 0x1, + HASH_DATA_8_BITS = 0x2, + HASH_DATA_1_BIT = 0x3 +}; + +/** + * enum hash_algo - Enumeration for selecting between SHA1 or SHA2 algorithm. + * @HASH_ALGO_SHA1: Indicates that SHA1 is used. + * @HASH_ALGO_SHA2: Indicates that SHA2 (SHA256) is used. + */ +enum hash_algo { + HASH_ALGO_SHA1 = 0x0, + HASH_ALGO_SHA256 = 0x1 +}; + +/** + * enum hash_op - Enumeration for selecting between HASH or HMAC mode. + * @HASH_OPER_MODE_HASH: Indicates usage of normal HASH mode. + * @HASH_OPER_MODE_HMAC: Indicates usage of HMAC. + */ +enum hash_op { + HASH_OPER_MODE_HASH = 0x0, + HASH_OPER_MODE_HMAC = 0x1 +}; + +/** + * struct hash_config - Configuration data for the hardware. + * @data_format: Format of data entered into the hash data in register. + * @algorithm: Algorithm selection bit. + * @oper_mode: Operating mode selection bit. + */ +struct hash_config { + int data_format; + int algorithm; + int oper_mode; +}; + +/** + * struct hash_dma - Structure used for dma. + * @mask: DMA capabilities bitmap mask. + * @complete: Used to maintain state for a "completion". + * @chan_mem2hash: DMA channel. + * @cfg_mem2hash: DMA channel configuration. + * @sg_len: Scatterlist length. + * @sg: Scatterlist. + * @nents: Number of sg entries. + */ +struct hash_dma { + dma_cap_mask_t mask; + struct completion complete; + struct dma_chan *chan_mem2hash; + void *cfg_mem2hash; + int sg_len; + struct scatterlist *sg; + int nents; +}; + +/** + * struct hash_ctx - The context used for hash calculations. + * @key: The key used in the operation. + * @keylen: The length of the key. + * @state: The state of the current calculations. + * @config: The current configuration. + * @digestsize: The size of current digest. + * @device: Pointer to the device structure. + */ +struct hash_ctx { + u8 *key; + u32 keylen; + struct hash_config config; + int digestsize; + struct hash_device_data *device; +}; + +/** + * struct hash_ctx - The request context used for hash calculations. + * @state: The state of the current calculations. + * @dma_mode: Used in special cases (workaround), e.g. need to change to + * cpu mode, if not supported/working in dma mode. + * @updated: Indicates if hardware is initialized for new operations. + */ +struct hash_req_ctx { + struct hash_state state; + bool dma_mode; + u8 updated; +}; + +/** + * struct hash_device_data - structure for a hash device. + * @base: Pointer to virtual base address of the hash device. + * @phybase: Pointer to physical memory location of the hash device. + * @list_node: For inclusion in klist. + * @dev: Pointer to the device dev structure. + * @ctx_lock: Spinlock for current_ctx. + * @current_ctx: Pointer to the currently allocated context. + * @power_state: TRUE = power state on, FALSE = power state off. + * @power_state_lock: Spinlock for power_state. + * @regulator: Pointer to the device's power control. + * @clk: Pointer to the device's clock control. + * @restore_dev_state: TRUE = saved state, FALSE = no saved state. + * @dma: Structure used for dma. + */ +struct hash_device_data { + struct hash_register __iomem *base; + phys_addr_t phybase; + struct klist_node list_node; + struct device *dev; + spinlock_t ctx_lock; + struct hash_ctx *current_ctx; + bool power_state; + spinlock_t power_state_lock; + struct regulator *regulator; + struct clk *clk; + bool restore_dev_state; + struct hash_state state; /* Used for saving and resuming state */ + struct hash_dma dma; +}; + +int hash_check_hw(struct hash_device_data *device_data); + +int hash_setconfiguration(struct hash_device_data *device_data, + struct hash_config *config); + +void hash_begin(struct hash_device_data *device_data, struct hash_ctx *ctx); + +void hash_get_digest(struct hash_device_data *device_data, + u8 *digest, int algorithm); + +int hash_hw_update(struct ahash_request *req); + +int hash_save_state(struct hash_device_data *device_data, + struct hash_state *state); + +int hash_resume_state(struct hash_device_data *device_data, + const struct hash_state *state); + +#endif diff --git a/drivers/crypto/ux500/hash/hash_core.c b/drivers/crypto/ux500/hash/hash_core.c new file mode 100644 index 000000000..1e2daf403 --- /dev/null +++ b/drivers/crypto/ux500/hash/hash_core.c @@ -0,0 +1,1973 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Cryptographic API. + * Support for Nomadik hardware crypto engine. + + * Copyright (C) ST-Ericsson SA 2010 + * Author: Shujuan Chen <shujuan.chen@stericsson.com> for ST-Ericsson + * Author: Joakim Bech <joakim.xx.bech@stericsson.com> for ST-Ericsson + * Author: Berne Hebark <berne.herbark@stericsson.com> for ST-Ericsson. + * Author: Niklas Hernaeus <niklas.hernaeus@stericsson.com> for ST-Ericsson. + * Author: Andreas Westin <andreas.westin@stericsson.com> for ST-Ericsson. + */ + +#define pr_fmt(fmt) "hashX hashX: " fmt + +#include <linux/clk.h> +#include <linux/device.h> +#include <linux/dma-mapping.h> +#include <linux/err.h> +#include <linux/init.h> +#include <linux/io.h> +#include <linux/klist.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/mod_devicetable.h> +#include <linux/platform_device.h> +#include <linux/crypto.h> + +#include <linux/regulator/consumer.h> +#include <linux/dmaengine.h> +#include <linux/bitops.h> + +#include <crypto/internal/hash.h> +#include <crypto/sha.h> +#include <crypto/scatterwalk.h> +#include <crypto/algapi.h> + +#include <linux/platform_data/crypto-ux500.h> + +#include "hash_alg.h" + +static int hash_mode; +module_param(hash_mode, int, 0); +MODULE_PARM_DESC(hash_mode, "CPU or DMA mode. CPU = 0 (default), DMA = 1"); + +/* HMAC-SHA1, no key */ +static const u8 zero_message_hmac_sha1[SHA1_DIGEST_SIZE] = { + 0xfb, 0xdb, 0x1d, 0x1b, 0x18, 0xaa, 0x6c, 0x08, + 0x32, 0x4b, 0x7d, 0x64, 0xb7, 0x1f, 0xb7, 0x63, + 0x70, 0x69, 0x0e, 0x1d +}; + +/* HMAC-SHA256, no key */ +static const u8 zero_message_hmac_sha256[SHA256_DIGEST_SIZE] = { + 0xb6, 0x13, 0x67, 0x9a, 0x08, 0x14, 0xd9, 0xec, + 0x77, 0x2f, 0x95, 0xd7, 0x78, 0xc3, 0x5f, 0xc5, + 0xff, 0x16, 0x97, 0xc4, 0x93, 0x71, 0x56, 0x53, + 0xc6, 0xc7, 0x12, 0x14, 0x42, 0x92, 0xc5, 0xad +}; + +/** + * struct hash_driver_data - data specific to the driver. + * + * @device_list: A list of registered devices to choose from. + * @device_allocation: A semaphore initialized with number of devices. + */ +struct hash_driver_data { + struct klist device_list; + struct semaphore device_allocation; +}; + +static struct hash_driver_data driver_data; + +/* Declaration of functions */ +/** + * hash_messagepad - Pads a message and write the nblw bits. + * @device_data: Structure for the hash device. + * @message: Last word of a message + * @index_bytes: The number of bytes in the last message + * + * This function manages the final part of the digest calculation, when less + * than 512 bits (64 bytes) remain in message. This means index_bytes < 64. + * + */ +static void hash_messagepad(struct hash_device_data *device_data, + const u32 *message, u8 index_bytes); + +/** + * release_hash_device - Releases a previously allocated hash device. + * @device_data: Structure for the hash device. + * + */ +static void release_hash_device(struct hash_device_data *device_data) +{ + spin_lock(&device_data->ctx_lock); + device_data->current_ctx->device = NULL; + device_data->current_ctx = NULL; + spin_unlock(&device_data->ctx_lock); + + /* + * The down_interruptible part for this semaphore is called in + * cryp_get_device_data. + */ + up(&driver_data.device_allocation); +} + +static void hash_dma_setup_channel(struct hash_device_data *device_data, + struct device *dev) +{ + struct hash_platform_data *platform_data = dev->platform_data; + struct dma_slave_config conf = { + .direction = DMA_MEM_TO_DEV, + .dst_addr = device_data->phybase + HASH_DMA_FIFO, + .dst_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES, + .dst_maxburst = 16, + }; + + dma_cap_zero(device_data->dma.mask); + dma_cap_set(DMA_SLAVE, device_data->dma.mask); + + device_data->dma.cfg_mem2hash = platform_data->mem_to_engine; + device_data->dma.chan_mem2hash = + dma_request_channel(device_data->dma.mask, + platform_data->dma_filter, + device_data->dma.cfg_mem2hash); + + dmaengine_slave_config(device_data->dma.chan_mem2hash, &conf); + + init_completion(&device_data->dma.complete); +} + +static void hash_dma_callback(void *data) +{ + struct hash_ctx *ctx = data; + + complete(&ctx->device->dma.complete); +} + +static int hash_set_dma_transfer(struct hash_ctx *ctx, struct scatterlist *sg, + int len, enum dma_data_direction direction) +{ + struct dma_async_tx_descriptor *desc = NULL; + struct dma_chan *channel = NULL; + + if (direction != DMA_TO_DEVICE) { + dev_err(ctx->device->dev, "%s: Invalid DMA direction\n", + __func__); + return -EFAULT; + } + + sg->length = ALIGN(sg->length, HASH_DMA_ALIGN_SIZE); + + channel = ctx->device->dma.chan_mem2hash; + ctx->device->dma.sg = sg; + ctx->device->dma.sg_len = dma_map_sg(channel->device->dev, + ctx->device->dma.sg, ctx->device->dma.nents, + direction); + + if (!ctx->device->dma.sg_len) { + dev_err(ctx->device->dev, "%s: Could not map the sg list (TO_DEVICE)\n", + __func__); + return -EFAULT; + } + + dev_dbg(ctx->device->dev, "%s: Setting up DMA for buffer (TO_DEVICE)\n", + __func__); + desc = dmaengine_prep_slave_sg(channel, + ctx->device->dma.sg, ctx->device->dma.sg_len, + DMA_MEM_TO_DEV, DMA_CTRL_ACK | DMA_PREP_INTERRUPT); + if (!desc) { + dev_err(ctx->device->dev, + "%s: dmaengine_prep_slave_sg() failed!\n", __func__); + return -EFAULT; + } + + desc->callback = hash_dma_callback; + desc->callback_param = ctx; + + dmaengine_submit(desc); + dma_async_issue_pending(channel); + + return 0; +} + +static void hash_dma_done(struct hash_ctx *ctx) +{ + struct dma_chan *chan; + + chan = ctx->device->dma.chan_mem2hash; + dmaengine_terminate_all(chan); + dma_unmap_sg(chan->device->dev, ctx->device->dma.sg, + ctx->device->dma.sg_len, DMA_TO_DEVICE); +} + +static int hash_dma_write(struct hash_ctx *ctx, + struct scatterlist *sg, int len) +{ + int error = hash_set_dma_transfer(ctx, sg, len, DMA_TO_DEVICE); + if (error) { + dev_dbg(ctx->device->dev, + "%s: hash_set_dma_transfer() failed\n", __func__); + return error; + } + + return len; +} + +/** + * get_empty_message_digest - Returns a pre-calculated digest for + * the empty message. + * @device_data: Structure for the hash device. + * @zero_hash: Buffer to return the empty message digest. + * @zero_hash_size: Hash size of the empty message digest. + * @zero_digest: True if zero_digest returned. + */ +static int get_empty_message_digest( + struct hash_device_data *device_data, + u8 *zero_hash, u32 *zero_hash_size, bool *zero_digest) +{ + int ret = 0; + struct hash_ctx *ctx = device_data->current_ctx; + *zero_digest = false; + + /** + * Caller responsible for ctx != NULL. + */ + + if (HASH_OPER_MODE_HASH == ctx->config.oper_mode) { + if (HASH_ALGO_SHA1 == ctx->config.algorithm) { + memcpy(zero_hash, &sha1_zero_message_hash[0], + SHA1_DIGEST_SIZE); + *zero_hash_size = SHA1_DIGEST_SIZE; + *zero_digest = true; + } else if (HASH_ALGO_SHA256 == + ctx->config.algorithm) { + memcpy(zero_hash, &sha256_zero_message_hash[0], + SHA256_DIGEST_SIZE); + *zero_hash_size = SHA256_DIGEST_SIZE; + *zero_digest = true; + } else { + dev_err(device_data->dev, "%s: Incorrect algorithm!\n", + __func__); + ret = -EINVAL; + goto out; + } + } else if (HASH_OPER_MODE_HMAC == ctx->config.oper_mode) { + if (!ctx->keylen) { + if (HASH_ALGO_SHA1 == ctx->config.algorithm) { + memcpy(zero_hash, &zero_message_hmac_sha1[0], + SHA1_DIGEST_SIZE); + *zero_hash_size = SHA1_DIGEST_SIZE; + *zero_digest = true; + } else if (HASH_ALGO_SHA256 == ctx->config.algorithm) { + memcpy(zero_hash, &zero_message_hmac_sha256[0], + SHA256_DIGEST_SIZE); + *zero_hash_size = SHA256_DIGEST_SIZE; + *zero_digest = true; + } else { + dev_err(device_data->dev, "%s: Incorrect algorithm!\n", + __func__); + ret = -EINVAL; + goto out; + } + } else { + dev_dbg(device_data->dev, + "%s: Continue hash calculation, since hmac key available\n", + __func__); + } + } +out: + + return ret; +} + +/** + * hash_disable_power - Request to disable power and clock. + * @device_data: Structure for the hash device. + * @save_device_state: If true, saves the current hw state. + * + * This function request for disabling power (regulator) and clock, + * and could also save current hw state. + */ +static int hash_disable_power(struct hash_device_data *device_data, + bool save_device_state) +{ + int ret = 0; + struct device *dev = device_data->dev; + + spin_lock(&device_data->power_state_lock); + if (!device_data->power_state) + goto out; + + if (save_device_state) { + hash_save_state(device_data, + &device_data->state); + device_data->restore_dev_state = true; + } + + clk_disable(device_data->clk); + ret = regulator_disable(device_data->regulator); + if (ret) + dev_err(dev, "%s: regulator_disable() failed!\n", __func__); + + device_data->power_state = false; + +out: + spin_unlock(&device_data->power_state_lock); + + return ret; +} + +/** + * hash_enable_power - Request to enable power and clock. + * @device_data: Structure for the hash device. + * @restore_device_state: If true, restores a previous saved hw state. + * + * This function request for enabling power (regulator) and clock, + * and could also restore a previously saved hw state. + */ +static int hash_enable_power(struct hash_device_data *device_data, + bool restore_device_state) +{ + int ret = 0; + struct device *dev = device_data->dev; + + spin_lock(&device_data->power_state_lock); + if (!device_data->power_state) { + ret = regulator_enable(device_data->regulator); + if (ret) { + dev_err(dev, "%s: regulator_enable() failed!\n", + __func__); + goto out; + } + ret = clk_enable(device_data->clk); + if (ret) { + dev_err(dev, "%s: clk_enable() failed!\n", __func__); + ret = regulator_disable( + device_data->regulator); + goto out; + } + device_data->power_state = true; + } + + if (device_data->restore_dev_state) { + if (restore_device_state) { + device_data->restore_dev_state = false; + hash_resume_state(device_data, &device_data->state); + } + } +out: + spin_unlock(&device_data->power_state_lock); + + return ret; +} + +/** + * hash_get_device_data - Checks for an available hash device and return it. + * @hash_ctx: Structure for the hash context. + * @device_data: Structure for the hash device. + * + * This function check for an available hash device and return it to + * the caller. + * Note! Caller need to release the device, calling up(). + */ +static int hash_get_device_data(struct hash_ctx *ctx, + struct hash_device_data **device_data) +{ + int ret; + struct klist_iter device_iterator; + struct klist_node *device_node; + struct hash_device_data *local_device_data = NULL; + + /* Wait until a device is available */ + ret = down_interruptible(&driver_data.device_allocation); + if (ret) + return ret; /* Interrupted */ + + /* Select a device */ + klist_iter_init(&driver_data.device_list, &device_iterator); + device_node = klist_next(&device_iterator); + while (device_node) { + local_device_data = container_of(device_node, + struct hash_device_data, list_node); + spin_lock(&local_device_data->ctx_lock); + /* current_ctx allocates a device, NULL = unallocated */ + if (local_device_data->current_ctx) { + device_node = klist_next(&device_iterator); + } else { + local_device_data->current_ctx = ctx; + ctx->device = local_device_data; + spin_unlock(&local_device_data->ctx_lock); + break; + } + spin_unlock(&local_device_data->ctx_lock); + } + klist_iter_exit(&device_iterator); + + if (!device_node) { + /** + * No free device found. + * Since we allocated a device with down_interruptible, this + * should not be able to happen. + * Number of available devices, which are contained in + * device_allocation, is therefore decremented by not doing + * an up(device_allocation). + */ + return -EBUSY; + } + + *device_data = local_device_data; + + return 0; +} + +/** + * hash_hw_write_key - Writes the key to the hardware registries. + * + * @device_data: Structure for the hash device. + * @key: Key to be written. + * @keylen: The lengt of the key. + * + * Note! This function DOES NOT write to the NBLW registry, even though + * specified in the the hw design spec. Either due to incorrect info in the + * spec or due to a bug in the hw. + */ +static void hash_hw_write_key(struct hash_device_data *device_data, + const u8 *key, unsigned int keylen) +{ + u32 word = 0; + int nwords = 1; + + HASH_CLEAR_BITS(&device_data->base->str, HASH_STR_NBLW_MASK); + + while (keylen >= 4) { + u32 *key_word = (u32 *)key; + + HASH_SET_DIN(key_word, nwords); + keylen -= 4; + key += 4; + } + + /* Take care of the remaining bytes in the last word */ + if (keylen) { + word = 0; + while (keylen) { + word |= (key[keylen - 1] << (8 * (keylen - 1))); + keylen--; + } + + HASH_SET_DIN(&word, nwords); + } + + while (readl(&device_data->base->str) & HASH_STR_DCAL_MASK) + cpu_relax(); + + HASH_SET_DCAL; + + while (readl(&device_data->base->str) & HASH_STR_DCAL_MASK) + cpu_relax(); +} + +/** + * init_hash_hw - Initialise the hash hardware for a new calculation. + * @device_data: Structure for the hash device. + * @ctx: The hash context. + * + * This function will enable the bits needed to clear and start a new + * calculation. + */ +static int init_hash_hw(struct hash_device_data *device_data, + struct hash_ctx *ctx) +{ + int ret = 0; + + ret = hash_setconfiguration(device_data, &ctx->config); + if (ret) { + dev_err(device_data->dev, "%s: hash_setconfiguration() failed!\n", + __func__); + return ret; + } + + hash_begin(device_data, ctx); + + if (ctx->config.oper_mode == HASH_OPER_MODE_HMAC) + hash_hw_write_key(device_data, ctx->key, ctx->keylen); + + return ret; +} + +/** + * hash_get_nents - Return number of entries (nents) in scatterlist (sg). + * + * @sg: Scatterlist. + * @size: Size in bytes. + * @aligned: True if sg data aligned to work in DMA mode. + * + */ +static int hash_get_nents(struct scatterlist *sg, int size, bool *aligned) +{ + int nents = 0; + bool aligned_data = true; + + while (size > 0 && sg) { + nents++; + size -= sg->length; + + /* hash_set_dma_transfer will align last nent */ + if ((aligned && !IS_ALIGNED(sg->offset, HASH_DMA_ALIGN_SIZE)) || + (!IS_ALIGNED(sg->length, HASH_DMA_ALIGN_SIZE) && size > 0)) + aligned_data = false; + + sg = sg_next(sg); + } + + if (aligned) + *aligned = aligned_data; + + if (size != 0) + return -EFAULT; + + return nents; +} + +/** + * hash_dma_valid_data - checks for dma valid sg data. + * @sg: Scatterlist. + * @datasize: Datasize in bytes. + * + * NOTE! This function checks for dma valid sg data, since dma + * only accept datasizes of even wordsize. + */ +static bool hash_dma_valid_data(struct scatterlist *sg, int datasize) +{ + bool aligned; + + /* Need to include at least one nent, else error */ + if (hash_get_nents(sg, datasize, &aligned) < 1) + return false; + + return aligned; +} + +/** + * hash_init - Common hash init function for SHA1/SHA2 (SHA256). + * @req: The hash request for the job. + * + * Initialize structures. + */ +static int ux500_hash_init(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct hash_ctx *ctx = crypto_ahash_ctx(tfm); + struct hash_req_ctx *req_ctx = ahash_request_ctx(req); + + if (!ctx->key) + ctx->keylen = 0; + + memset(&req_ctx->state, 0, sizeof(struct hash_state)); + req_ctx->updated = 0; + if (hash_mode == HASH_MODE_DMA) { + if (req->nbytes < HASH_DMA_ALIGN_SIZE) { + req_ctx->dma_mode = false; /* Don't use DMA */ + + pr_debug("%s: DMA mode, but direct to CPU mode for data size < %d\n", + __func__, HASH_DMA_ALIGN_SIZE); + } else { + if (req->nbytes >= HASH_DMA_PERFORMANCE_MIN_SIZE && + hash_dma_valid_data(req->src, req->nbytes)) { + req_ctx->dma_mode = true; + } else { + req_ctx->dma_mode = false; + pr_debug("%s: DMA mode, but use CPU mode for datalength < %d or non-aligned data, except in last nent\n", + __func__, + HASH_DMA_PERFORMANCE_MIN_SIZE); + } + } + } + return 0; +} + +/** + * hash_processblock - This function processes a single block of 512 bits (64 + * bytes), word aligned, starting at message. + * @device_data: Structure for the hash device. + * @message: Block (512 bits) of message to be written to + * the HASH hardware. + * + */ +static void hash_processblock(struct hash_device_data *device_data, + const u32 *message, int length) +{ + int len = length / HASH_BYTES_PER_WORD; + /* + * NBLW bits. Reset the number of bits in last word (NBLW). + */ + HASH_CLEAR_BITS(&device_data->base->str, HASH_STR_NBLW_MASK); + + /* + * Write message data to the HASH_DIN register. + */ + HASH_SET_DIN(message, len); +} + +/** + * hash_messagepad - Pads a message and write the nblw bits. + * @device_data: Structure for the hash device. + * @message: Last word of a message. + * @index_bytes: The number of bytes in the last message. + * + * This function manages the final part of the digest calculation, when less + * than 512 bits (64 bytes) remain in message. This means index_bytes < 64. + * + */ +static void hash_messagepad(struct hash_device_data *device_data, + const u32 *message, u8 index_bytes) +{ + int nwords = 1; + + /* + * Clear hash str register, only clear NBLW + * since DCAL will be reset by hardware. + */ + HASH_CLEAR_BITS(&device_data->base->str, HASH_STR_NBLW_MASK); + + /* Main loop */ + while (index_bytes >= 4) { + HASH_SET_DIN(message, nwords); + index_bytes -= 4; + message++; + } + + if (index_bytes) + HASH_SET_DIN(message, nwords); + + while (readl(&device_data->base->str) & HASH_STR_DCAL_MASK) + cpu_relax(); + + /* num_of_bytes == 0 => NBLW <- 0 (32 bits valid in DATAIN) */ + HASH_SET_NBLW(index_bytes * 8); + dev_dbg(device_data->dev, "%s: DIN=0x%08x NBLW=%lu\n", + __func__, readl_relaxed(&device_data->base->din), + readl_relaxed(&device_data->base->str) & HASH_STR_NBLW_MASK); + HASH_SET_DCAL; + dev_dbg(device_data->dev, "%s: after dcal -> DIN=0x%08x NBLW=%lu\n", + __func__, readl_relaxed(&device_data->base->din), + readl_relaxed(&device_data->base->str) & HASH_STR_NBLW_MASK); + + while (readl(&device_data->base->str) & HASH_STR_DCAL_MASK) + cpu_relax(); +} + +/** + * hash_incrementlength - Increments the length of the current message. + * @ctx: Hash context + * @incr: Length of message processed already + * + * Overflow cannot occur, because conditions for overflow are checked in + * hash_hw_update. + */ +static void hash_incrementlength(struct hash_req_ctx *ctx, u32 incr) +{ + ctx->state.length.low_word += incr; + + /* Check for wrap-around */ + if (ctx->state.length.low_word < incr) + ctx->state.length.high_word++; +} + +/** + * hash_setconfiguration - Sets the required configuration for the hash + * hardware. + * @device_data: Structure for the hash device. + * @config: Pointer to a configuration structure. + */ +int hash_setconfiguration(struct hash_device_data *device_data, + struct hash_config *config) +{ + int ret = 0; + + if (config->algorithm != HASH_ALGO_SHA1 && + config->algorithm != HASH_ALGO_SHA256) + return -EPERM; + + /* + * DATAFORM bits. Set the DATAFORM bits to 0b11, which means the data + * to be written to HASH_DIN is considered as 32 bits. + */ + HASH_SET_DATA_FORMAT(config->data_format); + + /* + * ALGO bit. Set to 0b1 for SHA-1 and 0b0 for SHA-256 + */ + switch (config->algorithm) { + case HASH_ALGO_SHA1: + HASH_SET_BITS(&device_data->base->cr, HASH_CR_ALGO_MASK); + break; + + case HASH_ALGO_SHA256: + HASH_CLEAR_BITS(&device_data->base->cr, HASH_CR_ALGO_MASK); + break; + + default: + dev_err(device_data->dev, "%s: Incorrect algorithm\n", + __func__); + return -EPERM; + } + + /* + * MODE bit. This bit selects between HASH or HMAC mode for the + * selected algorithm. 0b0 = HASH and 0b1 = HMAC. + */ + if (HASH_OPER_MODE_HASH == config->oper_mode) + HASH_CLEAR_BITS(&device_data->base->cr, + HASH_CR_MODE_MASK); + else if (HASH_OPER_MODE_HMAC == config->oper_mode) { + HASH_SET_BITS(&device_data->base->cr, HASH_CR_MODE_MASK); + if (device_data->current_ctx->keylen > HASH_BLOCK_SIZE) { + /* Truncate key to blocksize */ + dev_dbg(device_data->dev, "%s: LKEY set\n", __func__); + HASH_SET_BITS(&device_data->base->cr, + HASH_CR_LKEY_MASK); + } else { + dev_dbg(device_data->dev, "%s: LKEY cleared\n", + __func__); + HASH_CLEAR_BITS(&device_data->base->cr, + HASH_CR_LKEY_MASK); + } + } else { /* Wrong hash mode */ + ret = -EPERM; + dev_err(device_data->dev, "%s: HASH_INVALID_PARAMETER!\n", + __func__); + } + return ret; +} + +/** + * hash_begin - This routine resets some globals and initializes the hash + * hardware. + * @device_data: Structure for the hash device. + * @ctx: Hash context. + */ +void hash_begin(struct hash_device_data *device_data, struct hash_ctx *ctx) +{ + /* HW and SW initializations */ + /* Note: there is no need to initialize buffer and digest members */ + + while (readl(&device_data->base->str) & HASH_STR_DCAL_MASK) + cpu_relax(); + + /* + * INIT bit. Set this bit to 0b1 to reset the HASH processor core and + * prepare the initialize the HASH accelerator to compute the message + * digest of a new message. + */ + HASH_INITIALIZE; + + /* + * NBLW bits. Reset the number of bits in last word (NBLW). + */ + HASH_CLEAR_BITS(&device_data->base->str, HASH_STR_NBLW_MASK); +} + +static int hash_process_data(struct hash_device_data *device_data, + struct hash_ctx *ctx, struct hash_req_ctx *req_ctx, + int msg_length, u8 *data_buffer, u8 *buffer, + u8 *index) +{ + int ret = 0; + u32 count; + + do { + if ((*index + msg_length) < HASH_BLOCK_SIZE) { + for (count = 0; count < msg_length; count++) { + buffer[*index + count] = + *(data_buffer + count); + } + *index += msg_length; + msg_length = 0; + } else { + if (req_ctx->updated) { + ret = hash_resume_state(device_data, + &device_data->state); + memmove(req_ctx->state.buffer, + device_data->state.buffer, + HASH_BLOCK_SIZE); + if (ret) { + dev_err(device_data->dev, + "%s: hash_resume_state() failed!\n", + __func__); + goto out; + } + } else { + ret = init_hash_hw(device_data, ctx); + if (ret) { + dev_err(device_data->dev, + "%s: init_hash_hw() failed!\n", + __func__); + goto out; + } + req_ctx->updated = 1; + } + /* + * If 'data_buffer' is four byte aligned and + * local buffer does not have any data, we can + * write data directly from 'data_buffer' to + * HW peripheral, otherwise we first copy data + * to a local buffer + */ + if (IS_ALIGNED((unsigned long)data_buffer, 4) && + (0 == *index)) + hash_processblock(device_data, + (const u32 *)data_buffer, + HASH_BLOCK_SIZE); + else { + for (count = 0; + count < (u32)(HASH_BLOCK_SIZE - *index); + count++) { + buffer[*index + count] = + *(data_buffer + count); + } + hash_processblock(device_data, + (const u32 *)buffer, + HASH_BLOCK_SIZE); + } + hash_incrementlength(req_ctx, HASH_BLOCK_SIZE); + data_buffer += (HASH_BLOCK_SIZE - *index); + + msg_length -= (HASH_BLOCK_SIZE - *index); + *index = 0; + + ret = hash_save_state(device_data, + &device_data->state); + + memmove(device_data->state.buffer, + req_ctx->state.buffer, + HASH_BLOCK_SIZE); + if (ret) { + dev_err(device_data->dev, "%s: hash_save_state() failed!\n", + __func__); + goto out; + } + } + } while (msg_length != 0); +out: + + return ret; +} + +/** + * hash_dma_final - The hash dma final function for SHA1/SHA256. + * @req: The hash request for the job. + */ +static int hash_dma_final(struct ahash_request *req) +{ + int ret = 0; + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct hash_ctx *ctx = crypto_ahash_ctx(tfm); + struct hash_req_ctx *req_ctx = ahash_request_ctx(req); + struct hash_device_data *device_data; + u8 digest[SHA256_DIGEST_SIZE]; + int bytes_written = 0; + + ret = hash_get_device_data(ctx, &device_data); + if (ret) + return ret; + + dev_dbg(device_data->dev, "%s: (ctx=0x%lx)!\n", __func__, + (unsigned long)ctx); + + if (req_ctx->updated) { + ret = hash_resume_state(device_data, &device_data->state); + + if (ret) { + dev_err(device_data->dev, "%s: hash_resume_state() failed!\n", + __func__); + goto out; + } + } + + if (!req_ctx->updated) { + ret = hash_setconfiguration(device_data, &ctx->config); + if (ret) { + dev_err(device_data->dev, + "%s: hash_setconfiguration() failed!\n", + __func__); + goto out; + } + + /* Enable DMA input */ + if (hash_mode != HASH_MODE_DMA || !req_ctx->dma_mode) { + HASH_CLEAR_BITS(&device_data->base->cr, + HASH_CR_DMAE_MASK); + } else { + HASH_SET_BITS(&device_data->base->cr, + HASH_CR_DMAE_MASK); + HASH_SET_BITS(&device_data->base->cr, + HASH_CR_PRIVN_MASK); + } + + HASH_INITIALIZE; + + if (ctx->config.oper_mode == HASH_OPER_MODE_HMAC) + hash_hw_write_key(device_data, ctx->key, ctx->keylen); + + /* Number of bits in last word = (nbytes * 8) % 32 */ + HASH_SET_NBLW((req->nbytes * 8) % 32); + req_ctx->updated = 1; + } + + /* Store the nents in the dma struct. */ + ctx->device->dma.nents = hash_get_nents(req->src, req->nbytes, NULL); + if (!ctx->device->dma.nents) { + dev_err(device_data->dev, "%s: ctx->device->dma.nents = 0\n", + __func__); + ret = ctx->device->dma.nents; + goto out; + } + + bytes_written = hash_dma_write(ctx, req->src, req->nbytes); + if (bytes_written != req->nbytes) { + dev_err(device_data->dev, "%s: hash_dma_write() failed!\n", + __func__); + ret = bytes_written; + goto out; + } + + wait_for_completion(&ctx->device->dma.complete); + hash_dma_done(ctx); + + while (readl(&device_data->base->str) & HASH_STR_DCAL_MASK) + cpu_relax(); + + if (ctx->config.oper_mode == HASH_OPER_MODE_HMAC && ctx->key) { + unsigned int keylen = ctx->keylen; + u8 *key = ctx->key; + + dev_dbg(device_data->dev, "%s: keylen: %d\n", + __func__, ctx->keylen); + hash_hw_write_key(device_data, key, keylen); + } + + hash_get_digest(device_data, digest, ctx->config.algorithm); + memcpy(req->result, digest, ctx->digestsize); + +out: + release_hash_device(device_data); + + /** + * Allocated in setkey, and only used in HMAC. + */ + kfree(ctx->key); + + return ret; +} + +/** + * hash_hw_final - The final hash calculation function + * @req: The hash request for the job. + */ +static int hash_hw_final(struct ahash_request *req) +{ + int ret = 0; + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct hash_ctx *ctx = crypto_ahash_ctx(tfm); + struct hash_req_ctx *req_ctx = ahash_request_ctx(req); + struct hash_device_data *device_data; + u8 digest[SHA256_DIGEST_SIZE]; + + ret = hash_get_device_data(ctx, &device_data); + if (ret) + return ret; + + dev_dbg(device_data->dev, "%s: (ctx=0x%lx)!\n", __func__, + (unsigned long)ctx); + + if (req_ctx->updated) { + ret = hash_resume_state(device_data, &device_data->state); + + if (ret) { + dev_err(device_data->dev, + "%s: hash_resume_state() failed!\n", __func__); + goto out; + } + } else if (req->nbytes == 0 && ctx->keylen == 0) { + u8 zero_hash[SHA256_DIGEST_SIZE]; + u32 zero_hash_size = 0; + bool zero_digest = false; + /** + * Use a pre-calculated empty message digest + * (workaround since hw return zeroes, hw bug!?) + */ + ret = get_empty_message_digest(device_data, &zero_hash[0], + &zero_hash_size, &zero_digest); + if (!ret && likely(zero_hash_size == ctx->digestsize) && + zero_digest) { + memcpy(req->result, &zero_hash[0], ctx->digestsize); + goto out; + } else if (!ret && !zero_digest) { + dev_dbg(device_data->dev, + "%s: HMAC zero msg with key, continue...\n", + __func__); + } else { + dev_err(device_data->dev, + "%s: ret=%d, or wrong digest size? %s\n", + __func__, ret, + zero_hash_size == ctx->digestsize ? + "true" : "false"); + /* Return error */ + goto out; + } + } else if (req->nbytes == 0 && ctx->keylen > 0) { + ret = -EPERM; + dev_err(device_data->dev, "%s: Empty message with keylength > 0, NOT supported\n", + __func__); + goto out; + } + + if (!req_ctx->updated) { + ret = init_hash_hw(device_data, ctx); + if (ret) { + dev_err(device_data->dev, + "%s: init_hash_hw() failed!\n", __func__); + goto out; + } + } + + if (req_ctx->state.index) { + hash_messagepad(device_data, req_ctx->state.buffer, + req_ctx->state.index); + } else { + HASH_SET_DCAL; + while (readl(&device_data->base->str) & HASH_STR_DCAL_MASK) + cpu_relax(); + } + + if (ctx->config.oper_mode == HASH_OPER_MODE_HMAC && ctx->key) { + unsigned int keylen = ctx->keylen; + u8 *key = ctx->key; + + dev_dbg(device_data->dev, "%s: keylen: %d\n", + __func__, ctx->keylen); + hash_hw_write_key(device_data, key, keylen); + } + + hash_get_digest(device_data, digest, ctx->config.algorithm); + memcpy(req->result, digest, ctx->digestsize); + +out: + release_hash_device(device_data); + + /** + * Allocated in setkey, and only used in HMAC. + */ + kfree(ctx->key); + + return ret; +} + +/** + * hash_hw_update - Updates current HASH computation hashing another part of + * the message. + * @req: Byte array containing the message to be hashed (caller + * allocated). + */ +int hash_hw_update(struct ahash_request *req) +{ + int ret = 0; + u8 index = 0; + u8 *buffer; + struct hash_device_data *device_data; + u8 *data_buffer; + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct hash_ctx *ctx = crypto_ahash_ctx(tfm); + struct hash_req_ctx *req_ctx = ahash_request_ctx(req); + struct crypto_hash_walk walk; + int msg_length; + + index = req_ctx->state.index; + buffer = (u8 *)req_ctx->state.buffer; + + ret = hash_get_device_data(ctx, &device_data); + if (ret) + return ret; + + msg_length = crypto_hash_walk_first(req, &walk); + + /* Empty message ("") is correct indata */ + if (msg_length == 0) { + ret = 0; + goto release_dev; + } + + /* Check if ctx->state.length + msg_length + overflows */ + if (msg_length > (req_ctx->state.length.low_word + msg_length) && + HASH_HIGH_WORD_MAX_VAL == req_ctx->state.length.high_word) { + pr_err("%s: HASH_MSG_LENGTH_OVERFLOW!\n", __func__); + ret = crypto_hash_walk_done(&walk, -EPERM); + goto release_dev; + } + + /* Main loop */ + while (0 != msg_length) { + data_buffer = walk.data; + ret = hash_process_data(device_data, ctx, req_ctx, msg_length, + data_buffer, buffer, &index); + + if (ret) { + dev_err(device_data->dev, "%s: hash_internal_hw_update() failed!\n", + __func__); + crypto_hash_walk_done(&walk, ret); + goto release_dev; + } + + msg_length = crypto_hash_walk_done(&walk, 0); + } + + req_ctx->state.index = index; + dev_dbg(device_data->dev, "%s: indata length=%d, bin=%d\n", + __func__, req_ctx->state.index, req_ctx->state.bit_index); + +release_dev: + release_hash_device(device_data); + + return ret; +} + +/** + * hash_resume_state - Function that resumes the state of an calculation. + * @device_data: Pointer to the device structure. + * @device_state: The state to be restored in the hash hardware + */ +int hash_resume_state(struct hash_device_data *device_data, + const struct hash_state *device_state) +{ + u32 temp_cr; + s32 count; + int hash_mode = HASH_OPER_MODE_HASH; + + if (NULL == device_state) { + dev_err(device_data->dev, "%s: HASH_INVALID_PARAMETER!\n", + __func__); + return -EPERM; + } + + /* Check correctness of index and length members */ + if (device_state->index > HASH_BLOCK_SIZE || + (device_state->length.low_word % HASH_BLOCK_SIZE) != 0) { + dev_err(device_data->dev, "%s: HASH_INVALID_PARAMETER!\n", + __func__); + return -EPERM; + } + + /* + * INIT bit. Set this bit to 0b1 to reset the HASH processor core and + * prepare the initialize the HASH accelerator to compute the message + * digest of a new message. + */ + HASH_INITIALIZE; + + temp_cr = device_state->temp_cr; + writel_relaxed(temp_cr & HASH_CR_RESUME_MASK, &device_data->base->cr); + + if (readl(&device_data->base->cr) & HASH_CR_MODE_MASK) + hash_mode = HASH_OPER_MODE_HMAC; + else + hash_mode = HASH_OPER_MODE_HASH; + + for (count = 0; count < HASH_CSR_COUNT; count++) { + if ((count >= 36) && (hash_mode == HASH_OPER_MODE_HASH)) + break; + + writel_relaxed(device_state->csr[count], + &device_data->base->csrx[count]); + } + + writel_relaxed(device_state->csfull, &device_data->base->csfull); + writel_relaxed(device_state->csdatain, &device_data->base->csdatain); + + writel_relaxed(device_state->str_reg, &device_data->base->str); + writel_relaxed(temp_cr, &device_data->base->cr); + + return 0; +} + +/** + * hash_save_state - Function that saves the state of hardware. + * @device_data: Pointer to the device structure. + * @device_state: The strucure where the hardware state should be saved. + */ +int hash_save_state(struct hash_device_data *device_data, + struct hash_state *device_state) +{ + u32 temp_cr; + u32 count; + int hash_mode = HASH_OPER_MODE_HASH; + + if (NULL == device_state) { + dev_err(device_data->dev, "%s: HASH_INVALID_PARAMETER!\n", + __func__); + return -ENOTSUPP; + } + + /* Write dummy value to force digest intermediate calculation. This + * actually makes sure that there isn't any ongoing calculation in the + * hardware. + */ + while (readl(&device_data->base->str) & HASH_STR_DCAL_MASK) + cpu_relax(); + + temp_cr = readl_relaxed(&device_data->base->cr); + + device_state->str_reg = readl_relaxed(&device_data->base->str); + + device_state->din_reg = readl_relaxed(&device_data->base->din); + + if (readl(&device_data->base->cr) & HASH_CR_MODE_MASK) + hash_mode = HASH_OPER_MODE_HMAC; + else + hash_mode = HASH_OPER_MODE_HASH; + + for (count = 0; count < HASH_CSR_COUNT; count++) { + if ((count >= 36) && (hash_mode == HASH_OPER_MODE_HASH)) + break; + + device_state->csr[count] = + readl_relaxed(&device_data->base->csrx[count]); + } + + device_state->csfull = readl_relaxed(&device_data->base->csfull); + device_state->csdatain = readl_relaxed(&device_data->base->csdatain); + + device_state->temp_cr = temp_cr; + + return 0; +} + +/** + * hash_check_hw - This routine checks for peripheral Ids and PCell Ids. + * @device_data: + * + */ +int hash_check_hw(struct hash_device_data *device_data) +{ + /* Checking Peripheral Ids */ + if (HASH_P_ID0 == readl_relaxed(&device_data->base->periphid0) && + HASH_P_ID1 == readl_relaxed(&device_data->base->periphid1) && + HASH_P_ID2 == readl_relaxed(&device_data->base->periphid2) && + HASH_P_ID3 == readl_relaxed(&device_data->base->periphid3) && + HASH_CELL_ID0 == readl_relaxed(&device_data->base->cellid0) && + HASH_CELL_ID1 == readl_relaxed(&device_data->base->cellid1) && + HASH_CELL_ID2 == readl_relaxed(&device_data->base->cellid2) && + HASH_CELL_ID3 == readl_relaxed(&device_data->base->cellid3)) { + return 0; + } + + dev_err(device_data->dev, "%s: HASH_UNSUPPORTED_HW!\n", __func__); + return -ENOTSUPP; +} + +/** + * hash_get_digest - Gets the digest. + * @device_data: Pointer to the device structure. + * @digest: User allocated byte array for the calculated digest. + * @algorithm: The algorithm in use. + */ +void hash_get_digest(struct hash_device_data *device_data, + u8 *digest, int algorithm) +{ + u32 temp_hx_val, count; + int loop_ctr; + + if (algorithm != HASH_ALGO_SHA1 && algorithm != HASH_ALGO_SHA256) { + dev_err(device_data->dev, "%s: Incorrect algorithm %d\n", + __func__, algorithm); + return; + } + + if (algorithm == HASH_ALGO_SHA1) + loop_ctr = SHA1_DIGEST_SIZE / sizeof(u32); + else + loop_ctr = SHA256_DIGEST_SIZE / sizeof(u32); + + dev_dbg(device_data->dev, "%s: digest array:(0x%lx)\n", + __func__, (unsigned long)digest); + + /* Copy result into digest array */ + for (count = 0; count < loop_ctr; count++) { + temp_hx_val = readl_relaxed(&device_data->base->hx[count]); + digest[count * 4] = (u8) ((temp_hx_val >> 24) & 0xFF); + digest[count * 4 + 1] = (u8) ((temp_hx_val >> 16) & 0xFF); + digest[count * 4 + 2] = (u8) ((temp_hx_val >> 8) & 0xFF); + digest[count * 4 + 3] = (u8) ((temp_hx_val >> 0) & 0xFF); + } +} + +/** + * hash_update - The hash update function for SHA1/SHA2 (SHA256). + * @req: The hash request for the job. + */ +static int ahash_update(struct ahash_request *req) +{ + int ret = 0; + struct hash_req_ctx *req_ctx = ahash_request_ctx(req); + + if (hash_mode != HASH_MODE_DMA || !req_ctx->dma_mode) + ret = hash_hw_update(req); + /* Skip update for DMA, all data will be passed to DMA in final */ + + if (ret) { + pr_err("%s: hash_hw_update() failed!\n", __func__); + } + + return ret; +} + +/** + * hash_final - The hash final function for SHA1/SHA2 (SHA256). + * @req: The hash request for the job. + */ +static int ahash_final(struct ahash_request *req) +{ + int ret = 0; + struct hash_req_ctx *req_ctx = ahash_request_ctx(req); + + pr_debug("%s: data size: %d\n", __func__, req->nbytes); + + if ((hash_mode == HASH_MODE_DMA) && req_ctx->dma_mode) + ret = hash_dma_final(req); + else + ret = hash_hw_final(req); + + if (ret) { + pr_err("%s: hash_hw/dma_final() failed\n", __func__); + } + + return ret; +} + +static int hash_setkey(struct crypto_ahash *tfm, + const u8 *key, unsigned int keylen, int alg) +{ + int ret = 0; + struct hash_ctx *ctx = crypto_ahash_ctx(tfm); + + /** + * Freed in final. + */ + ctx->key = kmemdup(key, keylen, GFP_KERNEL); + if (!ctx->key) { + pr_err("%s: Failed to allocate ctx->key for %d\n", + __func__, alg); + return -ENOMEM; + } + ctx->keylen = keylen; + + return ret; +} + +static int ahash_sha1_init(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct hash_ctx *ctx = crypto_ahash_ctx(tfm); + + ctx->config.data_format = HASH_DATA_8_BITS; + ctx->config.algorithm = HASH_ALGO_SHA1; + ctx->config.oper_mode = HASH_OPER_MODE_HASH; + ctx->digestsize = SHA1_DIGEST_SIZE; + + return ux500_hash_init(req); +} + +static int ahash_sha256_init(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct hash_ctx *ctx = crypto_ahash_ctx(tfm); + + ctx->config.data_format = HASH_DATA_8_BITS; + ctx->config.algorithm = HASH_ALGO_SHA256; + ctx->config.oper_mode = HASH_OPER_MODE_HASH; + ctx->digestsize = SHA256_DIGEST_SIZE; + + return ux500_hash_init(req); +} + +static int ahash_sha1_digest(struct ahash_request *req) +{ + int ret2, ret1; + + ret1 = ahash_sha1_init(req); + if (ret1) + goto out; + + ret1 = ahash_update(req); + ret2 = ahash_final(req); + +out: + return ret1 ? ret1 : ret2; +} + +static int ahash_sha256_digest(struct ahash_request *req) +{ + int ret2, ret1; + + ret1 = ahash_sha256_init(req); + if (ret1) + goto out; + + ret1 = ahash_update(req); + ret2 = ahash_final(req); + +out: + return ret1 ? ret1 : ret2; +} + +static int ahash_noimport(struct ahash_request *req, const void *in) +{ + return -ENOSYS; +} + +static int ahash_noexport(struct ahash_request *req, void *out) +{ + return -ENOSYS; +} + +static int hmac_sha1_init(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct hash_ctx *ctx = crypto_ahash_ctx(tfm); + + ctx->config.data_format = HASH_DATA_8_BITS; + ctx->config.algorithm = HASH_ALGO_SHA1; + ctx->config.oper_mode = HASH_OPER_MODE_HMAC; + ctx->digestsize = SHA1_DIGEST_SIZE; + + return ux500_hash_init(req); +} + +static int hmac_sha256_init(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct hash_ctx *ctx = crypto_ahash_ctx(tfm); + + ctx->config.data_format = HASH_DATA_8_BITS; + ctx->config.algorithm = HASH_ALGO_SHA256; + ctx->config.oper_mode = HASH_OPER_MODE_HMAC; + ctx->digestsize = SHA256_DIGEST_SIZE; + + return ux500_hash_init(req); +} + +static int hmac_sha1_digest(struct ahash_request *req) +{ + int ret2, ret1; + + ret1 = hmac_sha1_init(req); + if (ret1) + goto out; + + ret1 = ahash_update(req); + ret2 = ahash_final(req); + +out: + return ret1 ? ret1 : ret2; +} + +static int hmac_sha256_digest(struct ahash_request *req) +{ + int ret2, ret1; + + ret1 = hmac_sha256_init(req); + if (ret1) + goto out; + + ret1 = ahash_update(req); + ret2 = ahash_final(req); + +out: + return ret1 ? ret1 : ret2; +} + +static int hmac_sha1_setkey(struct crypto_ahash *tfm, + const u8 *key, unsigned int keylen) +{ + return hash_setkey(tfm, key, keylen, HASH_ALGO_SHA1); +} + +static int hmac_sha256_setkey(struct crypto_ahash *tfm, + const u8 *key, unsigned int keylen) +{ + return hash_setkey(tfm, key, keylen, HASH_ALGO_SHA256); +} + +struct hash_algo_template { + struct hash_config conf; + struct ahash_alg hash; +}; + +static int hash_cra_init(struct crypto_tfm *tfm) +{ + struct hash_ctx *ctx = crypto_tfm_ctx(tfm); + struct crypto_alg *alg = tfm->__crt_alg; + struct hash_algo_template *hash_alg; + + hash_alg = container_of(__crypto_ahash_alg(alg), + struct hash_algo_template, + hash); + + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct hash_req_ctx)); + + ctx->config.data_format = HASH_DATA_8_BITS; + ctx->config.algorithm = hash_alg->conf.algorithm; + ctx->config.oper_mode = hash_alg->conf.oper_mode; + + ctx->digestsize = hash_alg->hash.halg.digestsize; + + return 0; +} + +static struct hash_algo_template hash_algs[] = { + { + .conf.algorithm = HASH_ALGO_SHA1, + .conf.oper_mode = HASH_OPER_MODE_HASH, + .hash = { + .init = ux500_hash_init, + .update = ahash_update, + .final = ahash_final, + .digest = ahash_sha1_digest, + .export = ahash_noexport, + .import = ahash_noimport, + .halg.digestsize = SHA1_DIGEST_SIZE, + .halg.statesize = sizeof(struct hash_ctx), + .halg.base = { + .cra_name = "sha1", + .cra_driver_name = "sha1-ux500", + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct hash_ctx), + .cra_init = hash_cra_init, + .cra_module = THIS_MODULE, + } + } + }, + { + .conf.algorithm = HASH_ALGO_SHA256, + .conf.oper_mode = HASH_OPER_MODE_HASH, + .hash = { + .init = ux500_hash_init, + .update = ahash_update, + .final = ahash_final, + .digest = ahash_sha256_digest, + .export = ahash_noexport, + .import = ahash_noimport, + .halg.digestsize = SHA256_DIGEST_SIZE, + .halg.statesize = sizeof(struct hash_ctx), + .halg.base = { + .cra_name = "sha256", + .cra_driver_name = "sha256-ux500", + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct hash_ctx), + .cra_init = hash_cra_init, + .cra_module = THIS_MODULE, + } + } + }, + { + .conf.algorithm = HASH_ALGO_SHA1, + .conf.oper_mode = HASH_OPER_MODE_HMAC, + .hash = { + .init = ux500_hash_init, + .update = ahash_update, + .final = ahash_final, + .digest = hmac_sha1_digest, + .setkey = hmac_sha1_setkey, + .export = ahash_noexport, + .import = ahash_noimport, + .halg.digestsize = SHA1_DIGEST_SIZE, + .halg.statesize = sizeof(struct hash_ctx), + .halg.base = { + .cra_name = "hmac(sha1)", + .cra_driver_name = "hmac-sha1-ux500", + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct hash_ctx), + .cra_init = hash_cra_init, + .cra_module = THIS_MODULE, + } + } + }, + { + .conf.algorithm = HASH_ALGO_SHA256, + .conf.oper_mode = HASH_OPER_MODE_HMAC, + .hash = { + .init = ux500_hash_init, + .update = ahash_update, + .final = ahash_final, + .digest = hmac_sha256_digest, + .setkey = hmac_sha256_setkey, + .export = ahash_noexport, + .import = ahash_noimport, + .halg.digestsize = SHA256_DIGEST_SIZE, + .halg.statesize = sizeof(struct hash_ctx), + .halg.base = { + .cra_name = "hmac(sha256)", + .cra_driver_name = "hmac-sha256-ux500", + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct hash_ctx), + .cra_init = hash_cra_init, + .cra_module = THIS_MODULE, + } + } + } +}; + +/** + * hash_algs_register_all - + */ +static int ahash_algs_register_all(struct hash_device_data *device_data) +{ + int ret; + int i; + int count; + + for (i = 0; i < ARRAY_SIZE(hash_algs); i++) { + ret = crypto_register_ahash(&hash_algs[i].hash); + if (ret) { + count = i; + dev_err(device_data->dev, "%s: alg registration failed\n", + hash_algs[i].hash.halg.base.cra_driver_name); + goto unreg; + } + } + return 0; +unreg: + for (i = 0; i < count; i++) + crypto_unregister_ahash(&hash_algs[i].hash); + return ret; +} + +/** + * hash_algs_unregister_all - + */ +static void ahash_algs_unregister_all(struct hash_device_data *device_data) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(hash_algs); i++) + crypto_unregister_ahash(&hash_algs[i].hash); +} + +/** + * ux500_hash_probe - Function that probes the hash hardware. + * @pdev: The platform device. + */ +static int ux500_hash_probe(struct platform_device *pdev) +{ + int ret = 0; + struct resource *res = NULL; + struct hash_device_data *device_data; + struct device *dev = &pdev->dev; + + device_data = devm_kzalloc(dev, sizeof(*device_data), GFP_ATOMIC); + if (!device_data) { + ret = -ENOMEM; + goto out; + } + + device_data->dev = dev; + device_data->current_ctx = NULL; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) { + dev_dbg(dev, "%s: platform_get_resource() failed!\n", __func__); + ret = -ENODEV; + goto out; + } + + device_data->phybase = res->start; + device_data->base = devm_ioremap_resource(dev, res); + if (IS_ERR(device_data->base)) { + dev_err(dev, "%s: ioremap() failed!\n", __func__); + ret = PTR_ERR(device_data->base); + goto out; + } + spin_lock_init(&device_data->ctx_lock); + spin_lock_init(&device_data->power_state_lock); + + /* Enable power for HASH1 hardware block */ + device_data->regulator = regulator_get(dev, "v-ape"); + if (IS_ERR(device_data->regulator)) { + dev_err(dev, "%s: regulator_get() failed!\n", __func__); + ret = PTR_ERR(device_data->regulator); + device_data->regulator = NULL; + goto out; + } + + /* Enable the clock for HASH1 hardware block */ + device_data->clk = devm_clk_get(dev, NULL); + if (IS_ERR(device_data->clk)) { + dev_err(dev, "%s: clk_get() failed!\n", __func__); + ret = PTR_ERR(device_data->clk); + goto out_regulator; + } + + ret = clk_prepare(device_data->clk); + if (ret) { + dev_err(dev, "%s: clk_prepare() failed!\n", __func__); + goto out_regulator; + } + + /* Enable device power (and clock) */ + ret = hash_enable_power(device_data, false); + if (ret) { + dev_err(dev, "%s: hash_enable_power() failed!\n", __func__); + goto out_clk_unprepare; + } + + ret = hash_check_hw(device_data); + if (ret) { + dev_err(dev, "%s: hash_check_hw() failed!\n", __func__); + goto out_power; + } + + if (hash_mode == HASH_MODE_DMA) + hash_dma_setup_channel(device_data, dev); + + platform_set_drvdata(pdev, device_data); + + /* Put the new device into the device list... */ + klist_add_tail(&device_data->list_node, &driver_data.device_list); + /* ... and signal that a new device is available. */ + up(&driver_data.device_allocation); + + ret = ahash_algs_register_all(device_data); + if (ret) { + dev_err(dev, "%s: ahash_algs_register_all() failed!\n", + __func__); + goto out_power; + } + + dev_info(dev, "successfully registered\n"); + return 0; + +out_power: + hash_disable_power(device_data, false); + +out_clk_unprepare: + clk_unprepare(device_data->clk); + +out_regulator: + regulator_put(device_data->regulator); + +out: + return ret; +} + +/** + * ux500_hash_remove - Function that removes the hash device from the platform. + * @pdev: The platform device. + */ +static int ux500_hash_remove(struct platform_device *pdev) +{ + struct hash_device_data *device_data; + struct device *dev = &pdev->dev; + + device_data = platform_get_drvdata(pdev); + if (!device_data) { + dev_err(dev, "%s: platform_get_drvdata() failed!\n", __func__); + return -ENOMEM; + } + + /* Try to decrease the number of available devices. */ + if (down_trylock(&driver_data.device_allocation)) + return -EBUSY; + + /* Check that the device is free */ + spin_lock(&device_data->ctx_lock); + /* current_ctx allocates a device, NULL = unallocated */ + if (device_data->current_ctx) { + /* The device is busy */ + spin_unlock(&device_data->ctx_lock); + /* Return the device to the pool. */ + up(&driver_data.device_allocation); + return -EBUSY; + } + + spin_unlock(&device_data->ctx_lock); + + /* Remove the device from the list */ + if (klist_node_attached(&device_data->list_node)) + klist_remove(&device_data->list_node); + + /* If this was the last device, remove the services */ + if (list_empty(&driver_data.device_list.k_list)) + ahash_algs_unregister_all(device_data); + + if (hash_disable_power(device_data, false)) + dev_err(dev, "%s: hash_disable_power() failed\n", + __func__); + + clk_unprepare(device_data->clk); + regulator_put(device_data->regulator); + + return 0; +} + +/** + * ux500_hash_shutdown - Function that shutdown the hash device. + * @pdev: The platform device + */ +static void ux500_hash_shutdown(struct platform_device *pdev) +{ + struct hash_device_data *device_data; + + device_data = platform_get_drvdata(pdev); + if (!device_data) { + dev_err(&pdev->dev, "%s: platform_get_drvdata() failed!\n", + __func__); + return; + } + + /* Check that the device is free */ + spin_lock(&device_data->ctx_lock); + /* current_ctx allocates a device, NULL = unallocated */ + if (!device_data->current_ctx) { + if (down_trylock(&driver_data.device_allocation)) + dev_dbg(&pdev->dev, "%s: Cryp still in use! Shutting down anyway...\n", + __func__); + /** + * (Allocate the device) + * Need to set this to non-null (dummy) value, + * to avoid usage if context switching. + */ + device_data->current_ctx++; + } + spin_unlock(&device_data->ctx_lock); + + /* Remove the device from the list */ + if (klist_node_attached(&device_data->list_node)) + klist_remove(&device_data->list_node); + + /* If this was the last device, remove the services */ + if (list_empty(&driver_data.device_list.k_list)) + ahash_algs_unregister_all(device_data); + + if (hash_disable_power(device_data, false)) + dev_err(&pdev->dev, "%s: hash_disable_power() failed\n", + __func__); +} + +#ifdef CONFIG_PM_SLEEP +/** + * ux500_hash_suspend - Function that suspends the hash device. + * @dev: Device to suspend. + */ +static int ux500_hash_suspend(struct device *dev) +{ + int ret; + struct hash_device_data *device_data; + struct hash_ctx *temp_ctx = NULL; + + device_data = dev_get_drvdata(dev); + if (!device_data) { + dev_err(dev, "%s: platform_get_drvdata() failed!\n", __func__); + return -ENOMEM; + } + + spin_lock(&device_data->ctx_lock); + if (!device_data->current_ctx) + device_data->current_ctx++; + spin_unlock(&device_data->ctx_lock); + + if (device_data->current_ctx == ++temp_ctx) { + if (down_interruptible(&driver_data.device_allocation)) + dev_dbg(dev, "%s: down_interruptible() failed\n", + __func__); + ret = hash_disable_power(device_data, false); + + } else { + ret = hash_disable_power(device_data, true); + } + + if (ret) + dev_err(dev, "%s: hash_disable_power()\n", __func__); + + return ret; +} + +/** + * ux500_hash_resume - Function that resume the hash device. + * @dev: Device to resume. + */ +static int ux500_hash_resume(struct device *dev) +{ + int ret = 0; + struct hash_device_data *device_data; + struct hash_ctx *temp_ctx = NULL; + + device_data = dev_get_drvdata(dev); + if (!device_data) { + dev_err(dev, "%s: platform_get_drvdata() failed!\n", __func__); + return -ENOMEM; + } + + spin_lock(&device_data->ctx_lock); + if (device_data->current_ctx == ++temp_ctx) + device_data->current_ctx = NULL; + spin_unlock(&device_data->ctx_lock); + + if (!device_data->current_ctx) + up(&driver_data.device_allocation); + else + ret = hash_enable_power(device_data, true); + + if (ret) + dev_err(dev, "%s: hash_enable_power() failed!\n", __func__); + + return ret; +} +#endif + +static SIMPLE_DEV_PM_OPS(ux500_hash_pm, ux500_hash_suspend, ux500_hash_resume); + +static const struct of_device_id ux500_hash_match[] = { + { .compatible = "stericsson,ux500-hash" }, + { }, +}; +MODULE_DEVICE_TABLE(of, ux500_hash_match); + +static struct platform_driver hash_driver = { + .probe = ux500_hash_probe, + .remove = ux500_hash_remove, + .shutdown = ux500_hash_shutdown, + .driver = { + .name = "hash1", + .of_match_table = ux500_hash_match, + .pm = &ux500_hash_pm, + } +}; + +/** + * ux500_hash_mod_init - The kernel module init function. + */ +static int __init ux500_hash_mod_init(void) +{ + klist_init(&driver_data.device_list, NULL, NULL); + /* Initialize the semaphore to 0 devices (locked state) */ + sema_init(&driver_data.device_allocation, 0); + + return platform_driver_register(&hash_driver); +} + +/** + * ux500_hash_mod_fini - The kernel module exit function. + */ +static void __exit ux500_hash_mod_fini(void) +{ + platform_driver_unregister(&hash_driver); +} + +module_init(ux500_hash_mod_init); +module_exit(ux500_hash_mod_fini); + +MODULE_DESCRIPTION("Driver for ST-Ericsson UX500 HASH engine."); +MODULE_LICENSE("GPL"); + +MODULE_ALIAS_CRYPTO("sha1-all"); +MODULE_ALIAS_CRYPTO("sha256-all"); +MODULE_ALIAS_CRYPTO("hmac-sha1-all"); +MODULE_ALIAS_CRYPTO("hmac-sha256-all"); |