From 5d1646d90e1f2cceb9f0828f4b28318cd0ec7744 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sat, 27 Apr 2024 12:05:51 +0200 Subject: Adding upstream version 5.10.209. Signed-off-by: Daniel Baumann --- drivers/crypto/hisilicon/sec2/Makefile | 2 + drivers/crypto/hisilicon/sec2/sec.h | 189 ++++ drivers/crypto/hisilicon/sec2/sec_crypto.c | 1661 ++++++++++++++++++++++++++++ drivers/crypto/hisilicon/sec2/sec_crypto.h | 215 ++++ drivers/crypto/hisilicon/sec2/sec_main.c | 998 +++++++++++++++++ 5 files changed, 3065 insertions(+) create mode 100644 drivers/crypto/hisilicon/sec2/Makefile create mode 100644 drivers/crypto/hisilicon/sec2/sec.h create mode 100644 drivers/crypto/hisilicon/sec2/sec_crypto.c create mode 100644 drivers/crypto/hisilicon/sec2/sec_crypto.h create mode 100644 drivers/crypto/hisilicon/sec2/sec_main.c (limited to 'drivers/crypto/hisilicon/sec2') diff --git a/drivers/crypto/hisilicon/sec2/Makefile b/drivers/crypto/hisilicon/sec2/Makefile new file mode 100644 index 000000000..b4f6cf14b --- /dev/null +++ b/drivers/crypto/hisilicon/sec2/Makefile @@ -0,0 +1,2 @@ +obj-$(CONFIG_CRYPTO_DEV_HISI_SEC2) += hisi_sec2.o +hisi_sec2-objs = sec_main.o sec_crypto.o diff --git a/drivers/crypto/hisilicon/sec2/sec.h b/drivers/crypto/hisilicon/sec2/sec.h new file mode 100644 index 000000000..249735b7c --- /dev/null +++ b/drivers/crypto/hisilicon/sec2/sec.h @@ -0,0 +1,189 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2019 HiSilicon Limited. */ + +#ifndef __HISI_SEC_V2_H +#define __HISI_SEC_V2_H + +#include "../qm.h" +#include "sec_crypto.h" + +/* Algorithm resource per hardware SEC queue */ +struct sec_alg_res { + u8 *pbuf; + dma_addr_t pbuf_dma; + u8 *c_ivin; + dma_addr_t c_ivin_dma; + u8 *out_mac; + dma_addr_t out_mac_dma; +}; + +/* Cipher request of SEC private */ +struct sec_cipher_req { + struct hisi_acc_hw_sgl *c_in; + dma_addr_t c_in_dma; + struct hisi_acc_hw_sgl *c_out; + dma_addr_t c_out_dma; + u8 *c_ivin; + dma_addr_t c_ivin_dma; + struct skcipher_request *sk_req; + u32 c_len; + bool encrypt; +}; + +struct sec_aead_req { + u8 *out_mac; + dma_addr_t out_mac_dma; + struct aead_request *aead_req; +}; + +/* SEC request of Crypto */ +struct sec_req { + struct sec_sqe sec_sqe; + struct sec_ctx *ctx; + struct sec_qp_ctx *qp_ctx; + + struct sec_cipher_req c_req; + struct sec_aead_req aead_req; + struct list_head backlog_head; + + int err_type; + int req_id; + u32 flag; + + /* Status of the SEC request */ + bool fake_busy; + bool use_pbuf; +}; + +/** + * struct sec_req_op - Operations for SEC request + * @buf_map: DMA map the SGL buffers of the request + * @buf_unmap: DMA unmap the SGL buffers of the request + * @bd_fill: Fill the SEC queue BD + * @bd_send: Send the SEC BD into the hardware queue + * @callback: Call back for the request + * @process: Main processing logic of Skcipher + */ +struct sec_req_op { + int (*buf_map)(struct sec_ctx *ctx, struct sec_req *req); + void (*buf_unmap)(struct sec_ctx *ctx, struct sec_req *req); + void (*do_transfer)(struct sec_ctx *ctx, struct sec_req *req); + int (*bd_fill)(struct sec_ctx *ctx, struct sec_req *req); + int (*bd_send)(struct sec_ctx *ctx, struct sec_req *req); + void (*callback)(struct sec_ctx *ctx, struct sec_req *req, int err); + int (*process)(struct sec_ctx *ctx, struct sec_req *req); +}; + +/* SEC auth context */ +struct sec_auth_ctx { + dma_addr_t a_key_dma; + u8 *a_key; + u8 a_key_len; + u8 mac_len; + u8 a_alg; + struct crypto_shash *hash_tfm; +}; + +/* SEC cipher context which cipher's relatives */ +struct sec_cipher_ctx { + u8 *c_key; + dma_addr_t c_key_dma; + sector_t iv_offset; + u32 c_gran_size; + u32 ivsize; + u8 c_mode; + u8 c_alg; + u8 c_key_len; +}; + +/* SEC queue context which defines queue's relatives */ +struct sec_qp_ctx { + struct hisi_qp *qp; + struct sec_req *req_list[QM_Q_DEPTH]; + struct idr req_idr; + struct sec_alg_res res[QM_Q_DEPTH]; + struct sec_ctx *ctx; + spinlock_t req_lock; + struct list_head backlog; + struct hisi_acc_sgl_pool *c_in_pool; + struct hisi_acc_sgl_pool *c_out_pool; + atomic_t pending_reqs; +}; + +enum sec_alg_type { + SEC_SKCIPHER, + SEC_AEAD +}; + +/* SEC Crypto TFM context which defines queue and cipher .etc relatives */ +struct sec_ctx { + struct sec_qp_ctx *qp_ctx; + struct sec_dev *sec; + const struct sec_req_op *req_op; + struct hisi_qp **qps; + + /* Half queues for encipher, and half for decipher */ + u32 hlf_q_num; + + /* Threshold for fake busy, trigger to return -EBUSY to user */ + u32 fake_req_limit; + + /* Currrent cyclic index to select a queue for encipher */ + atomic_t enc_qcyclic; + + /* Currrent cyclic index to select a queue for decipher */ + atomic_t dec_qcyclic; + + enum sec_alg_type alg_type; + bool pbuf_supported; + struct sec_cipher_ctx c_ctx; + struct sec_auth_ctx a_ctx; + struct device *dev; +}; + +enum sec_endian { + SEC_LE = 0, + SEC_32BE, + SEC_64BE +}; + +enum sec_debug_file_index { + SEC_CURRENT_QM, + SEC_CLEAR_ENABLE, + SEC_DEBUG_FILE_NUM, +}; + +struct sec_debug_file { + enum sec_debug_file_index index; + spinlock_t lock; + struct hisi_qm *qm; +}; + +struct sec_dfx { + atomic64_t send_cnt; + atomic64_t recv_cnt; + atomic64_t send_busy_cnt; + atomic64_t recv_busy_cnt; + atomic64_t err_bd_cnt; + atomic64_t invalid_req_cnt; + atomic64_t done_flag_cnt; +}; + +struct sec_debug { + struct sec_dfx dfx; + struct sec_debug_file files[SEC_DEBUG_FILE_NUM]; +}; + +struct sec_dev { + struct hisi_qm qm; + struct sec_debug debug; + u32 ctx_q_num; + bool iommu_used; + unsigned long status; +}; + +void sec_destroy_qps(struct hisi_qp **qps, int qp_num); +struct hisi_qp **sec_create_qps(void); +int sec_register_to_crypto(void); +void sec_unregister_from_crypto(void); +#endif diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c new file mode 100644 index 000000000..2dbec638c --- /dev/null +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -0,0 +1,1661 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2019 HiSilicon Limited. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sec.h" +#include "sec_crypto.h" + +#define SEC_PRIORITY 4001 +#define SEC_XTS_MIN_KEY_SIZE (2 * AES_MIN_KEY_SIZE) +#define SEC_XTS_MAX_KEY_SIZE (2 * AES_MAX_KEY_SIZE) +#define SEC_DES3_2KEY_SIZE (2 * DES_KEY_SIZE) +#define SEC_DES3_3KEY_SIZE (3 * DES_KEY_SIZE) + +/* SEC sqe(bd) bit operational relative MACRO */ +#define SEC_DE_OFFSET 1 +#define SEC_CIPHER_OFFSET 4 +#define SEC_SCENE_OFFSET 3 +#define SEC_DST_SGL_OFFSET 2 +#define SEC_SRC_SGL_OFFSET 7 +#define SEC_CKEY_OFFSET 9 +#define SEC_CMODE_OFFSET 12 +#define SEC_AKEY_OFFSET 5 +#define SEC_AEAD_ALG_OFFSET 11 +#define SEC_AUTH_OFFSET 6 + +#define SEC_FLAG_OFFSET 7 +#define SEC_FLAG_MASK 0x0780 +#define SEC_TYPE_MASK 0x0F +#define SEC_DONE_MASK 0x0001 + +#define SEC_TOTAL_IV_SZ (SEC_IV_SIZE * QM_Q_DEPTH) +#define SEC_SGL_SGE_NR 128 +#define SEC_CIPHER_AUTH 0xfe +#define SEC_AUTH_CIPHER 0x1 +#define SEC_MAX_MAC_LEN 64 +#define SEC_MAX_AAD_LEN 65535 +#define SEC_TOTAL_MAC_SZ (SEC_MAX_MAC_LEN * QM_Q_DEPTH) + +#define SEC_PBUF_SZ 512 +#define SEC_PBUF_IV_OFFSET SEC_PBUF_SZ +#define SEC_PBUF_MAC_OFFSET (SEC_PBUF_SZ + SEC_IV_SIZE) +#define SEC_PBUF_PKG (SEC_PBUF_SZ + SEC_IV_SIZE + \ + SEC_MAX_MAC_LEN * 2) +#define SEC_PBUF_NUM (PAGE_SIZE / SEC_PBUF_PKG) +#define SEC_PBUF_PAGE_NUM (QM_Q_DEPTH / SEC_PBUF_NUM) +#define SEC_PBUF_LEFT_SZ (SEC_PBUF_PKG * (QM_Q_DEPTH - \ + SEC_PBUF_PAGE_NUM * SEC_PBUF_NUM)) +#define SEC_TOTAL_PBUF_SZ (PAGE_SIZE * SEC_PBUF_PAGE_NUM + \ + SEC_PBUF_LEFT_SZ) + +#define SEC_SQE_LEN_RATE 4 +#define SEC_SQE_CFLAG 2 +#define SEC_SQE_AEAD_FLAG 3 +#define SEC_SQE_DONE 0x1 + +/* Get an en/de-cipher queue cyclically to balance load over queues of TFM */ +static inline int sec_alloc_queue_id(struct sec_ctx *ctx, struct sec_req *req) +{ + if (req->c_req.encrypt) + return (u32)atomic_inc_return(&ctx->enc_qcyclic) % + ctx->hlf_q_num; + + return (u32)atomic_inc_return(&ctx->dec_qcyclic) % ctx->hlf_q_num + + ctx->hlf_q_num; +} + +static inline void sec_free_queue_id(struct sec_ctx *ctx, struct sec_req *req) +{ + if (req->c_req.encrypt) + atomic_dec(&ctx->enc_qcyclic); + else + atomic_dec(&ctx->dec_qcyclic); +} + +static int sec_alloc_req_id(struct sec_req *req, struct sec_qp_ctx *qp_ctx) +{ + int req_id; + + spin_lock_bh(&qp_ctx->req_lock); + + req_id = idr_alloc_cyclic(&qp_ctx->req_idr, NULL, + 0, QM_Q_DEPTH, GFP_ATOMIC); + spin_unlock_bh(&qp_ctx->req_lock); + if (unlikely(req_id < 0)) { + dev_err(req->ctx->dev, "alloc req id fail!\n"); + return req_id; + } + + req->qp_ctx = qp_ctx; + qp_ctx->req_list[req_id] = req; + return req_id; +} + +static void sec_free_req_id(struct sec_req *req) +{ + struct sec_qp_ctx *qp_ctx = req->qp_ctx; + int req_id = req->req_id; + + if (unlikely(req_id < 0 || req_id >= QM_Q_DEPTH)) { + dev_err(req->ctx->dev, "free request id invalid!\n"); + return; + } + + qp_ctx->req_list[req_id] = NULL; + req->qp_ctx = NULL; + + spin_lock_bh(&qp_ctx->req_lock); + idr_remove(&qp_ctx->req_idr, req_id); + spin_unlock_bh(&qp_ctx->req_lock); +} + +static int sec_aead_verify(struct sec_req *req) +{ + struct aead_request *aead_req = req->aead_req.aead_req; + struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req); + size_t authsize = crypto_aead_authsize(tfm); + u8 *mac_out = req->aead_req.out_mac; + u8 *mac = mac_out + SEC_MAX_MAC_LEN; + struct scatterlist *sgl = aead_req->src; + size_t sz; + + sz = sg_pcopy_to_buffer(sgl, sg_nents(sgl), mac, authsize, + aead_req->cryptlen + aead_req->assoclen - + authsize); + if (unlikely(sz != authsize || memcmp(mac_out, mac, sz))) { + dev_err(req->ctx->dev, "aead verify failure!\n"); + return -EBADMSG; + } + + return 0; +} + +static void sec_req_cb(struct hisi_qp *qp, void *resp) +{ + struct sec_qp_ctx *qp_ctx = qp->qp_ctx; + struct sec_dfx *dfx = &qp_ctx->ctx->sec->debug.dfx; + struct sec_sqe *bd = resp; + struct sec_ctx *ctx; + struct sec_req *req; + u16 done, flag; + int err = 0; + u8 type; + + type = bd->type_cipher_auth & SEC_TYPE_MASK; + if (unlikely(type != SEC_BD_TYPE2)) { + atomic64_inc(&dfx->err_bd_cnt); + pr_err("err bd type [%d]\n", type); + return; + } + + req = qp_ctx->req_list[le16_to_cpu(bd->type2.tag)]; + if (unlikely(!req)) { + atomic64_inc(&dfx->invalid_req_cnt); + atomic_inc(&qp->qp_status.used); + return; + } + req->err_type = bd->type2.error_type; + ctx = req->ctx; + done = le16_to_cpu(bd->type2.done_flag) & SEC_DONE_MASK; + flag = (le16_to_cpu(bd->type2.done_flag) & + SEC_FLAG_MASK) >> SEC_FLAG_OFFSET; + if (unlikely(req->err_type || done != SEC_SQE_DONE || + (ctx->alg_type == SEC_SKCIPHER && flag != SEC_SQE_CFLAG) || + (ctx->alg_type == SEC_AEAD && flag != SEC_SQE_AEAD_FLAG))) { + dev_err_ratelimited(ctx->dev, + "err_type[%d],done[%d],flag[%d]\n", + req->err_type, done, flag); + err = -EIO; + atomic64_inc(&dfx->done_flag_cnt); + } + + if (ctx->alg_type == SEC_AEAD && !req->c_req.encrypt) + err = sec_aead_verify(req); + + atomic64_inc(&dfx->recv_cnt); + + ctx->req_op->buf_unmap(ctx, req); + + ctx->req_op->callback(ctx, req, err); +} + +static int sec_bd_send(struct sec_ctx *ctx, struct sec_req *req) +{ + struct sec_qp_ctx *qp_ctx = req->qp_ctx; + int ret; + + if (ctx->fake_req_limit <= + atomic_read(&qp_ctx->qp->qp_status.used) && + !(req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG)) + return -EBUSY; + + spin_lock_bh(&qp_ctx->req_lock); + ret = hisi_qp_send(qp_ctx->qp, &req->sec_sqe); + + if (ctx->fake_req_limit <= + atomic_read(&qp_ctx->qp->qp_status.used) && !ret) { + list_add_tail(&req->backlog_head, &qp_ctx->backlog); + atomic64_inc(&ctx->sec->debug.dfx.send_cnt); + atomic64_inc(&ctx->sec->debug.dfx.send_busy_cnt); + spin_unlock_bh(&qp_ctx->req_lock); + return -EBUSY; + } + spin_unlock_bh(&qp_ctx->req_lock); + + if (unlikely(ret == -EBUSY)) + return -ENOBUFS; + + if (likely(!ret)) { + ret = -EINPROGRESS; + atomic64_inc(&ctx->sec->debug.dfx.send_cnt); + } + + return ret; +} + +/* Get DMA memory resources */ +static int sec_alloc_civ_resource(struct device *dev, struct sec_alg_res *res) +{ + int i; + + res->c_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ, + &res->c_ivin_dma, GFP_KERNEL); + if (!res->c_ivin) + return -ENOMEM; + + for (i = 1; i < QM_Q_DEPTH; i++) { + res[i].c_ivin_dma = res->c_ivin_dma + i * SEC_IV_SIZE; + res[i].c_ivin = res->c_ivin + i * SEC_IV_SIZE; + } + + return 0; +} + +static void sec_free_civ_resource(struct device *dev, struct sec_alg_res *res) +{ + if (res->c_ivin) + dma_free_coherent(dev, SEC_TOTAL_IV_SZ, + res->c_ivin, res->c_ivin_dma); +} + +static int sec_alloc_mac_resource(struct device *dev, struct sec_alg_res *res) +{ + int i; + + res->out_mac = dma_alloc_coherent(dev, SEC_TOTAL_MAC_SZ << 1, + &res->out_mac_dma, GFP_KERNEL); + if (!res->out_mac) + return -ENOMEM; + + for (i = 1; i < QM_Q_DEPTH; i++) { + res[i].out_mac_dma = res->out_mac_dma + + i * (SEC_MAX_MAC_LEN << 1); + res[i].out_mac = res->out_mac + i * (SEC_MAX_MAC_LEN << 1); + } + + return 0; +} + +static void sec_free_mac_resource(struct device *dev, struct sec_alg_res *res) +{ + if (res->out_mac) + dma_free_coherent(dev, SEC_TOTAL_MAC_SZ << 1, + res->out_mac, res->out_mac_dma); +} + +static void sec_free_pbuf_resource(struct device *dev, struct sec_alg_res *res) +{ + if (res->pbuf) + dma_free_coherent(dev, SEC_TOTAL_PBUF_SZ, + res->pbuf, res->pbuf_dma); +} + +/* + * To improve performance, pbuffer is used for + * small packets (< 512Bytes) as IOMMU translation using. + */ +static int sec_alloc_pbuf_resource(struct device *dev, struct sec_alg_res *res) +{ + int pbuf_page_offset; + int i, j, k; + + res->pbuf = dma_alloc_coherent(dev, SEC_TOTAL_PBUF_SZ, + &res->pbuf_dma, GFP_KERNEL); + if (!res->pbuf) + return -ENOMEM; + + /* + * SEC_PBUF_PKG contains data pbuf, iv and + * out_mac : + * Every PAGE contains six SEC_PBUF_PKG + * The sec_qp_ctx contains QM_Q_DEPTH numbers of SEC_PBUF_PKG + * So we need SEC_PBUF_PAGE_NUM numbers of PAGE + * for the SEC_TOTAL_PBUF_SZ + */ + for (i = 0; i <= SEC_PBUF_PAGE_NUM; i++) { + pbuf_page_offset = PAGE_SIZE * i; + for (j = 0; j < SEC_PBUF_NUM; j++) { + k = i * SEC_PBUF_NUM + j; + if (k == QM_Q_DEPTH) + break; + res[k].pbuf = res->pbuf + + j * SEC_PBUF_PKG + pbuf_page_offset; + res[k].pbuf_dma = res->pbuf_dma + + j * SEC_PBUF_PKG + pbuf_page_offset; + } + } + return 0; +} + +static int sec_alg_resource_alloc(struct sec_ctx *ctx, + struct sec_qp_ctx *qp_ctx) +{ + struct sec_alg_res *res = qp_ctx->res; + struct device *dev = ctx->dev; + int ret; + + ret = sec_alloc_civ_resource(dev, res); + if (ret) + return ret; + + if (ctx->alg_type == SEC_AEAD) { + ret = sec_alloc_mac_resource(dev, res); + if (ret) + goto alloc_fail; + } + if (ctx->pbuf_supported) { + ret = sec_alloc_pbuf_resource(dev, res); + if (ret) { + dev_err(dev, "fail to alloc pbuf dma resource!\n"); + goto alloc_pbuf_fail; + } + } + + return 0; +alloc_pbuf_fail: + if (ctx->alg_type == SEC_AEAD) + sec_free_mac_resource(dev, qp_ctx->res); +alloc_fail: + sec_free_civ_resource(dev, res); + + return ret; +} + +static void sec_alg_resource_free(struct sec_ctx *ctx, + struct sec_qp_ctx *qp_ctx) +{ + struct device *dev = ctx->dev; + + sec_free_civ_resource(dev, qp_ctx->res); + + if (ctx->pbuf_supported) + sec_free_pbuf_resource(dev, qp_ctx->res); + if (ctx->alg_type == SEC_AEAD) + sec_free_mac_resource(dev, qp_ctx->res); +} + +static int sec_create_qp_ctx(struct hisi_qm *qm, struct sec_ctx *ctx, + int qp_ctx_id, int alg_type) +{ + struct device *dev = ctx->dev; + struct sec_qp_ctx *qp_ctx; + struct hisi_qp *qp; + int ret = -ENOMEM; + + qp_ctx = &ctx->qp_ctx[qp_ctx_id]; + qp = ctx->qps[qp_ctx_id]; + qp->req_type = 0; + qp->qp_ctx = qp_ctx; + qp->req_cb = sec_req_cb; + qp_ctx->qp = qp; + qp_ctx->ctx = ctx; + + spin_lock_init(&qp_ctx->req_lock); + idr_init(&qp_ctx->req_idr); + INIT_LIST_HEAD(&qp_ctx->backlog); + + qp_ctx->c_in_pool = hisi_acc_create_sgl_pool(dev, QM_Q_DEPTH, + SEC_SGL_SGE_NR); + if (IS_ERR(qp_ctx->c_in_pool)) { + dev_err(dev, "fail to create sgl pool for input!\n"); + goto err_destroy_idr; + } + + qp_ctx->c_out_pool = hisi_acc_create_sgl_pool(dev, QM_Q_DEPTH, + SEC_SGL_SGE_NR); + if (IS_ERR(qp_ctx->c_out_pool)) { + dev_err(dev, "fail to create sgl pool for output!\n"); + goto err_free_c_in_pool; + } + + ret = sec_alg_resource_alloc(ctx, qp_ctx); + if (ret) + goto err_free_c_out_pool; + + ret = hisi_qm_start_qp(qp, 0); + if (ret < 0) + goto err_queue_free; + + return 0; + +err_queue_free: + sec_alg_resource_free(ctx, qp_ctx); +err_free_c_out_pool: + hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool); +err_free_c_in_pool: + hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool); +err_destroy_idr: + idr_destroy(&qp_ctx->req_idr); + + return ret; +} + +static void sec_release_qp_ctx(struct sec_ctx *ctx, + struct sec_qp_ctx *qp_ctx) +{ + struct device *dev = ctx->dev; + + hisi_qm_stop_qp(qp_ctx->qp); + sec_alg_resource_free(ctx, qp_ctx); + + hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool); + hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool); + + idr_destroy(&qp_ctx->req_idr); +} + +static int sec_ctx_base_init(struct sec_ctx *ctx) +{ + struct sec_dev *sec; + int i, ret; + + ctx->qps = sec_create_qps(); + if (!ctx->qps) { + pr_err("Can not create sec qps!\n"); + return -ENODEV; + } + + sec = container_of(ctx->qps[0]->qm, struct sec_dev, qm); + ctx->sec = sec; + ctx->dev = &sec->qm.pdev->dev; + ctx->hlf_q_num = sec->ctx_q_num >> 1; + + ctx->pbuf_supported = ctx->sec->iommu_used; + + /* Half of queue depth is taken as fake requests limit in the queue. */ + ctx->fake_req_limit = QM_Q_DEPTH >> 1; + ctx->qp_ctx = kcalloc(sec->ctx_q_num, sizeof(struct sec_qp_ctx), + GFP_KERNEL); + if (!ctx->qp_ctx) { + ret = -ENOMEM; + goto err_destroy_qps; + } + + for (i = 0; i < sec->ctx_q_num; i++) { + ret = sec_create_qp_ctx(&sec->qm, ctx, i, 0); + if (ret) + goto err_sec_release_qp_ctx; + } + + return 0; + +err_sec_release_qp_ctx: + for (i = i - 1; i >= 0; i--) + sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]); + kfree(ctx->qp_ctx); +err_destroy_qps: + sec_destroy_qps(ctx->qps, sec->ctx_q_num); + return ret; +} + +static void sec_ctx_base_uninit(struct sec_ctx *ctx) +{ + int i; + + for (i = 0; i < ctx->sec->ctx_q_num; i++) + sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]); + + sec_destroy_qps(ctx->qps, ctx->sec->ctx_q_num); + kfree(ctx->qp_ctx); +} + +static int sec_cipher_init(struct sec_ctx *ctx) +{ + struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; + + c_ctx->c_key = dma_alloc_coherent(ctx->dev, SEC_MAX_KEY_SIZE, + &c_ctx->c_key_dma, GFP_KERNEL); + if (!c_ctx->c_key) + return -ENOMEM; + + return 0; +} + +static void sec_cipher_uninit(struct sec_ctx *ctx) +{ + struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; + + memzero_explicit(c_ctx->c_key, SEC_MAX_KEY_SIZE); + dma_free_coherent(ctx->dev, SEC_MAX_KEY_SIZE, + c_ctx->c_key, c_ctx->c_key_dma); +} + +static int sec_auth_init(struct sec_ctx *ctx) +{ + struct sec_auth_ctx *a_ctx = &ctx->a_ctx; + + a_ctx->a_key = dma_alloc_coherent(ctx->dev, SEC_MAX_AKEY_SIZE, + &a_ctx->a_key_dma, GFP_KERNEL); + if (!a_ctx->a_key) + return -ENOMEM; + + return 0; +} + +static void sec_auth_uninit(struct sec_ctx *ctx) +{ + struct sec_auth_ctx *a_ctx = &ctx->a_ctx; + + memzero_explicit(a_ctx->a_key, SEC_MAX_AKEY_SIZE); + dma_free_coherent(ctx->dev, SEC_MAX_AKEY_SIZE, + a_ctx->a_key, a_ctx->a_key_dma); +} + +static int sec_skcipher_init(struct crypto_skcipher *tfm) +{ + struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); + int ret; + + ctx->alg_type = SEC_SKCIPHER; + crypto_skcipher_set_reqsize(tfm, sizeof(struct sec_req)); + ctx->c_ctx.ivsize = crypto_skcipher_ivsize(tfm); + if (ctx->c_ctx.ivsize > SEC_IV_SIZE) { + pr_err("get error skcipher iv size!\n"); + return -EINVAL; + } + + ret = sec_ctx_base_init(ctx); + if (ret) + return ret; + + ret = sec_cipher_init(ctx); + if (ret) + goto err_cipher_init; + + return 0; +err_cipher_init: + sec_ctx_base_uninit(ctx); + + return ret; +} + +static void sec_skcipher_uninit(struct crypto_skcipher *tfm) +{ + struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); + + sec_cipher_uninit(ctx); + sec_ctx_base_uninit(ctx); +} + +static int sec_skcipher_3des_setkey(struct sec_cipher_ctx *c_ctx, + const u32 keylen, + const enum sec_cmode c_mode) +{ + switch (keylen) { + case SEC_DES3_2KEY_SIZE: + c_ctx->c_key_len = SEC_CKEY_3DES_2KEY; + break; + case SEC_DES3_3KEY_SIZE: + c_ctx->c_key_len = SEC_CKEY_3DES_3KEY; + break; + default: + return -EINVAL; + } + + return 0; +} + +static int sec_skcipher_aes_sm4_setkey(struct sec_cipher_ctx *c_ctx, + const u32 keylen, + const enum sec_cmode c_mode) +{ + if (c_mode == SEC_CMODE_XTS) { + switch (keylen) { + case SEC_XTS_MIN_KEY_SIZE: + c_ctx->c_key_len = SEC_CKEY_128BIT; + break; + case SEC_XTS_MAX_KEY_SIZE: + c_ctx->c_key_len = SEC_CKEY_256BIT; + break; + default: + pr_err("hisi_sec2: xts mode key error!\n"); + return -EINVAL; + } + } else { + switch (keylen) { + case AES_KEYSIZE_128: + c_ctx->c_key_len = SEC_CKEY_128BIT; + break; + case AES_KEYSIZE_192: + c_ctx->c_key_len = SEC_CKEY_192BIT; + break; + case AES_KEYSIZE_256: + c_ctx->c_key_len = SEC_CKEY_256BIT; + break; + default: + pr_err("hisi_sec2: aes key error!\n"); + return -EINVAL; + } + } + + return 0; +} + +static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, + const u32 keylen, const enum sec_calg c_alg, + const enum sec_cmode c_mode) +{ + struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); + struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; + struct device *dev = ctx->dev; + int ret; + + if (c_mode == SEC_CMODE_XTS) { + ret = xts_verify_key(tfm, key, keylen); + if (ret) { + dev_err(dev, "xts mode key err!\n"); + return ret; + } + } + + c_ctx->c_alg = c_alg; + c_ctx->c_mode = c_mode; + + switch (c_alg) { + case SEC_CALG_3DES: + ret = sec_skcipher_3des_setkey(c_ctx, keylen, c_mode); + break; + case SEC_CALG_AES: + case SEC_CALG_SM4: + ret = sec_skcipher_aes_sm4_setkey(c_ctx, keylen, c_mode); + break; + default: + return -EINVAL; + } + + if (ret) { + dev_err(dev, "set sec key err!\n"); + return ret; + } + + memcpy(c_ctx->c_key, key, keylen); + + return 0; +} + +#define GEN_SEC_SETKEY_FUNC(name, c_alg, c_mode) \ +static int sec_setkey_##name(struct crypto_skcipher *tfm, const u8 *key,\ + u32 keylen) \ +{ \ + return sec_skcipher_setkey(tfm, key, keylen, c_alg, c_mode); \ +} + +GEN_SEC_SETKEY_FUNC(aes_ecb, SEC_CALG_AES, SEC_CMODE_ECB) +GEN_SEC_SETKEY_FUNC(aes_cbc, SEC_CALG_AES, SEC_CMODE_CBC) +GEN_SEC_SETKEY_FUNC(aes_xts, SEC_CALG_AES, SEC_CMODE_XTS) + +GEN_SEC_SETKEY_FUNC(3des_ecb, SEC_CALG_3DES, SEC_CMODE_ECB) +GEN_SEC_SETKEY_FUNC(3des_cbc, SEC_CALG_3DES, SEC_CMODE_CBC) + +GEN_SEC_SETKEY_FUNC(sm4_xts, SEC_CALG_SM4, SEC_CMODE_XTS) +GEN_SEC_SETKEY_FUNC(sm4_cbc, SEC_CALG_SM4, SEC_CMODE_CBC) + +static int sec_cipher_pbuf_map(struct sec_ctx *ctx, struct sec_req *req, + struct scatterlist *src) +{ + struct aead_request *aead_req = req->aead_req.aead_req; + struct sec_cipher_req *c_req = &req->c_req; + struct sec_qp_ctx *qp_ctx = req->qp_ctx; + struct device *dev = ctx->dev; + int copy_size, pbuf_length; + int req_id = req->req_id; + + if (ctx->alg_type == SEC_AEAD) + copy_size = aead_req->cryptlen + aead_req->assoclen; + else + copy_size = c_req->c_len; + + pbuf_length = sg_copy_to_buffer(src, sg_nents(src), + qp_ctx->res[req_id].pbuf, + copy_size); + if (unlikely(pbuf_length != copy_size)) { + dev_err(dev, "copy src data to pbuf error!\n"); + return -EINVAL; + } + + c_req->c_in_dma = qp_ctx->res[req_id].pbuf_dma; + + if (!c_req->c_in_dma) { + dev_err(dev, "fail to set pbuffer address!\n"); + return -ENOMEM; + } + + c_req->c_out_dma = c_req->c_in_dma; + + return 0; +} + +static void sec_cipher_pbuf_unmap(struct sec_ctx *ctx, struct sec_req *req, + struct scatterlist *dst) +{ + struct aead_request *aead_req = req->aead_req.aead_req; + struct sec_cipher_req *c_req = &req->c_req; + struct sec_qp_ctx *qp_ctx = req->qp_ctx; + struct device *dev = ctx->dev; + int copy_size, pbuf_length; + int req_id = req->req_id; + + if (ctx->alg_type == SEC_AEAD) + copy_size = c_req->c_len + aead_req->assoclen; + else + copy_size = c_req->c_len; + + pbuf_length = sg_copy_from_buffer(dst, sg_nents(dst), + qp_ctx->res[req_id].pbuf, + copy_size); + if (unlikely(pbuf_length != copy_size)) + dev_err(dev, "copy pbuf data to dst error!\n"); + +} + +static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req, + struct scatterlist *src, struct scatterlist *dst) +{ + struct sec_cipher_req *c_req = &req->c_req; + struct sec_aead_req *a_req = &req->aead_req; + struct sec_qp_ctx *qp_ctx = req->qp_ctx; + struct sec_alg_res *res = &qp_ctx->res[req->req_id]; + struct device *dev = ctx->dev; + int ret; + + if (req->use_pbuf) { + ret = sec_cipher_pbuf_map(ctx, req, src); + c_req->c_ivin = res->pbuf + SEC_PBUF_IV_OFFSET; + c_req->c_ivin_dma = res->pbuf_dma + SEC_PBUF_IV_OFFSET; + if (ctx->alg_type == SEC_AEAD) { + a_req->out_mac = res->pbuf + SEC_PBUF_MAC_OFFSET; + a_req->out_mac_dma = res->pbuf_dma + + SEC_PBUF_MAC_OFFSET; + } + + return ret; + } + c_req->c_ivin = res->c_ivin; + c_req->c_ivin_dma = res->c_ivin_dma; + if (ctx->alg_type == SEC_AEAD) { + a_req->out_mac = res->out_mac; + a_req->out_mac_dma = res->out_mac_dma; + } + + c_req->c_in = hisi_acc_sg_buf_map_to_hw_sgl(dev, src, + qp_ctx->c_in_pool, + req->req_id, + &c_req->c_in_dma); + + if (IS_ERR(c_req->c_in)) { + dev_err(dev, "fail to dma map input sgl buffers!\n"); + return PTR_ERR(c_req->c_in); + } + + if (dst == src) { + c_req->c_out = c_req->c_in; + c_req->c_out_dma = c_req->c_in_dma; + } else { + c_req->c_out = hisi_acc_sg_buf_map_to_hw_sgl(dev, dst, + qp_ctx->c_out_pool, + req->req_id, + &c_req->c_out_dma); + + if (IS_ERR(c_req->c_out)) { + dev_err(dev, "fail to dma map output sgl buffers!\n"); + hisi_acc_sg_buf_unmap(dev, src, c_req->c_in); + return PTR_ERR(c_req->c_out); + } + } + + return 0; +} + +static void sec_cipher_unmap(struct sec_ctx *ctx, struct sec_req *req, + struct scatterlist *src, struct scatterlist *dst) +{ + struct sec_cipher_req *c_req = &req->c_req; + struct device *dev = ctx->dev; + + if (req->use_pbuf) { + sec_cipher_pbuf_unmap(ctx, req, dst); + } else { + if (dst != src) + hisi_acc_sg_buf_unmap(dev, src, c_req->c_in); + + hisi_acc_sg_buf_unmap(dev, dst, c_req->c_out); + } +} + +static int sec_skcipher_sgl_map(struct sec_ctx *ctx, struct sec_req *req) +{ + struct skcipher_request *sq = req->c_req.sk_req; + + return sec_cipher_map(ctx, req, sq->src, sq->dst); +} + +static void sec_skcipher_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req) +{ + struct skcipher_request *sq = req->c_req.sk_req; + + sec_cipher_unmap(ctx, req, sq->src, sq->dst); +} + +static int sec_aead_aes_set_key(struct sec_cipher_ctx *c_ctx, + struct crypto_authenc_keys *keys) +{ + switch (keys->enckeylen) { + case AES_KEYSIZE_128: + c_ctx->c_key_len = SEC_CKEY_128BIT; + break; + case AES_KEYSIZE_192: + c_ctx->c_key_len = SEC_CKEY_192BIT; + break; + case AES_KEYSIZE_256: + c_ctx->c_key_len = SEC_CKEY_256BIT; + break; + default: + pr_err("hisi_sec2: aead aes key error!\n"); + return -EINVAL; + } + memcpy(c_ctx->c_key, keys->enckey, keys->enckeylen); + + return 0; +} + +static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx, + struct crypto_authenc_keys *keys) +{ + struct crypto_shash *hash_tfm = ctx->hash_tfm; + int blocksize, ret; + + if (!keys->authkeylen) { + pr_err("hisi_sec2: aead auth key error!\n"); + return -EINVAL; + } + + blocksize = crypto_shash_blocksize(hash_tfm); + if (keys->authkeylen > blocksize) { + ret = crypto_shash_tfm_digest(hash_tfm, keys->authkey, + keys->authkeylen, ctx->a_key); + if (ret) { + pr_err("hisi_sec2: aead auth digest error!\n"); + return -EINVAL; + } + ctx->a_key_len = blocksize; + } else { + memcpy(ctx->a_key, keys->authkey, keys->authkeylen); + ctx->a_key_len = keys->authkeylen; + } + + return 0; +} + +static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, + const u32 keylen, const enum sec_hash_alg a_alg, + const enum sec_calg c_alg, + const enum sec_mac_len mac_len, + const enum sec_cmode c_mode) +{ + struct sec_ctx *ctx = crypto_aead_ctx(tfm); + struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; + struct device *dev = ctx->dev; + struct crypto_authenc_keys keys; + int ret; + + ctx->a_ctx.a_alg = a_alg; + ctx->c_ctx.c_alg = c_alg; + ctx->a_ctx.mac_len = mac_len; + c_ctx->c_mode = c_mode; + + if (crypto_authenc_extractkeys(&keys, key, keylen)) + goto bad_key; + + ret = sec_aead_aes_set_key(c_ctx, &keys); + if (ret) { + dev_err(dev, "set sec cipher key err!\n"); + goto bad_key; + } + + ret = sec_aead_auth_set_key(&ctx->a_ctx, &keys); + if (ret) { + dev_err(dev, "set sec auth key err!\n"); + goto bad_key; + } + + return 0; +bad_key: + memzero_explicit(&keys, sizeof(struct crypto_authenc_keys)); + + return -EINVAL; +} + + +#define GEN_SEC_AEAD_SETKEY_FUNC(name, aalg, calg, maclen, cmode) \ +static int sec_setkey_##name(struct crypto_aead *tfm, const u8 *key, \ + u32 keylen) \ +{ \ + return sec_aead_setkey(tfm, key, keylen, aalg, calg, maclen, cmode);\ +} + +GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha1, SEC_A_HMAC_SHA1, + SEC_CALG_AES, SEC_HMAC_SHA1_MAC, SEC_CMODE_CBC) +GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha256, SEC_A_HMAC_SHA256, + SEC_CALG_AES, SEC_HMAC_SHA256_MAC, SEC_CMODE_CBC) +GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha512, SEC_A_HMAC_SHA512, + SEC_CALG_AES, SEC_HMAC_SHA512_MAC, SEC_CMODE_CBC) + +static int sec_aead_sgl_map(struct sec_ctx *ctx, struct sec_req *req) +{ + struct aead_request *aq = req->aead_req.aead_req; + + return sec_cipher_map(ctx, req, aq->src, aq->dst); +} + +static void sec_aead_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req) +{ + struct aead_request *aq = req->aead_req.aead_req; + + sec_cipher_unmap(ctx, req, aq->src, aq->dst); +} + +static int sec_request_transfer(struct sec_ctx *ctx, struct sec_req *req) +{ + int ret; + + ret = ctx->req_op->buf_map(ctx, req); + if (unlikely(ret)) + return ret; + + ctx->req_op->do_transfer(ctx, req); + + ret = ctx->req_op->bd_fill(ctx, req); + if (unlikely(ret)) + goto unmap_req_buf; + + return ret; + +unmap_req_buf: + ctx->req_op->buf_unmap(ctx, req); + + return ret; +} + +static void sec_request_untransfer(struct sec_ctx *ctx, struct sec_req *req) +{ + ctx->req_op->buf_unmap(ctx, req); +} + +static void sec_skcipher_copy_iv(struct sec_ctx *ctx, struct sec_req *req) +{ + struct skcipher_request *sk_req = req->c_req.sk_req; + struct sec_cipher_req *c_req = &req->c_req; + + memcpy(c_req->c_ivin, sk_req->iv, ctx->c_ctx.ivsize); +} + +static int sec_skcipher_bd_fill(struct sec_ctx *ctx, struct sec_req *req) +{ + struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; + struct sec_cipher_req *c_req = &req->c_req; + struct sec_sqe *sec_sqe = &req->sec_sqe; + u8 scene, sa_type, da_type; + u8 bd_type, cipher; + u8 de = 0; + + memset(sec_sqe, 0, sizeof(struct sec_sqe)); + + sec_sqe->type2.c_key_addr = cpu_to_le64(c_ctx->c_key_dma); + sec_sqe->type2.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma); + sec_sqe->type2.data_src_addr = cpu_to_le64(c_req->c_in_dma); + sec_sqe->type2.data_dst_addr = cpu_to_le64(c_req->c_out_dma); + + sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_mode) << + SEC_CMODE_OFFSET); + sec_sqe->type2.c_alg = c_ctx->c_alg; + sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_key_len) << + SEC_CKEY_OFFSET); + + bd_type = SEC_BD_TYPE2; + if (c_req->encrypt) + cipher = SEC_CIPHER_ENC << SEC_CIPHER_OFFSET; + else + cipher = SEC_CIPHER_DEC << SEC_CIPHER_OFFSET; + sec_sqe->type_cipher_auth = bd_type | cipher; + + if (req->use_pbuf) + sa_type = SEC_PBUF << SEC_SRC_SGL_OFFSET; + else + sa_type = SEC_SGL << SEC_SRC_SGL_OFFSET; + scene = SEC_COMM_SCENE << SEC_SCENE_OFFSET; + if (c_req->c_in_dma != c_req->c_out_dma) + de = 0x1 << SEC_DE_OFFSET; + + sec_sqe->sds_sa_type = (de | scene | sa_type); + + /* Just set DST address type */ + if (req->use_pbuf) + da_type = SEC_PBUF << SEC_DST_SGL_OFFSET; + else + da_type = SEC_SGL << SEC_DST_SGL_OFFSET; + sec_sqe->sdm_addr_type |= da_type; + + sec_sqe->type2.clen_ivhlen |= cpu_to_le32(c_req->c_len); + sec_sqe->type2.tag = cpu_to_le16((u16)req->req_id); + + return 0; +} + +static void sec_update_iv(struct sec_req *req, enum sec_alg_type alg_type) +{ + struct aead_request *aead_req = req->aead_req.aead_req; + struct skcipher_request *sk_req = req->c_req.sk_req; + u32 iv_size = req->ctx->c_ctx.ivsize; + struct scatterlist *sgl; + unsigned int cryptlen; + size_t sz; + u8 *iv; + + if (req->c_req.encrypt) + sgl = alg_type == SEC_SKCIPHER ? sk_req->dst : aead_req->dst; + else + sgl = alg_type == SEC_SKCIPHER ? sk_req->src : aead_req->src; + + if (alg_type == SEC_SKCIPHER) { + iv = sk_req->iv; + cryptlen = sk_req->cryptlen; + } else { + iv = aead_req->iv; + cryptlen = aead_req->cryptlen; + } + + sz = sg_pcopy_to_buffer(sgl, sg_nents(sgl), iv, iv_size, + cryptlen - iv_size); + if (unlikely(sz != iv_size)) + dev_err(req->ctx->dev, "copy output iv error!\n"); +} + +static struct sec_req *sec_back_req_clear(struct sec_ctx *ctx, + struct sec_qp_ctx *qp_ctx) +{ + struct sec_req *backlog_req = NULL; + + spin_lock_bh(&qp_ctx->req_lock); + if (ctx->fake_req_limit >= + atomic_read(&qp_ctx->qp->qp_status.used) && + !list_empty(&qp_ctx->backlog)) { + backlog_req = list_first_entry(&qp_ctx->backlog, + typeof(*backlog_req), backlog_head); + list_del(&backlog_req->backlog_head); + } + spin_unlock_bh(&qp_ctx->req_lock); + + return backlog_req; +} + +static void sec_skcipher_callback(struct sec_ctx *ctx, struct sec_req *req, + int err) +{ + struct skcipher_request *sk_req = req->c_req.sk_req; + struct sec_qp_ctx *qp_ctx = req->qp_ctx; + struct skcipher_request *backlog_sk_req; + struct sec_req *backlog_req; + + sec_free_req_id(req); + + /* IV output at encrypto of CBC mode */ + if (!err && ctx->c_ctx.c_mode == SEC_CMODE_CBC && req->c_req.encrypt) + sec_update_iv(req, SEC_SKCIPHER); + + while (1) { + backlog_req = sec_back_req_clear(ctx, qp_ctx); + if (!backlog_req) + break; + + backlog_sk_req = backlog_req->c_req.sk_req; + backlog_sk_req->base.complete(&backlog_sk_req->base, + -EINPROGRESS); + atomic64_inc(&ctx->sec->debug.dfx.recv_busy_cnt); + } + + + sk_req->base.complete(&sk_req->base, err); +} + +static void sec_aead_copy_iv(struct sec_ctx *ctx, struct sec_req *req) +{ + struct aead_request *aead_req = req->aead_req.aead_req; + struct sec_cipher_req *c_req = &req->c_req; + + memcpy(c_req->c_ivin, aead_req->iv, ctx->c_ctx.ivsize); +} + +static void sec_auth_bd_fill_ex(struct sec_auth_ctx *ctx, int dir, + struct sec_req *req, struct sec_sqe *sec_sqe) +{ + struct sec_aead_req *a_req = &req->aead_req; + struct sec_cipher_req *c_req = &req->c_req; + struct aead_request *aq = a_req->aead_req; + + sec_sqe->type2.a_key_addr = cpu_to_le64(ctx->a_key_dma); + + sec_sqe->type2.mac_key_alg = + cpu_to_le32(ctx->mac_len / SEC_SQE_LEN_RATE); + + sec_sqe->type2.mac_key_alg |= + cpu_to_le32((u32)((ctx->a_key_len) / + SEC_SQE_LEN_RATE) << SEC_AKEY_OFFSET); + + sec_sqe->type2.mac_key_alg |= + cpu_to_le32((u32)(ctx->a_alg) << SEC_AEAD_ALG_OFFSET); + + sec_sqe->type_cipher_auth |= SEC_AUTH_TYPE1 << SEC_AUTH_OFFSET; + + if (dir) + sec_sqe->sds_sa_type &= SEC_CIPHER_AUTH; + else + sec_sqe->sds_sa_type |= SEC_AUTH_CIPHER; + + sec_sqe->type2.alen_ivllen = cpu_to_le32(c_req->c_len + aq->assoclen); + + sec_sqe->type2.cipher_src_offset = cpu_to_le16((u16)aq->assoclen); + + sec_sqe->type2.mac_addr = cpu_to_le64(a_req->out_mac_dma); +} + +static int sec_aead_bd_fill(struct sec_ctx *ctx, struct sec_req *req) +{ + struct sec_auth_ctx *auth_ctx = &ctx->a_ctx; + struct sec_sqe *sec_sqe = &req->sec_sqe; + int ret; + + ret = sec_skcipher_bd_fill(ctx, req); + if (unlikely(ret)) { + dev_err(ctx->dev, "skcipher bd fill is error!\n"); + return ret; + } + + sec_auth_bd_fill_ex(auth_ctx, req->c_req.encrypt, req, sec_sqe); + + return 0; +} + +static void sec_aead_callback(struct sec_ctx *c, struct sec_req *req, int err) +{ + struct aead_request *a_req = req->aead_req.aead_req; + struct crypto_aead *tfm = crypto_aead_reqtfm(a_req); + struct sec_aead_req *aead_req = &req->aead_req; + struct sec_cipher_req *c_req = &req->c_req; + size_t authsize = crypto_aead_authsize(tfm); + struct sec_qp_ctx *qp_ctx = req->qp_ctx; + struct aead_request *backlog_aead_req; + struct sec_req *backlog_req; + size_t sz; + + if (!err && c->c_ctx.c_mode == SEC_CMODE_CBC && c_req->encrypt) + sec_update_iv(req, SEC_AEAD); + + /* Copy output mac */ + if (!err && c_req->encrypt) { + struct scatterlist *sgl = a_req->dst; + + sz = sg_pcopy_from_buffer(sgl, sg_nents(sgl), + aead_req->out_mac, + authsize, a_req->cryptlen + + a_req->assoclen); + + if (unlikely(sz != authsize)) { + dev_err(c->dev, "copy out mac err!\n"); + err = -EINVAL; + } + } + + sec_free_req_id(req); + + while (1) { + backlog_req = sec_back_req_clear(c, qp_ctx); + if (!backlog_req) + break; + + backlog_aead_req = backlog_req->aead_req.aead_req; + backlog_aead_req->base.complete(&backlog_aead_req->base, + -EINPROGRESS); + atomic64_inc(&c->sec->debug.dfx.recv_busy_cnt); + } + + a_req->base.complete(&a_req->base, err); +} + +static void sec_request_uninit(struct sec_ctx *ctx, struct sec_req *req) +{ + sec_free_req_id(req); + sec_free_queue_id(ctx, req); +} + +static int sec_request_init(struct sec_ctx *ctx, struct sec_req *req) +{ + struct sec_qp_ctx *qp_ctx; + int queue_id; + + /* To load balance */ + queue_id = sec_alloc_queue_id(ctx, req); + qp_ctx = &ctx->qp_ctx[queue_id]; + + req->req_id = sec_alloc_req_id(req, qp_ctx); + if (unlikely(req->req_id < 0)) { + sec_free_queue_id(ctx, req); + return req->req_id; + } + + return 0; +} + +static int sec_process(struct sec_ctx *ctx, struct sec_req *req) +{ + struct sec_cipher_req *c_req = &req->c_req; + int ret; + + ret = sec_request_init(ctx, req); + if (unlikely(ret)) + return ret; + + ret = sec_request_transfer(ctx, req); + if (unlikely(ret)) + goto err_uninit_req; + + /* Output IV as decrypto */ + if (ctx->c_ctx.c_mode == SEC_CMODE_CBC && !req->c_req.encrypt) + sec_update_iv(req, ctx->alg_type); + + ret = ctx->req_op->bd_send(ctx, req); + if (unlikely((ret != -EBUSY && ret != -EINPROGRESS) || + (ret == -EBUSY && !(req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG)))) { + dev_err_ratelimited(ctx->dev, "send sec request failed!\n"); + goto err_send_req; + } + + return ret; + +err_send_req: + /* As failing, restore the IV from user */ + if (ctx->c_ctx.c_mode == SEC_CMODE_CBC && !req->c_req.encrypt) { + if (ctx->alg_type == SEC_SKCIPHER) + memcpy(req->c_req.sk_req->iv, c_req->c_ivin, + ctx->c_ctx.ivsize); + else + memcpy(req->aead_req.aead_req->iv, c_req->c_ivin, + ctx->c_ctx.ivsize); + } + + sec_request_untransfer(ctx, req); +err_uninit_req: + sec_request_uninit(ctx, req); + + return ret; +} + +static const struct sec_req_op sec_skcipher_req_ops = { + .buf_map = sec_skcipher_sgl_map, + .buf_unmap = sec_skcipher_sgl_unmap, + .do_transfer = sec_skcipher_copy_iv, + .bd_fill = sec_skcipher_bd_fill, + .bd_send = sec_bd_send, + .callback = sec_skcipher_callback, + .process = sec_process, +}; + +static const struct sec_req_op sec_aead_req_ops = { + .buf_map = sec_aead_sgl_map, + .buf_unmap = sec_aead_sgl_unmap, + .do_transfer = sec_aead_copy_iv, + .bd_fill = sec_aead_bd_fill, + .bd_send = sec_bd_send, + .callback = sec_aead_callback, + .process = sec_process, +}; + +static int sec_skcipher_ctx_init(struct crypto_skcipher *tfm) +{ + struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); + + ctx->req_op = &sec_skcipher_req_ops; + + return sec_skcipher_init(tfm); +} + +static void sec_skcipher_ctx_exit(struct crypto_skcipher *tfm) +{ + sec_skcipher_uninit(tfm); +} + +static int sec_aead_init(struct crypto_aead *tfm) +{ + struct sec_ctx *ctx = crypto_aead_ctx(tfm); + int ret; + + crypto_aead_set_reqsize(tfm, sizeof(struct sec_req)); + ctx->alg_type = SEC_AEAD; + ctx->c_ctx.ivsize = crypto_aead_ivsize(tfm); + if (ctx->c_ctx.ivsize > SEC_IV_SIZE) { + dev_err(ctx->dev, "get error aead iv size!\n"); + return -EINVAL; + } + + ctx->req_op = &sec_aead_req_ops; + ret = sec_ctx_base_init(ctx); + if (ret) + return ret; + + ret = sec_auth_init(ctx); + if (ret) + goto err_auth_init; + + ret = sec_cipher_init(ctx); + if (ret) + goto err_cipher_init; + + return ret; + +err_cipher_init: + sec_auth_uninit(ctx); +err_auth_init: + sec_ctx_base_uninit(ctx); + + return ret; +} + +static void sec_aead_exit(struct crypto_aead *tfm) +{ + struct sec_ctx *ctx = crypto_aead_ctx(tfm); + + sec_cipher_uninit(ctx); + sec_auth_uninit(ctx); + sec_ctx_base_uninit(ctx); +} + +static int sec_aead_ctx_init(struct crypto_aead *tfm, const char *hash_name) +{ + struct sec_ctx *ctx = crypto_aead_ctx(tfm); + struct sec_auth_ctx *auth_ctx = &ctx->a_ctx; + int ret; + + ret = sec_aead_init(tfm); + if (ret) { + pr_err("hisi_sec2: aead init error!\n"); + return ret; + } + + auth_ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0); + if (IS_ERR(auth_ctx->hash_tfm)) { + dev_err(ctx->dev, "aead alloc shash error!\n"); + sec_aead_exit(tfm); + return PTR_ERR(auth_ctx->hash_tfm); + } + + return 0; +} + +static void sec_aead_ctx_exit(struct crypto_aead *tfm) +{ + struct sec_ctx *ctx = crypto_aead_ctx(tfm); + + crypto_free_shash(ctx->a_ctx.hash_tfm); + sec_aead_exit(tfm); +} + +static int sec_aead_sha1_ctx_init(struct crypto_aead *tfm) +{ + return sec_aead_ctx_init(tfm, "sha1"); +} + +static int sec_aead_sha256_ctx_init(struct crypto_aead *tfm) +{ + return sec_aead_ctx_init(tfm, "sha256"); +} + +static int sec_aead_sha512_ctx_init(struct crypto_aead *tfm) +{ + return sec_aead_ctx_init(tfm, "sha512"); +} + +static int sec_skcipher_param_check(struct sec_ctx *ctx, struct sec_req *sreq) +{ + struct skcipher_request *sk_req = sreq->c_req.sk_req; + struct device *dev = ctx->dev; + u8 c_alg = ctx->c_ctx.c_alg; + + if (unlikely(!sk_req->src || !sk_req->dst)) { + dev_err(dev, "skcipher input param error!\n"); + return -EINVAL; + } + sreq->c_req.c_len = sk_req->cryptlen; + + if (ctx->pbuf_supported && sk_req->cryptlen <= SEC_PBUF_SZ) + sreq->use_pbuf = true; + else + sreq->use_pbuf = false; + + if (c_alg == SEC_CALG_3DES) { + if (unlikely(sk_req->cryptlen & (DES3_EDE_BLOCK_SIZE - 1))) { + dev_err(dev, "skcipher 3des input length error!\n"); + return -EINVAL; + } + return 0; + } else if (c_alg == SEC_CALG_AES || c_alg == SEC_CALG_SM4) { + if (unlikely(sk_req->cryptlen & (AES_BLOCK_SIZE - 1))) { + dev_err(dev, "skcipher aes input length error!\n"); + return -EINVAL; + } + return 0; + } + + dev_err(dev, "skcipher algorithm error!\n"); + return -EINVAL; +} + +static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(sk_req); + struct sec_req *req = skcipher_request_ctx(sk_req); + struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); + int ret; + + if (!sk_req->cryptlen) + return 0; + + req->flag = sk_req->base.flags; + req->c_req.sk_req = sk_req; + req->c_req.encrypt = encrypt; + req->ctx = ctx; + + ret = sec_skcipher_param_check(ctx, req); + if (unlikely(ret)) + return -EINVAL; + + return ctx->req_op->process(ctx, req); +} + +static int sec_skcipher_encrypt(struct skcipher_request *sk_req) +{ + return sec_skcipher_crypto(sk_req, true); +} + +static int sec_skcipher_decrypt(struct skcipher_request *sk_req) +{ + return sec_skcipher_crypto(sk_req, false); +} + +#define SEC_SKCIPHER_GEN_ALG(sec_cra_name, sec_set_key, sec_min_key_size, \ + sec_max_key_size, ctx_init, ctx_exit, blk_size, iv_size)\ +{\ + .base = {\ + .cra_name = sec_cra_name,\ + .cra_driver_name = "hisi_sec_"sec_cra_name,\ + .cra_priority = SEC_PRIORITY,\ + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,\ + .cra_blocksize = blk_size,\ + .cra_ctxsize = sizeof(struct sec_ctx),\ + .cra_module = THIS_MODULE,\ + },\ + .init = ctx_init,\ + .exit = ctx_exit,\ + .setkey = sec_set_key,\ + .decrypt = sec_skcipher_decrypt,\ + .encrypt = sec_skcipher_encrypt,\ + .min_keysize = sec_min_key_size,\ + .max_keysize = sec_max_key_size,\ + .ivsize = iv_size,\ +}, + +#define SEC_SKCIPHER_ALG(name, key_func, min_key_size, \ + max_key_size, blk_size, iv_size) \ + SEC_SKCIPHER_GEN_ALG(name, key_func, min_key_size, max_key_size, \ + sec_skcipher_ctx_init, sec_skcipher_ctx_exit, blk_size, iv_size) + +static struct skcipher_alg sec_skciphers[] = { + SEC_SKCIPHER_ALG("ecb(aes)", sec_setkey_aes_ecb, + AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE, + AES_BLOCK_SIZE, 0) + + SEC_SKCIPHER_ALG("cbc(aes)", sec_setkey_aes_cbc, + AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE, + AES_BLOCK_SIZE, AES_BLOCK_SIZE) + + SEC_SKCIPHER_ALG("xts(aes)", sec_setkey_aes_xts, + SEC_XTS_MIN_KEY_SIZE, SEC_XTS_MAX_KEY_SIZE, + AES_BLOCK_SIZE, AES_BLOCK_SIZE) + + SEC_SKCIPHER_ALG("ecb(des3_ede)", sec_setkey_3des_ecb, + SEC_DES3_3KEY_SIZE, SEC_DES3_3KEY_SIZE, + DES3_EDE_BLOCK_SIZE, 0) + + SEC_SKCIPHER_ALG("cbc(des3_ede)", sec_setkey_3des_cbc, + SEC_DES3_3KEY_SIZE, SEC_DES3_3KEY_SIZE, + DES3_EDE_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE) + + SEC_SKCIPHER_ALG("xts(sm4)", sec_setkey_sm4_xts, + SEC_XTS_MIN_KEY_SIZE, SEC_XTS_MIN_KEY_SIZE, + AES_BLOCK_SIZE, AES_BLOCK_SIZE) + + SEC_SKCIPHER_ALG("cbc(sm4)", sec_setkey_sm4_cbc, + AES_MIN_KEY_SIZE, AES_MIN_KEY_SIZE, + AES_BLOCK_SIZE, AES_BLOCK_SIZE) +}; + +static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq) +{ + struct aead_request *req = sreq->aead_req.aead_req; + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + size_t authsize = crypto_aead_authsize(tfm); + struct device *dev = ctx->dev; + u8 c_alg = ctx->c_ctx.c_alg; + + if (unlikely(!req->src || !req->dst || !req->cryptlen || + req->assoclen > SEC_MAX_AAD_LEN)) { + dev_err(dev, "aead input param error!\n"); + return -EINVAL; + } + + if (ctx->pbuf_supported && (req->cryptlen + req->assoclen) <= + SEC_PBUF_SZ) + sreq->use_pbuf = true; + else + sreq->use_pbuf = false; + + /* Support AES only */ + if (unlikely(c_alg != SEC_CALG_AES)) { + dev_err(dev, "aead crypto alg error!\n"); + return -EINVAL; + + } + if (sreq->c_req.encrypt) + sreq->c_req.c_len = req->cryptlen; + else + sreq->c_req.c_len = req->cryptlen - authsize; + + if (unlikely(sreq->c_req.c_len & (AES_BLOCK_SIZE - 1))) { + dev_err(dev, "aead crypto length error!\n"); + return -EINVAL; + } + + return 0; +} + +static int sec_aead_crypto(struct aead_request *a_req, bool encrypt) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(a_req); + struct sec_req *req = aead_request_ctx(a_req); + struct sec_ctx *ctx = crypto_aead_ctx(tfm); + int ret; + + req->flag = a_req->base.flags; + req->aead_req.aead_req = a_req; + req->c_req.encrypt = encrypt; + req->ctx = ctx; + + ret = sec_aead_param_check(ctx, req); + if (unlikely(ret)) + return -EINVAL; + + return ctx->req_op->process(ctx, req); +} + +static int sec_aead_encrypt(struct aead_request *a_req) +{ + return sec_aead_crypto(a_req, true); +} + +static int sec_aead_decrypt(struct aead_request *a_req) +{ + return sec_aead_crypto(a_req, false); +} + +#define SEC_AEAD_GEN_ALG(sec_cra_name, sec_set_key, ctx_init,\ + ctx_exit, blk_size, iv_size, max_authsize)\ +{\ + .base = {\ + .cra_name = sec_cra_name,\ + .cra_driver_name = "hisi_sec_"sec_cra_name,\ + .cra_priority = SEC_PRIORITY,\ + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,\ + .cra_blocksize = blk_size,\ + .cra_ctxsize = sizeof(struct sec_ctx),\ + .cra_module = THIS_MODULE,\ + },\ + .init = ctx_init,\ + .exit = ctx_exit,\ + .setkey = sec_set_key,\ + .decrypt = sec_aead_decrypt,\ + .encrypt = sec_aead_encrypt,\ + .ivsize = iv_size,\ + .maxauthsize = max_authsize,\ +} + +#define SEC_AEAD_ALG(algname, keyfunc, aead_init, blksize, ivsize, authsize)\ + SEC_AEAD_GEN_ALG(algname, keyfunc, aead_init,\ + sec_aead_ctx_exit, blksize, ivsize, authsize) + +static struct aead_alg sec_aeads[] = { + SEC_AEAD_ALG("authenc(hmac(sha1),cbc(aes))", + sec_setkey_aes_cbc_sha1, sec_aead_sha1_ctx_init, + AES_BLOCK_SIZE, AES_BLOCK_SIZE, SHA1_DIGEST_SIZE), + + SEC_AEAD_ALG("authenc(hmac(sha256),cbc(aes))", + sec_setkey_aes_cbc_sha256, sec_aead_sha256_ctx_init, + AES_BLOCK_SIZE, AES_BLOCK_SIZE, SHA256_DIGEST_SIZE), + + SEC_AEAD_ALG("authenc(hmac(sha512),cbc(aes))", + sec_setkey_aes_cbc_sha512, sec_aead_sha512_ctx_init, + AES_BLOCK_SIZE, AES_BLOCK_SIZE, SHA512_DIGEST_SIZE), +}; + +int sec_register_to_crypto(void) +{ + int ret; + + /* To avoid repeat register */ + ret = crypto_register_skciphers(sec_skciphers, + ARRAY_SIZE(sec_skciphers)); + if (ret) + return ret; + + ret = crypto_register_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); + if (ret) + crypto_unregister_skciphers(sec_skciphers, + ARRAY_SIZE(sec_skciphers)); + return ret; +} + +void sec_unregister_from_crypto(void) +{ + crypto_unregister_skciphers(sec_skciphers, + ARRAY_SIZE(sec_skciphers)); + crypto_unregister_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); +} diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.h b/drivers/crypto/hisilicon/sec2/sec_crypto.h new file mode 100644 index 000000000..20f11e5bb --- /dev/null +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.h @@ -0,0 +1,215 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2019 HiSilicon Limited. */ + +#ifndef __HISI_SEC_V2_CRYPTO_H +#define __HISI_SEC_V2_CRYPTO_H + +#define SEC_IV_SIZE 24 +#define SEC_MAX_KEY_SIZE 64 +#define SEC_MAX_AKEY_SIZE 128 +#define SEC_COMM_SCENE 0 + +enum sec_calg { + SEC_CALG_3DES = 0x1, + SEC_CALG_AES = 0x2, + SEC_CALG_SM4 = 0x3, +}; + +enum sec_hash_alg { + SEC_A_HMAC_SHA1 = 0x10, + SEC_A_HMAC_SHA256 = 0x11, + SEC_A_HMAC_SHA512 = 0x15, +}; + +enum sec_mac_len { + SEC_HMAC_SHA1_MAC = 20, + SEC_HMAC_SHA256_MAC = 32, + SEC_HMAC_SHA512_MAC = 64, +}; + +enum sec_cmode { + SEC_CMODE_ECB = 0x0, + SEC_CMODE_CBC = 0x1, + SEC_CMODE_CTR = 0x4, + SEC_CMODE_XTS = 0x7, +}; + +enum sec_ckey_type { + SEC_CKEY_128BIT = 0x0, + SEC_CKEY_192BIT = 0x1, + SEC_CKEY_256BIT = 0x2, + SEC_CKEY_3DES_3KEY = 0x1, + SEC_CKEY_3DES_2KEY = 0x3, +}; + +enum sec_bd_type { + SEC_BD_TYPE1 = 0x1, + SEC_BD_TYPE2 = 0x2, +}; + +enum sec_auth { + SEC_NO_AUTH = 0x0, + SEC_AUTH_TYPE1 = 0x1, + SEC_AUTH_TYPE2 = 0x2, +}; + +enum sec_cipher_dir { + SEC_CIPHER_ENC = 0x1, + SEC_CIPHER_DEC = 0x2, +}; + +enum sec_addr_type { + SEC_PBUF = 0x0, + SEC_SGL = 0x1, + SEC_PRP = 0x2, +}; + +struct sec_sqe_type2 { + /* + * mac_len: 0~4 bits + * a_key_len: 5~10 bits + * a_alg: 11~16 bits + */ + __le32 mac_key_alg; + + /* + * c_icv_len: 0~5 bits + * c_width: 6~8 bits + * c_key_len: 9~11 bits + * c_mode: 12~15 bits + */ + __le16 icvw_kmode; + + /* c_alg: 0~3 bits */ + __u8 c_alg; + __u8 rsvd4; + + /* + * a_len: 0~23 bits + * iv_offset_l: 24~31 bits + */ + __le32 alen_ivllen; + + /* + * c_len: 0~23 bits + * iv_offset_h: 24~31 bits + */ + __le32 clen_ivhlen; + + __le16 auth_src_offset; + __le16 cipher_src_offset; + __le16 cs_ip_header_offset; + __le16 cs_udp_header_offset; + __le16 pass_word_len; + __le16 dk_len; + __u8 salt3; + __u8 salt2; + __u8 salt1; + __u8 salt0; + + __le16 tag; + __le16 rsvd5; + + /* + * c_pad_type: 0~3 bits + * c_pad_len: 4~11 bits + * c_pad_data_type: 12~15 bits + */ + __le16 cph_pad; + + /* c_pad_len_field: 0~1 bits */ + __le16 c_pad_len_field; + + __le64 long_a_data_len; + __le64 a_ivin_addr; + __le64 a_key_addr; + __le64 mac_addr; + __le64 c_ivin_addr; + __le64 c_key_addr; + + __le64 data_src_addr; + __le64 data_dst_addr; + + /* + * done: 0 bit + * icv: 1~3 bits + * csc: 4~6 bits + * flag: 7-10 bits + * dif_check: 11~13 bits + */ + __le16 done_flag; + + __u8 error_type; + __u8 warning_type; + __u8 mac_i3; + __u8 mac_i2; + __u8 mac_i1; + __u8 mac_i0; + __le16 check_sum_i; + __u8 tls_pad_len_i; + __u8 rsvd12; + __le32 counter; +}; + +struct sec_sqe { + /* + * type: 0~3 bits + * cipher: 4~5 bits + * auth: 6~7 bit s + */ + __u8 type_cipher_auth; + + /* + * seq: 0 bit + * de: 1~2 bits + * scene: 3~6 bits + * src_addr_type: ~7 bit, with sdm_addr_type 0-1 bits + */ + __u8 sds_sa_type; + + /* + * src_addr_type: 0~1 bits, not used now, + * if support PRP, set this field, or set zero. + * dst_addr_type: 2~4 bits + * mac_addr_type: 5~7 bits + */ + __u8 sdm_addr_type; + __u8 rsvd0; + + /* + * nonce_len(type2): 0~3 bits + * huk(type2): 4 bit + * key_s(type2): 5 bit + * ci_gen: 6~7 bits + */ + __u8 huk_key_ci; + + /* + * ai_gen: 0~1 bits + * a_pad(type2): 2~3 bits + * c_s(type2): 4~5 bits + */ + __u8 ai_apd_cs; + + /* + * rhf(type2): 0 bit + * c_key_type: 1~2 bits + * a_key_type: 3~4 bits + * write_frame_len(type2): 5~7 bits + */ + __u8 rca_key_frm; + + /* + * cal_iv_addr_en(type2): 0 bit + * tls_up(type2): 1 bit + * inveld: 7 bit + */ + __u8 iv_tls_ld; + + /* Just using type2 BD now */ + struct sec_sqe_type2 type2; +}; + +int sec_register_to_crypto(void); +void sec_unregister_from_crypto(void); +#endif diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c new file mode 100644 index 000000000..548896394 --- /dev/null +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -0,0 +1,998 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2019 HiSilicon Limited. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sec.h" + +#define SEC_VF_NUM 63 +#define SEC_QUEUE_NUM_V1 4096 +#define SEC_QUEUE_NUM_V2 1024 +#define SEC_PF_PCI_DEVICE_ID 0xa255 +#define SEC_VF_PCI_DEVICE_ID 0xa256 + +#define SEC_BD_ERR_CHK_EN0 0xEFFFFFFF +#define SEC_BD_ERR_CHK_EN1 0x7ffff7fd +#define SEC_BD_ERR_CHK_EN3 0xffffbfff + +#define SEC_SQE_SIZE 128 +#define SEC_SQ_SIZE (SEC_SQE_SIZE * QM_Q_DEPTH) +#define SEC_PF_DEF_Q_NUM 256 +#define SEC_PF_DEF_Q_BASE 0 +#define SEC_CTX_Q_NUM_DEF 2 +#define SEC_CTX_Q_NUM_MAX 32 + +#define SEC_CTRL_CNT_CLR_CE 0x301120 +#define SEC_CTRL_CNT_CLR_CE_BIT BIT(0) +#define SEC_ENGINE_PF_CFG_OFF 0x300000 +#define SEC_ACC_COMMON_REG_OFF 0x1000 +#define SEC_CORE_INT_SOURCE 0x301010 +#define SEC_CORE_INT_MASK 0x301000 +#define SEC_CORE_INT_STATUS 0x301008 +#define SEC_CORE_SRAM_ECC_ERR_INFO 0x301C14 +#define SEC_ECC_NUM(err) (((err) >> 16) & 0xFF) +#define SEC_ECC_ADDR(err) ((err) >> 0) +#define SEC_CORE_INT_DISABLE 0x0 +#define SEC_CORE_INT_ENABLE 0x1ff +#define SEC_CORE_INT_CLEAR 0x1ff +#define SEC_SAA_ENABLE 0x17f + +#define SEC_RAS_CE_REG 0x301050 +#define SEC_RAS_FE_REG 0x301054 +#define SEC_RAS_NFE_REG 0x301058 +#define SEC_RAS_CE_ENB_MSK 0x88 +#define SEC_RAS_FE_ENB_MSK 0x0 +#define SEC_RAS_NFE_ENB_MSK 0x177 +#define SEC_RAS_DISABLE 0x0 +#define SEC_MEM_START_INIT_REG 0x0100 +#define SEC_MEM_INIT_DONE_REG 0x0104 + +#define SEC_CONTROL_REG 0x0200 +#define SEC_TRNG_EN_SHIFT 8 +#define SEC_CLK_GATE_ENABLE BIT(3) +#define SEC_CLK_GATE_DISABLE (~BIT(3)) +#define SEC_AXI_SHUTDOWN_ENABLE BIT(12) +#define SEC_AXI_SHUTDOWN_DISABLE 0xFFFFEFFF + +#define SEC_INTERFACE_USER_CTRL0_REG 0x0220 +#define SEC_INTERFACE_USER_CTRL1_REG 0x0224 +#define SEC_SAA_EN_REG 0x0270 +#define SEC_BD_ERR_CHK_EN_REG0 0x0380 +#define SEC_BD_ERR_CHK_EN_REG1 0x0384 +#define SEC_BD_ERR_CHK_EN_REG3 0x038c + +#define SEC_USER0_SMMU_NORMAL (BIT(23) | BIT(15)) +#define SEC_USER1_SMMU_NORMAL (BIT(31) | BIT(23) | BIT(15) | BIT(7)) +#define SEC_CORE_INT_STATUS_M_ECC BIT(2) + +#define SEC_DELAY_10_US 10 +#define SEC_POLL_TIMEOUT_US 1000 +#define SEC_DBGFS_VAL_MAX_LEN 20 +#define SEC_SINGLE_PORT_MAX_TRANS 0x2060 + +#define SEC_SQE_MASK_OFFSET 64 +#define SEC_SQE_MASK_LEN 48 + +#define SEC_ADDR(qm, offset) ((qm)->io_base + (offset) + \ + SEC_ENGINE_PF_CFG_OFF + SEC_ACC_COMMON_REG_OFF) + +struct sec_hw_error { + u32 int_msk; + const char *msg; +}; + +struct sec_dfx_item { + const char *name; + u32 offset; +}; + +static const char sec_name[] = "hisi_sec2"; +static struct dentry *sec_debugfs_root; + +static struct hisi_qm_list sec_devices = { + .register_to_crypto = sec_register_to_crypto, + .unregister_from_crypto = sec_unregister_from_crypto, +}; + +static const struct sec_hw_error sec_hw_errors[] = { + {.int_msk = BIT(0), .msg = "sec_axi_rresp_err_rint"}, + {.int_msk = BIT(1), .msg = "sec_axi_bresp_err_rint"}, + {.int_msk = BIT(2), .msg = "sec_ecc_2bit_err_rint"}, + {.int_msk = BIT(3), .msg = "sec_ecc_1bit_err_rint"}, + {.int_msk = BIT(4), .msg = "sec_req_trng_timeout_rint"}, + {.int_msk = BIT(5), .msg = "sec_fsm_hbeat_rint"}, + {.int_msk = BIT(6), .msg = "sec_channel_req_rng_timeout_rint"}, + {.int_msk = BIT(7), .msg = "sec_bd_err_rint"}, + {.int_msk = BIT(8), .msg = "sec_chain_buff_err_rint"}, + { /* sentinel */ } +}; + +static const char * const sec_dbg_file_name[] = { + [SEC_CURRENT_QM] = "current_qm", + [SEC_CLEAR_ENABLE] = "clear_enable", +}; + +static struct sec_dfx_item sec_dfx_labels[] = { + {"send_cnt", offsetof(struct sec_dfx, send_cnt)}, + {"recv_cnt", offsetof(struct sec_dfx, recv_cnt)}, + {"send_busy_cnt", offsetof(struct sec_dfx, send_busy_cnt)}, + {"recv_busy_cnt", offsetof(struct sec_dfx, recv_busy_cnt)}, + {"err_bd_cnt", offsetof(struct sec_dfx, err_bd_cnt)}, + {"invalid_req_cnt", offsetof(struct sec_dfx, invalid_req_cnt)}, + {"done_flag_cnt", offsetof(struct sec_dfx, done_flag_cnt)}, +}; + +static const struct debugfs_reg32 sec_dfx_regs[] = { + {"SEC_PF_ABNORMAL_INT_SOURCE ", 0x301010}, + {"SEC_SAA_EN ", 0x301270}, + {"SEC_BD_LATENCY_MIN ", 0x301600}, + {"SEC_BD_LATENCY_MAX ", 0x301608}, + {"SEC_BD_LATENCY_AVG ", 0x30160C}, + {"SEC_BD_NUM_IN_SAA0 ", 0x301670}, + {"SEC_BD_NUM_IN_SAA1 ", 0x301674}, + {"SEC_BD_NUM_IN_SEC ", 0x301680}, + {"SEC_ECC_1BIT_CNT ", 0x301C00}, + {"SEC_ECC_1BIT_INFO ", 0x301C04}, + {"SEC_ECC_2BIT_CNT ", 0x301C10}, + {"SEC_ECC_2BIT_INFO ", 0x301C14}, + {"SEC_BD_SAA0 ", 0x301C20}, + {"SEC_BD_SAA1 ", 0x301C24}, + {"SEC_BD_SAA2 ", 0x301C28}, + {"SEC_BD_SAA3 ", 0x301C2C}, + {"SEC_BD_SAA4 ", 0x301C30}, + {"SEC_BD_SAA5 ", 0x301C34}, + {"SEC_BD_SAA6 ", 0x301C38}, + {"SEC_BD_SAA7 ", 0x301C3C}, + {"SEC_BD_SAA8 ", 0x301C40}, +}; + +static int sec_pf_q_num_set(const char *val, const struct kernel_param *kp) +{ + return q_num_set(val, kp, SEC_PF_PCI_DEVICE_ID); +} + +static const struct kernel_param_ops sec_pf_q_num_ops = { + .set = sec_pf_q_num_set, + .get = param_get_int, +}; + +static u32 pf_q_num = SEC_PF_DEF_Q_NUM; +module_param_cb(pf_q_num, &sec_pf_q_num_ops, &pf_q_num, 0444); +MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 2-4096, v2 2-1024)"); + +static int sec_ctx_q_num_set(const char *val, const struct kernel_param *kp) +{ + u32 ctx_q_num; + int ret; + + if (!val) + return -EINVAL; + + ret = kstrtou32(val, 10, &ctx_q_num); + if (ret) + return -EINVAL; + + if (!ctx_q_num || ctx_q_num > SEC_CTX_Q_NUM_MAX || ctx_q_num & 0x1) { + pr_err("ctx queue num[%u] is invalid!\n", ctx_q_num); + return -EINVAL; + } + + return param_set_int(val, kp); +} + +static const struct kernel_param_ops sec_ctx_q_num_ops = { + .set = sec_ctx_q_num_set, + .get = param_get_int, +}; +static u32 ctx_q_num = SEC_CTX_Q_NUM_DEF; +module_param_cb(ctx_q_num, &sec_ctx_q_num_ops, &ctx_q_num, 0444); +MODULE_PARM_DESC(ctx_q_num, "Queue num in ctx (2 default, 2, 4, ..., 32)"); + +static const struct kernel_param_ops vfs_num_ops = { + .set = vfs_num_set, + .get = param_get_int, +}; + +static u32 vfs_num; +module_param_cb(vfs_num, &vfs_num_ops, &vfs_num, 0444); +MODULE_PARM_DESC(vfs_num, "Number of VFs to enable(1-63), 0(default)"); + +void sec_destroy_qps(struct hisi_qp **qps, int qp_num) +{ + hisi_qm_free_qps(qps, qp_num); + kfree(qps); +} + +struct hisi_qp **sec_create_qps(void) +{ + int node = cpu_to_node(smp_processor_id()); + u32 ctx_num = ctx_q_num; + struct hisi_qp **qps; + int ret; + + qps = kcalloc(ctx_num, sizeof(struct hisi_qp *), GFP_KERNEL); + if (!qps) + return NULL; + + ret = hisi_qm_alloc_qps_node(&sec_devices, ctx_num, 0, node, qps); + if (!ret) + return qps; + + kfree(qps); + return NULL; +} + + +static const struct pci_device_id sec_dev_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, SEC_PF_PCI_DEVICE_ID) }, + { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, SEC_VF_PCI_DEVICE_ID) }, + { 0, } +}; +MODULE_DEVICE_TABLE(pci, sec_dev_ids); + +static u8 sec_get_endian(struct hisi_qm *qm) +{ + u32 reg; + + /* + * As for VF, it is a wrong way to get endian setting by + * reading a register of the engine + */ + if (qm->pdev->is_virtfn) { + dev_err_ratelimited(&qm->pdev->dev, + "cannot access a register in VF!\n"); + return SEC_LE; + } + reg = readl_relaxed(qm->io_base + SEC_ENGINE_PF_CFG_OFF + + SEC_ACC_COMMON_REG_OFF + SEC_CONTROL_REG); + + /* BD little endian mode */ + if (!(reg & BIT(0))) + return SEC_LE; + + /* BD 32-bits big endian mode */ + else if (!(reg & BIT(1))) + return SEC_32BE; + + /* BD 64-bits big endian mode */ + else + return SEC_64BE; +} + +static int sec_engine_init(struct hisi_qm *qm) +{ + int ret; + u32 reg; + + /* disable clock gate control */ + reg = readl_relaxed(SEC_ADDR(qm, SEC_CONTROL_REG)); + reg &= SEC_CLK_GATE_DISABLE; + writel_relaxed(reg, SEC_ADDR(qm, SEC_CONTROL_REG)); + + writel_relaxed(0x1, SEC_ADDR(qm, SEC_MEM_START_INIT_REG)); + + ret = readl_relaxed_poll_timeout(SEC_ADDR(qm, SEC_MEM_INIT_DONE_REG), + reg, reg & 0x1, SEC_DELAY_10_US, + SEC_POLL_TIMEOUT_US); + if (ret) { + pci_err(qm->pdev, "fail to init sec mem\n"); + return ret; + } + + reg = readl_relaxed(SEC_ADDR(qm, SEC_CONTROL_REG)); + reg |= (0x1 << SEC_TRNG_EN_SHIFT); + writel_relaxed(reg, SEC_ADDR(qm, SEC_CONTROL_REG)); + + reg = readl_relaxed(SEC_ADDR(qm, SEC_INTERFACE_USER_CTRL0_REG)); + reg |= SEC_USER0_SMMU_NORMAL; + writel_relaxed(reg, SEC_ADDR(qm, SEC_INTERFACE_USER_CTRL0_REG)); + + reg = readl_relaxed(SEC_ADDR(qm, SEC_INTERFACE_USER_CTRL1_REG)); + reg |= SEC_USER1_SMMU_NORMAL; + writel_relaxed(reg, SEC_ADDR(qm, SEC_INTERFACE_USER_CTRL1_REG)); + + writel(SEC_SINGLE_PORT_MAX_TRANS, + qm->io_base + AM_CFG_SINGLE_PORT_MAX_TRANS); + + writel(SEC_SAA_ENABLE, SEC_ADDR(qm, SEC_SAA_EN_REG)); + + /* Enable sm4 extra mode, as ctr/ecb */ + writel_relaxed(SEC_BD_ERR_CHK_EN0, + SEC_ADDR(qm, SEC_BD_ERR_CHK_EN_REG0)); + /* Enable sm4 xts mode multiple iv */ + writel_relaxed(SEC_BD_ERR_CHK_EN1, + SEC_ADDR(qm, SEC_BD_ERR_CHK_EN_REG1)); + writel_relaxed(SEC_BD_ERR_CHK_EN3, + SEC_ADDR(qm, SEC_BD_ERR_CHK_EN_REG3)); + + /* config endian */ + reg = readl_relaxed(SEC_ADDR(qm, SEC_CONTROL_REG)); + reg |= sec_get_endian(qm); + writel_relaxed(reg, SEC_ADDR(qm, SEC_CONTROL_REG)); + + return 0; +} + +static int sec_set_user_domain_and_cache(struct hisi_qm *qm) +{ + /* qm user domain */ + writel(AXUSER_BASE, qm->io_base + QM_ARUSER_M_CFG_1); + writel(ARUSER_M_CFG_ENABLE, qm->io_base + QM_ARUSER_M_CFG_ENABLE); + writel(AXUSER_BASE, qm->io_base + QM_AWUSER_M_CFG_1); + writel(AWUSER_M_CFG_ENABLE, qm->io_base + QM_AWUSER_M_CFG_ENABLE); + writel(WUSER_M_CFG_ENABLE, qm->io_base + QM_WUSER_M_CFG_ENABLE); + + /* qm cache */ + writel(AXI_M_CFG, qm->io_base + QM_AXI_M_CFG); + writel(AXI_M_CFG_ENABLE, qm->io_base + QM_AXI_M_CFG_ENABLE); + + /* disable FLR triggered by BME(bus master enable) */ + writel(PEH_AXUSER_CFG, qm->io_base + QM_PEH_AXUSER_CFG); + writel(PEH_AXUSER_CFG_ENABLE, qm->io_base + QM_PEH_AXUSER_CFG_ENABLE); + + /* enable sqc,cqc writeback */ + writel(SQC_CACHE_ENABLE | CQC_CACHE_ENABLE | SQC_CACHE_WB_ENABLE | + CQC_CACHE_WB_ENABLE | FIELD_PREP(SQC_CACHE_WB_THRD, 1) | + FIELD_PREP(CQC_CACHE_WB_THRD, 1), qm->io_base + QM_CACHE_CTL); + + return sec_engine_init(qm); +} + +/* sec_debug_regs_clear() - clear the sec debug regs */ +static void sec_debug_regs_clear(struct hisi_qm *qm) +{ + int i; + + /* clear current_qm */ + writel(0x0, qm->io_base + QM_DFX_MB_CNT_VF); + writel(0x0, qm->io_base + QM_DFX_DB_CNT_VF); + + /* clear sec dfx regs */ + writel(0x1, qm->io_base + SEC_CTRL_CNT_CLR_CE); + for (i = 0; i < ARRAY_SIZE(sec_dfx_regs); i++) + readl(qm->io_base + sec_dfx_regs[i].offset); + + /* clear rdclr_en */ + writel(0x0, qm->io_base + SEC_CTRL_CNT_CLR_CE); + + hisi_qm_debug_regs_clear(qm); +} + +static void sec_hw_error_enable(struct hisi_qm *qm) +{ + u32 val; + + if (qm->ver == QM_HW_V1) { + writel(SEC_CORE_INT_DISABLE, qm->io_base + SEC_CORE_INT_MASK); + pci_info(qm->pdev, "V1 not support hw error handle\n"); + return; + } + + val = readl(SEC_ADDR(qm, SEC_CONTROL_REG)); + + /* clear SEC hw error source if having */ + writel(SEC_CORE_INT_CLEAR, qm->io_base + SEC_CORE_INT_SOURCE); + + /* enable SEC hw error interrupts */ + writel(SEC_CORE_INT_ENABLE, qm->io_base + SEC_CORE_INT_MASK); + + /* enable RAS int */ + writel(SEC_RAS_CE_ENB_MSK, qm->io_base + SEC_RAS_CE_REG); + writel(SEC_RAS_FE_ENB_MSK, qm->io_base + SEC_RAS_FE_REG); + writel(SEC_RAS_NFE_ENB_MSK, qm->io_base + SEC_RAS_NFE_REG); + + /* enable SEC block master OOO when m-bit error occur */ + val = val | SEC_AXI_SHUTDOWN_ENABLE; + + writel(val, SEC_ADDR(qm, SEC_CONTROL_REG)); +} + +static void sec_hw_error_disable(struct hisi_qm *qm) +{ + u32 val; + + val = readl(SEC_ADDR(qm, SEC_CONTROL_REG)); + + /* disable RAS int */ + writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_CE_REG); + writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_FE_REG); + writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_NFE_REG); + + /* disable SEC hw error interrupts */ + writel(SEC_CORE_INT_DISABLE, qm->io_base + SEC_CORE_INT_MASK); + + /* disable SEC block master OOO when m-bit error occur */ + val = val & SEC_AXI_SHUTDOWN_DISABLE; + + writel(val, SEC_ADDR(qm, SEC_CONTROL_REG)); +} + +static u32 sec_current_qm_read(struct sec_debug_file *file) +{ + struct hisi_qm *qm = file->qm; + + return readl(qm->io_base + QM_DFX_MB_CNT_VF); +} + +static int sec_current_qm_write(struct sec_debug_file *file, u32 val) +{ + struct hisi_qm *qm = file->qm; + u32 vfq_num; + u32 tmp; + + if (val > qm->vfs_num) + return -EINVAL; + + /* According PF or VF Dev ID to calculation curr_qm_qp_num and store */ + if (!val) { + qm->debug.curr_qm_qp_num = qm->qp_num; + } else { + vfq_num = (qm->ctrl_qp_num - qm->qp_num) / qm->vfs_num; + + if (val == qm->vfs_num) + qm->debug.curr_qm_qp_num = + qm->ctrl_qp_num - qm->qp_num - + (qm->vfs_num - 1) * vfq_num; + else + qm->debug.curr_qm_qp_num = vfq_num; + } + + writel(val, qm->io_base + QM_DFX_MB_CNT_VF); + writel(val, qm->io_base + QM_DFX_DB_CNT_VF); + + tmp = val | + (readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) & CURRENT_Q_MASK); + writel(tmp, qm->io_base + QM_DFX_SQE_CNT_VF_SQN); + + tmp = val | + (readl(qm->io_base + QM_DFX_CQE_CNT_VF_CQN) & CURRENT_Q_MASK); + writel(tmp, qm->io_base + QM_DFX_CQE_CNT_VF_CQN); + + return 0; +} + +static u32 sec_clear_enable_read(struct sec_debug_file *file) +{ + struct hisi_qm *qm = file->qm; + + return readl(qm->io_base + SEC_CTRL_CNT_CLR_CE) & + SEC_CTRL_CNT_CLR_CE_BIT; +} + +static int sec_clear_enable_write(struct sec_debug_file *file, u32 val) +{ + struct hisi_qm *qm = file->qm; + u32 tmp; + + if (val != 1 && val) + return -EINVAL; + + tmp = (readl(qm->io_base + SEC_CTRL_CNT_CLR_CE) & + ~SEC_CTRL_CNT_CLR_CE_BIT) | val; + writel(tmp, qm->io_base + SEC_CTRL_CNT_CLR_CE); + + return 0; +} + +static ssize_t sec_debug_read(struct file *filp, char __user *buf, + size_t count, loff_t *pos) +{ + struct sec_debug_file *file = filp->private_data; + char tbuf[SEC_DBGFS_VAL_MAX_LEN]; + u32 val; + int ret; + + spin_lock_irq(&file->lock); + + switch (file->index) { + case SEC_CURRENT_QM: + val = sec_current_qm_read(file); + break; + case SEC_CLEAR_ENABLE: + val = sec_clear_enable_read(file); + break; + default: + spin_unlock_irq(&file->lock); + return -EINVAL; + } + + spin_unlock_irq(&file->lock); + ret = snprintf(tbuf, SEC_DBGFS_VAL_MAX_LEN, "%u\n", val); + + return simple_read_from_buffer(buf, count, pos, tbuf, ret); +} + +static ssize_t sec_debug_write(struct file *filp, const char __user *buf, + size_t count, loff_t *pos) +{ + struct sec_debug_file *file = filp->private_data; + char tbuf[SEC_DBGFS_VAL_MAX_LEN]; + unsigned long val; + int len, ret; + + if (*pos != 0) + return 0; + + if (count >= SEC_DBGFS_VAL_MAX_LEN) + return -ENOSPC; + + len = simple_write_to_buffer(tbuf, SEC_DBGFS_VAL_MAX_LEN - 1, + pos, buf, count); + if (len < 0) + return len; + + tbuf[len] = '\0'; + if (kstrtoul(tbuf, 0, &val)) + return -EFAULT; + + spin_lock_irq(&file->lock); + + switch (file->index) { + case SEC_CURRENT_QM: + ret = sec_current_qm_write(file, val); + if (ret) + goto err_input; + break; + case SEC_CLEAR_ENABLE: + ret = sec_clear_enable_write(file, val); + if (ret) + goto err_input; + break; + default: + ret = -EINVAL; + goto err_input; + } + + spin_unlock_irq(&file->lock); + + return count; + + err_input: + spin_unlock_irq(&file->lock); + return ret; +} + +static const struct file_operations sec_dbg_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = sec_debug_read, + .write = sec_debug_write, +}; + +static int sec_debugfs_atomic64_get(void *data, u64 *val) +{ + *val = atomic64_read((atomic64_t *)data); + + return 0; +} + +static int sec_debugfs_atomic64_set(void *data, u64 val) +{ + if (val) + return -EINVAL; + + atomic64_set((atomic64_t *)data, 0); + + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(sec_atomic64_ops, sec_debugfs_atomic64_get, + sec_debugfs_atomic64_set, "%lld\n"); + +static int sec_core_debug_init(struct hisi_qm *qm) +{ + struct sec_dev *sec = container_of(qm, struct sec_dev, qm); + struct device *dev = &qm->pdev->dev; + struct sec_dfx *dfx = &sec->debug.dfx; + struct debugfs_regset32 *regset; + struct dentry *tmp_d; + int i; + + tmp_d = debugfs_create_dir("sec_dfx", qm->debug.debug_root); + + regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL); + if (!regset) + return -ENOMEM; + + regset->regs = sec_dfx_regs; + regset->nregs = ARRAY_SIZE(sec_dfx_regs); + regset->base = qm->io_base; + + if (qm->pdev->device == SEC_PF_PCI_DEVICE_ID) + debugfs_create_regset32("regs", 0444, tmp_d, regset); + + for (i = 0; i < ARRAY_SIZE(sec_dfx_labels); i++) { + atomic64_t *data = (atomic64_t *)((uintptr_t)dfx + + sec_dfx_labels[i].offset); + debugfs_create_file(sec_dfx_labels[i].name, 0644, + tmp_d, data, &sec_atomic64_ops); + } + + return 0; +} + +static int sec_debug_init(struct hisi_qm *qm) +{ + struct sec_dev *sec = container_of(qm, struct sec_dev, qm); + int i; + + if (qm->pdev->device == SEC_PF_PCI_DEVICE_ID) { + for (i = SEC_CURRENT_QM; i < SEC_DEBUG_FILE_NUM; i++) { + spin_lock_init(&sec->debug.files[i].lock); + sec->debug.files[i].index = i; + sec->debug.files[i].qm = qm; + + debugfs_create_file(sec_dbg_file_name[i], 0600, + qm->debug.debug_root, + sec->debug.files + i, + &sec_dbg_fops); + } + } + + return sec_core_debug_init(qm); +} + +static int sec_debugfs_init(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + int ret; + + qm->debug.debug_root = debugfs_create_dir(dev_name(dev), + sec_debugfs_root); + qm->debug.sqe_mask_offset = SEC_SQE_MASK_OFFSET; + qm->debug.sqe_mask_len = SEC_SQE_MASK_LEN; + ret = hisi_qm_debug_init(qm); + if (ret) + goto failed_to_create; + + ret = sec_debug_init(qm); + if (ret) + goto failed_to_create; + + + return 0; + +failed_to_create: + debugfs_remove_recursive(sec_debugfs_root); + + return ret; +} + +static void sec_debugfs_exit(struct hisi_qm *qm) +{ + debugfs_remove_recursive(qm->debug.debug_root); +} + +static void sec_log_hw_error(struct hisi_qm *qm, u32 err_sts) +{ + const struct sec_hw_error *errs = sec_hw_errors; + struct device *dev = &qm->pdev->dev; + u32 err_val; + + while (errs->msg) { + if (errs->int_msk & err_sts) { + dev_err(dev, "%s [error status=0x%x] found\n", + errs->msg, errs->int_msk); + + if (SEC_CORE_INT_STATUS_M_ECC & errs->int_msk) { + err_val = readl(qm->io_base + + SEC_CORE_SRAM_ECC_ERR_INFO); + dev_err(dev, "multi ecc sram num=0x%x\n", + SEC_ECC_NUM(err_val)); + } + } + errs++; + } +} + +static u32 sec_get_hw_err_status(struct hisi_qm *qm) +{ + return readl(qm->io_base + SEC_CORE_INT_STATUS); +} + +static void sec_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts) +{ + writel(err_sts, qm->io_base + SEC_CORE_INT_SOURCE); +} + +static void sec_open_axi_master_ooo(struct hisi_qm *qm) +{ + u32 val; + + val = readl(SEC_ADDR(qm, SEC_CONTROL_REG)); + writel(val & SEC_AXI_SHUTDOWN_DISABLE, SEC_ADDR(qm, SEC_CONTROL_REG)); + writel(val | SEC_AXI_SHUTDOWN_ENABLE, SEC_ADDR(qm, SEC_CONTROL_REG)); +} + +static const struct hisi_qm_err_ini sec_err_ini = { + .hw_init = sec_set_user_domain_and_cache, + .hw_err_enable = sec_hw_error_enable, + .hw_err_disable = sec_hw_error_disable, + .get_dev_hw_err_status = sec_get_hw_err_status, + .clear_dev_hw_err_status = sec_clear_hw_err_status, + .log_dev_hw_err = sec_log_hw_error, + .open_axi_master_ooo = sec_open_axi_master_ooo, + .err_info = { + .ce = QM_BASE_CE, + .nfe = QM_BASE_NFE | QM_ACC_DO_TASK_TIMEOUT | + QM_ACC_WB_NOT_READY_TIMEOUT, + .fe = 0, + .ecc_2bits_mask = SEC_CORE_INT_STATUS_M_ECC, + .msi_wr_port = BIT(0), + .acpi_rst = "SRST", + } +}; + +static int sec_pf_probe_init(struct sec_dev *sec) +{ + struct hisi_qm *qm = &sec->qm; + int ret; + + if (qm->ver == QM_HW_V1) + qm->ctrl_qp_num = SEC_QUEUE_NUM_V1; + else + qm->ctrl_qp_num = SEC_QUEUE_NUM_V2; + + qm->err_ini = &sec_err_ini; + + ret = sec_set_user_domain_and_cache(qm); + if (ret) + return ret; + + hisi_qm_dev_err_init(qm); + sec_debug_regs_clear(qm); + + return 0; +} + +static int sec_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) +{ + int ret; + + qm->pdev = pdev; + qm->ver = pdev->revision; + qm->sqe_size = SEC_SQE_SIZE; + qm->dev_name = sec_name; + + qm->fun_type = (pdev->device == SEC_PF_PCI_DEVICE_ID) ? + QM_HW_PF : QM_HW_VF; + if (qm->fun_type == QM_HW_PF) { + qm->qp_base = SEC_PF_DEF_Q_BASE; + qm->qp_num = pf_q_num; + qm->debug.curr_qm_qp_num = pf_q_num; + qm->qm_list = &sec_devices; + } else if (qm->fun_type == QM_HW_VF && qm->ver == QM_HW_V1) { + /* + * have no way to get qm configure in VM in v1 hardware, + * so currently force PF to uses SEC_PF_DEF_Q_NUM, and force + * to trigger only one VF in v1 hardware. + * v2 hardware has no such problem. + */ + qm->qp_base = SEC_PF_DEF_Q_NUM; + qm->qp_num = SEC_QUEUE_NUM_V1 - SEC_PF_DEF_Q_NUM; + } + + /* + * WQ_HIGHPRI: SEC request must be low delayed, + * so need a high priority workqueue. + * WQ_UNBOUND: SEC task is likely with long + * running CPU intensive workloads. + */ + qm->wq = alloc_workqueue("%s", WQ_HIGHPRI | WQ_MEM_RECLAIM | + WQ_UNBOUND, num_online_cpus(), + pci_name(qm->pdev)); + if (!qm->wq) { + pci_err(qm->pdev, "fail to alloc workqueue\n"); + return -ENOMEM; + } + + ret = hisi_qm_init(qm); + if (ret) + destroy_workqueue(qm->wq); + + return ret; +} + +static void sec_qm_uninit(struct hisi_qm *qm) +{ + hisi_qm_uninit(qm); +} + +static int sec_probe_init(struct sec_dev *sec) +{ + struct hisi_qm *qm = &sec->qm; + int ret; + + if (qm->fun_type == QM_HW_PF) { + ret = sec_pf_probe_init(sec); + if (ret) + return ret; + } + + return 0; +} + +static void sec_probe_uninit(struct hisi_qm *qm) +{ + hisi_qm_dev_err_uninit(qm); + + destroy_workqueue(qm->wq); +} + +static void sec_iommu_used_check(struct sec_dev *sec) +{ + struct iommu_domain *domain; + struct device *dev = &sec->qm.pdev->dev; + + domain = iommu_get_domain_for_dev(dev); + + /* Check if iommu is used */ + sec->iommu_used = false; + if (domain) { + if (domain->type & __IOMMU_DOMAIN_PAGING) + sec->iommu_used = true; + dev_info(dev, "SMMU Opened, the iommu type = %u\n", + domain->type); + } +} + +static int sec_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct sec_dev *sec; + struct hisi_qm *qm; + int ret; + + sec = devm_kzalloc(&pdev->dev, sizeof(*sec), GFP_KERNEL); + if (!sec) + return -ENOMEM; + + qm = &sec->qm; + ret = sec_qm_init(qm, pdev); + if (ret) { + pci_err(pdev, "Failed to init SEC QM (%d)!\n", ret); + return ret; + } + + sec->ctx_q_num = ctx_q_num; + sec_iommu_used_check(sec); + + ret = sec_probe_init(sec); + if (ret) { + pci_err(pdev, "Failed to probe!\n"); + goto err_qm_uninit; + } + + ret = hisi_qm_start(qm); + if (ret) { + pci_err(pdev, "Failed to start sec qm!\n"); + goto err_probe_uninit; + } + + ret = sec_debugfs_init(qm); + if (ret) + pci_warn(pdev, "Failed to init debugfs!\n"); + + ret = hisi_qm_alg_register(qm, &sec_devices); + if (ret < 0) { + pr_err("Failed to register driver to crypto.\n"); + goto err_qm_stop; + } + + if (qm->fun_type == QM_HW_PF && vfs_num) { + ret = hisi_qm_sriov_enable(pdev, vfs_num); + if (ret < 0) + goto err_alg_unregister; + } + + return 0; + +err_alg_unregister: + hisi_qm_alg_unregister(qm, &sec_devices); + +err_qm_stop: + sec_debugfs_exit(qm); + hisi_qm_stop(qm, QM_NORMAL); + +err_probe_uninit: + sec_probe_uninit(qm); + +err_qm_uninit: + sec_qm_uninit(qm); + + return ret; +} + +static void sec_remove(struct pci_dev *pdev) +{ + struct hisi_qm *qm = pci_get_drvdata(pdev); + + hisi_qm_wait_task_finish(qm, &sec_devices); + hisi_qm_alg_unregister(qm, &sec_devices); + if (qm->fun_type == QM_HW_PF && qm->vfs_num) + hisi_qm_sriov_disable(pdev, qm->is_frozen); + + sec_debugfs_exit(qm); + + (void)hisi_qm_stop(qm, QM_NORMAL); + + if (qm->fun_type == QM_HW_PF) + sec_debug_regs_clear(qm); + + sec_probe_uninit(qm); + + sec_qm_uninit(qm); +} + +static const struct pci_error_handlers sec_err_handler = { + .error_detected = hisi_qm_dev_err_detected, + .slot_reset = hisi_qm_dev_slot_reset, + .reset_prepare = hisi_qm_reset_prepare, + .reset_done = hisi_qm_reset_done, +}; + +static struct pci_driver sec_pci_driver = { + .name = "hisi_sec2", + .id_table = sec_dev_ids, + .probe = sec_probe, + .remove = sec_remove, + .err_handler = &sec_err_handler, + .sriov_configure = hisi_qm_sriov_configure, + .shutdown = hisi_qm_dev_shutdown, +}; + +static void sec_register_debugfs(void) +{ + if (!debugfs_initialized()) + return; + + sec_debugfs_root = debugfs_create_dir("hisi_sec2", NULL); +} + +static void sec_unregister_debugfs(void) +{ + debugfs_remove_recursive(sec_debugfs_root); +} + +static int __init sec_init(void) +{ + int ret; + + hisi_qm_init_list(&sec_devices); + sec_register_debugfs(); + + ret = pci_register_driver(&sec_pci_driver); + if (ret < 0) { + sec_unregister_debugfs(); + pr_err("Failed to register pci driver.\n"); + return ret; + } + + return 0; +} + +static void __exit sec_exit(void) +{ + pci_unregister_driver(&sec_pci_driver); + sec_unregister_debugfs(); +} + +module_init(sec_init); +module_exit(sec_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Zaibo Xu "); +MODULE_AUTHOR("Longfang Liu "); +MODULE_AUTHOR("Kai Ye "); +MODULE_AUTHOR("Wei Zhang "); +MODULE_DESCRIPTION("Driver for HiSilicon SEC accelerator"); -- cgit v1.2.3