diff options
Diffstat (limited to 'drivers/crypto/cavium/nitrox/nitrox_reqmgr.c')
| -rw-r--r-- | drivers/crypto/cavium/nitrox/nitrox_reqmgr.c | 607 |
1 files changed, 607 insertions, 0 deletions
diff --git a/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c b/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c new file mode 100644 index 000000000..55c18da4a --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c @@ -0,0 +1,607 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/gfp.h> +#include <linux/workqueue.h> +#include <crypto/internal/skcipher.h> + +#include "nitrox_common.h" +#include "nitrox_dev.h" +#include "nitrox_req.h" +#include "nitrox_csr.h" + +/* SLC_STORE_INFO */ +#define MIN_UDD_LEN 16 +/* PKT_IN_HDR + SLC_STORE_INFO */ +#define FDATA_SIZE 32 +/* Base destination port for the solicited requests */ +#define SOLICIT_BASE_DPORT 256 + +#define REQ_NOT_POSTED 1 +#define REQ_BACKLOG 2 +#define REQ_POSTED 3 + +/* + * Response codes from SE microcode + * 0x00 - Success + * Completion with no error + * 0x43 - ERR_GC_DATA_LEN_INVALID + * Invalid Data length if Encryption Data length is + * less than 16 bytes for AES-XTS and AES-CTS. + * 0x45 - ERR_GC_CTX_LEN_INVALID + * Invalid context length: CTXL != 23 words. + * 0x4F - ERR_GC_DOCSIS_CIPHER_INVALID + * DOCSIS support is enabled with other than + * AES/DES-CBC mode encryption. + * 0x50 - ERR_GC_DOCSIS_OFFSET_INVALID + * Authentication offset is other than 0 with + * Encryption IV source = 0. + * Authentication offset is other than 8 (DES)/16 (AES) + * with Encryption IV source = 1 + * 0x51 - ERR_GC_CRC32_INVALID_SELECTION + * CRC32 is enabled for other than DOCSIS encryption. + * 0x52 - ERR_GC_AES_CCM_FLAG_INVALID + * Invalid flag options in AES-CCM IV. + */ + +static inline int incr_index(int index, int count, int max) +{ + if ((index + count) >= max) + index = index + count - max; + else + index += count; + + return index; +} + +static void softreq_unmap_sgbufs(struct nitrox_softreq *sr) +{ + struct nitrox_device *ndev = sr->ndev; + struct device *dev = DEV(ndev); + + + dma_unmap_sg(dev, sr->in.sg, sg_nents(sr->in.sg), + DMA_BIDIRECTIONAL); + dma_unmap_single(dev, sr->in.sgcomp_dma, sr->in.sgcomp_len, + DMA_TO_DEVICE); + kfree(sr->in.sgcomp); + sr->in.sg = NULL; + sr->in.sgmap_cnt = 0; + + dma_unmap_sg(dev, sr->out.sg, sg_nents(sr->out.sg), + DMA_BIDIRECTIONAL); + dma_unmap_single(dev, sr->out.sgcomp_dma, sr->out.sgcomp_len, + DMA_TO_DEVICE); + kfree(sr->out.sgcomp); + sr->out.sg = NULL; + sr->out.sgmap_cnt = 0; +} + +static void softreq_destroy(struct nitrox_softreq *sr) +{ + softreq_unmap_sgbufs(sr); + kfree(sr); +} + +/** + * create_sg_component - create SG componets for N5 device. + * @sr: Request structure + * @sgtbl: SG table + * @map_nents: number of dma mapped entries + * + * Component structure + * + * 63 48 47 32 31 16 15 0 + * -------------------------------------- + * | LEN0 | LEN1 | LEN2 | LEN3 | + * |------------------------------------- + * | PTR0 | + * -------------------------------------- + * | PTR1 | + * -------------------------------------- + * | PTR2 | + * -------------------------------------- + * | PTR3 | + * -------------------------------------- + * + * Returns 0 if success or a negative errno code on error. + */ +static int create_sg_component(struct nitrox_softreq *sr, + struct nitrox_sgtable *sgtbl, int map_nents) +{ + struct nitrox_device *ndev = sr->ndev; + struct nitrox_sgcomp *sgcomp; + struct scatterlist *sg; + dma_addr_t dma; + size_t sz_comp; + int i, j, nr_sgcomp; + + nr_sgcomp = roundup(map_nents, 4) / 4; + + /* each component holds 4 dma pointers */ + sz_comp = nr_sgcomp * sizeof(*sgcomp); + sgcomp = kzalloc(sz_comp, sr->gfp); + if (!sgcomp) + return -ENOMEM; + + sgtbl->sgcomp = sgcomp; + + sg = sgtbl->sg; + /* populate device sg component */ + for (i = 0; i < nr_sgcomp; i++) { + for (j = 0; j < 4 && sg; j++) { + sgcomp[i].len[j] = cpu_to_be16(sg_dma_len(sg)); + sgcomp[i].dma[j] = cpu_to_be64(sg_dma_address(sg)); + sg = sg_next(sg); + } + } + /* map the device sg component */ + dma = dma_map_single(DEV(ndev), sgtbl->sgcomp, sz_comp, DMA_TO_DEVICE); + if (dma_mapping_error(DEV(ndev), dma)) { + kfree(sgtbl->sgcomp); + sgtbl->sgcomp = NULL; + return -ENOMEM; + } + + sgtbl->sgcomp_dma = dma; + sgtbl->sgcomp_len = sz_comp; + + return 0; +} + +/** + * dma_map_inbufs - DMA map input sglist and creates sglist component + * for N5 device. + * @sr: Request structure + * @req: Crypto request structre + * + * Returns 0 if successful or a negative errno code on error. + */ +static int dma_map_inbufs(struct nitrox_softreq *sr, + struct se_crypto_request *req) +{ + struct device *dev = DEV(sr->ndev); + struct scatterlist *sg; + int i, nents, ret = 0; + + nents = dma_map_sg(dev, req->src, sg_nents(req->src), + DMA_BIDIRECTIONAL); + if (!nents) + return -EINVAL; + + for_each_sg(req->src, sg, nents, i) + sr->in.total_bytes += sg_dma_len(sg); + + sr->in.sg = req->src; + sr->in.sgmap_cnt = nents; + ret = create_sg_component(sr, &sr->in, sr->in.sgmap_cnt); + if (ret) + goto incomp_err; + + return 0; + +incomp_err: + dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_BIDIRECTIONAL); + sr->in.sgmap_cnt = 0; + return ret; +} + +static int dma_map_outbufs(struct nitrox_softreq *sr, + struct se_crypto_request *req) +{ + struct device *dev = DEV(sr->ndev); + int nents, ret = 0; + + nents = dma_map_sg(dev, req->dst, sg_nents(req->dst), + DMA_BIDIRECTIONAL); + if (!nents) + return -EINVAL; + + sr->out.sg = req->dst; + sr->out.sgmap_cnt = nents; + ret = create_sg_component(sr, &sr->out, sr->out.sgmap_cnt); + if (ret) + goto outcomp_map_err; + + return 0; + +outcomp_map_err: + dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_BIDIRECTIONAL); + sr->out.sgmap_cnt = 0; + sr->out.sg = NULL; + return ret; +} + +static inline int softreq_map_iobuf(struct nitrox_softreq *sr, + struct se_crypto_request *creq) +{ + int ret; + + ret = dma_map_inbufs(sr, creq); + if (ret) + return ret; + + ret = dma_map_outbufs(sr, creq); + if (ret) + softreq_unmap_sgbufs(sr); + + return ret; +} + +static inline void backlog_list_add(struct nitrox_softreq *sr, + struct nitrox_cmdq *cmdq) +{ + INIT_LIST_HEAD(&sr->backlog); + + spin_lock_bh(&cmdq->backlog_qlock); + list_add_tail(&sr->backlog, &cmdq->backlog_head); + atomic_inc(&cmdq->backlog_count); + atomic_set(&sr->status, REQ_BACKLOG); + spin_unlock_bh(&cmdq->backlog_qlock); +} + +static inline void response_list_add(struct nitrox_softreq *sr, + struct nitrox_cmdq *cmdq) +{ + INIT_LIST_HEAD(&sr->response); + + spin_lock_bh(&cmdq->resp_qlock); + list_add_tail(&sr->response, &cmdq->response_head); + spin_unlock_bh(&cmdq->resp_qlock); +} + +static inline void response_list_del(struct nitrox_softreq *sr, + struct nitrox_cmdq *cmdq) +{ + spin_lock_bh(&cmdq->resp_qlock); + list_del(&sr->response); + spin_unlock_bh(&cmdq->resp_qlock); +} + +static struct nitrox_softreq * +get_first_response_entry(struct nitrox_cmdq *cmdq) +{ + return list_first_entry_or_null(&cmdq->response_head, + struct nitrox_softreq, response); +} + +static inline bool cmdq_full(struct nitrox_cmdq *cmdq, int qlen) +{ + if (atomic_inc_return(&cmdq->pending_count) > qlen) { + atomic_dec(&cmdq->pending_count); + /* sync with other cpus */ + smp_mb__after_atomic(); + return true; + } + /* sync with other cpus */ + smp_mb__after_atomic(); + return false; +} + +/** + * post_se_instr - Post SE instruction to Packet Input ring + * @sr: Request structure + * @cmdq: Command queue structure + * + * Returns 0 if successful or a negative error code, + * if no space in ring. + */ +static void post_se_instr(struct nitrox_softreq *sr, + struct nitrox_cmdq *cmdq) +{ + struct nitrox_device *ndev = sr->ndev; + int idx; + u8 *ent; + + spin_lock_bh(&cmdq->cmd_qlock); + + idx = cmdq->write_idx; + /* copy the instruction */ + ent = cmdq->base + (idx * cmdq->instr_size); + memcpy(ent, &sr->instr, cmdq->instr_size); + + atomic_set(&sr->status, REQ_POSTED); + response_list_add(sr, cmdq); + sr->tstamp = jiffies; + /* flush the command queue updates */ + dma_wmb(); + + /* Ring doorbell with count 1 */ + writeq(1, cmdq->dbell_csr_addr); + + cmdq->write_idx = incr_index(idx, 1, ndev->qlen); + + spin_unlock_bh(&cmdq->cmd_qlock); + + /* increment the posted command count */ + atomic64_inc(&ndev->stats.posted); +} + +static int post_backlog_cmds(struct nitrox_cmdq *cmdq) +{ + struct nitrox_device *ndev = cmdq->ndev; + struct nitrox_softreq *sr, *tmp; + int ret = 0; + + if (!atomic_read(&cmdq->backlog_count)) + return 0; + + spin_lock_bh(&cmdq->backlog_qlock); + + list_for_each_entry_safe(sr, tmp, &cmdq->backlog_head, backlog) { + /* submit until space available */ + if (unlikely(cmdq_full(cmdq, ndev->qlen))) { + ret = -ENOSPC; + break; + } + /* delete from backlog list */ + list_del(&sr->backlog); + atomic_dec(&cmdq->backlog_count); + /* sync with other cpus */ + smp_mb__after_atomic(); + + /* post the command */ + post_se_instr(sr, cmdq); + } + spin_unlock_bh(&cmdq->backlog_qlock); + + return ret; +} + +static int nitrox_enqueue_request(struct nitrox_softreq *sr) +{ + struct nitrox_cmdq *cmdq = sr->cmdq; + struct nitrox_device *ndev = sr->ndev; + + /* try to post backlog requests */ + post_backlog_cmds(cmdq); + + if (unlikely(cmdq_full(cmdq, ndev->qlen))) { + if (!(sr->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { + /* increment drop count */ + atomic64_inc(&ndev->stats.dropped); + return -ENOSPC; + } + /* add to backlog list */ + backlog_list_add(sr, cmdq); + return -EINPROGRESS; + } + post_se_instr(sr, cmdq); + + return -EINPROGRESS; +} + +/** + * nitrox_process_se_request - Send request to SE core + * @ndev: NITROX device + * @req: Crypto request + * @callback: Completion callback + * @cb_arg: Completion callback arguments + * + * Returns 0 on success, or a negative error code. + */ +int nitrox_process_se_request(struct nitrox_device *ndev, + struct se_crypto_request *req, + completion_t callback, + void *cb_arg) +{ + struct nitrox_softreq *sr; + dma_addr_t ctx_handle = 0; + int qno, ret = 0; + + if (!nitrox_ready(ndev)) + return -ENODEV; + + sr = kzalloc(sizeof(*sr), req->gfp); + if (!sr) + return -ENOMEM; + + sr->ndev = ndev; + sr->flags = req->flags; + sr->gfp = req->gfp; + sr->callback = callback; + sr->cb_arg = cb_arg; + + atomic_set(&sr->status, REQ_NOT_POSTED); + + sr->resp.orh = req->orh; + sr->resp.completion = req->comp; + + ret = softreq_map_iobuf(sr, req); + if (ret) { + kfree(sr); + return ret; + } + + /* get the context handle */ + if (req->ctx_handle) { + struct ctx_hdr *hdr; + u8 *ctx_ptr; + + ctx_ptr = (u8 *)(uintptr_t)req->ctx_handle; + hdr = (struct ctx_hdr *)(ctx_ptr - sizeof(struct ctx_hdr)); + ctx_handle = hdr->ctx_dma; + } + + /* select the queue */ + qno = smp_processor_id() % ndev->nr_queues; + + sr->cmdq = &ndev->pkt_inq[qno]; + + /* + * 64-Byte Instruction Format + * + * ---------------------- + * | DPTR0 | 8 bytes + * ---------------------- + * | PKT_IN_INSTR_HDR | 8 bytes + * ---------------------- + * | PKT_IN_HDR | 16 bytes + * ---------------------- + * | SLC_INFO | 16 bytes + * ---------------------- + * | Front data | 16 bytes + * ---------------------- + */ + + /* fill the packet instruction */ + /* word 0 */ + sr->instr.dptr0 = cpu_to_be64(sr->in.sgcomp_dma); + + /* word 1 */ + sr->instr.ih.value = 0; + sr->instr.ih.s.g = 1; + sr->instr.ih.s.gsz = sr->in.sgmap_cnt; + sr->instr.ih.s.ssz = sr->out.sgmap_cnt; + sr->instr.ih.s.fsz = FDATA_SIZE + sizeof(struct gphdr); + sr->instr.ih.s.tlen = sr->instr.ih.s.fsz + sr->in.total_bytes; + sr->instr.ih.bev = cpu_to_be64(sr->instr.ih.value); + + /* word 2 */ + sr->instr.irh.value[0] = 0; + sr->instr.irh.s.uddl = MIN_UDD_LEN; + /* context length in 64-bit words */ + sr->instr.irh.s.ctxl = (req->ctrl.s.ctxl / 8); + /* offset from solicit base port 256 */ + sr->instr.irh.s.destport = SOLICIT_BASE_DPORT + qno; + sr->instr.irh.s.ctxc = req->ctrl.s.ctxc; + sr->instr.irh.s.arg = req->ctrl.s.arg; + sr->instr.irh.s.opcode = req->opcode; + sr->instr.irh.bev[0] = cpu_to_be64(sr->instr.irh.value[0]); + + /* word 3 */ + sr->instr.irh.s.ctxp = cpu_to_be64(ctx_handle); + + /* word 4 */ + sr->instr.slc.value[0] = 0; + sr->instr.slc.s.ssz = sr->out.sgmap_cnt; + sr->instr.slc.bev[0] = cpu_to_be64(sr->instr.slc.value[0]); + + /* word 5 */ + sr->instr.slc.s.rptr = cpu_to_be64(sr->out.sgcomp_dma); + + /* + * No conversion for front data, + * It goes into payload + * put GP Header in front data + */ + sr->instr.fdata[0] = *((u64 *)&req->gph); + sr->instr.fdata[1] = 0; + + ret = nitrox_enqueue_request(sr); + if (ret == -ENOSPC) + goto send_fail; + + return ret; + +send_fail: + softreq_destroy(sr); + return ret; +} + +static inline int cmd_timeout(unsigned long tstamp, unsigned long timeout) +{ + return time_after_eq(jiffies, (tstamp + timeout)); +} + +void backlog_qflush_work(struct work_struct *work) +{ + struct nitrox_cmdq *cmdq; + + cmdq = container_of(work, struct nitrox_cmdq, backlog_qflush); + post_backlog_cmds(cmdq); +} + +static bool sr_completed(struct nitrox_softreq *sr) +{ + u64 orh = READ_ONCE(*sr->resp.orh); + unsigned long timeout = jiffies + msecs_to_jiffies(1); + + if ((orh != PENDING_SIG) && (orh & 0xff)) + return true; + + while (READ_ONCE(*sr->resp.completion) == PENDING_SIG) { + if (time_after(jiffies, timeout)) { + pr_err("comp not done\n"); + return false; + } + } + + return true; +} + +/** + * process_response_list - process completed requests + * @cmdq: Command queue structure + * + * Returns the number of responses processed. + */ +static void process_response_list(struct nitrox_cmdq *cmdq) +{ + struct nitrox_device *ndev = cmdq->ndev; + struct nitrox_softreq *sr; + int req_completed = 0, err = 0, budget; + completion_t callback; + void *cb_arg; + + /* check all pending requests */ + budget = atomic_read(&cmdq->pending_count); + + while (req_completed < budget) { + sr = get_first_response_entry(cmdq); + if (!sr) + break; + + if (atomic_read(&sr->status) != REQ_POSTED) + break; + + /* check orh and completion bytes updates */ + if (!sr_completed(sr)) { + /* request not completed, check for timeout */ + if (!cmd_timeout(sr->tstamp, ndev->timeout)) + break; + dev_err_ratelimited(DEV(ndev), + "Request timeout, orh 0x%016llx\n", + READ_ONCE(*sr->resp.orh)); + } + atomic_dec(&cmdq->pending_count); + atomic64_inc(&ndev->stats.completed); + /* sync with other cpus */ + smp_mb__after_atomic(); + /* remove from response list */ + response_list_del(sr, cmdq); + /* ORH error code */ + err = READ_ONCE(*sr->resp.orh) & 0xff; + callback = sr->callback; + cb_arg = sr->cb_arg; + softreq_destroy(sr); + if (callback) + callback(cb_arg, err); + + req_completed++; + } +} + +/* + * pkt_slc_resp_tasklet - post processing of SE responses + */ +void pkt_slc_resp_tasklet(unsigned long data) +{ + struct nitrox_q_vector *qvec = (void *)(uintptr_t)(data); + struct nitrox_cmdq *cmdq = qvec->cmdq; + union nps_pkt_slc_cnts slc_cnts; + + /* read completion count */ + slc_cnts.value = readq(cmdq->compl_cnt_csr_addr); + /* resend the interrupt if more work to do */ + slc_cnts.s.resend = 1; + + process_response_list(cmdq); + + /* + * clear the interrupt with resend bit enabled, + * MSI-X interrupt generates if Completion count > Threshold + */ + writeq(slc_cnts.value, cmdq->compl_cnt_csr_addr); + + if (atomic_read(&cmdq->backlog_count)) + schedule_work(&cmdq->backlog_qflush); +} |