diff options
Diffstat (limited to 'drivers/crypto/cavium')
51 files changed, 14911 insertions, 0 deletions
diff --git a/drivers/crypto/cavium/Makefile b/drivers/crypto/cavium/Makefile new file mode 100644 index 000000000..4679c06b6 --- /dev/null +++ b/drivers/crypto/cavium/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for Cavium crypto device drivers +# +obj-$(CONFIG_CRYPTO_DEV_CAVIUM_ZIP) += zip/ diff --git a/drivers/crypto/cavium/cpt/Kconfig b/drivers/crypto/cavium/cpt/Kconfig new file mode 100644 index 000000000..b9874058d --- /dev/null +++ b/drivers/crypto/cavium/cpt/Kconfig @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Cavium crypto device configuration +# + +config CRYPTO_DEV_CPT + tristate + +config CAVIUM_CPT + tristate "Cavium Cryptographic Accelerator driver" + depends on ARCH_THUNDER || COMPILE_TEST + depends on PCI_MSI && 64BIT + select CRYPTO_DEV_CPT + help + Support for Cavium CPT block found in octeon-tx series of + processors. + + To compile this as a module, choose M here. diff --git a/drivers/crypto/cavium/cpt/Makefile b/drivers/crypto/cavium/cpt/Makefile new file mode 100644 index 000000000..0f04f1b5c --- /dev/null +++ b/drivers/crypto/cavium/cpt/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_CAVIUM_CPT) += cptpf.o cptvf.o +cptpf-objs := cptpf_main.o cptpf_mbox.o +cptvf-objs := cptvf_main.o cptvf_reqmanager.o cptvf_mbox.o cptvf_algs.o diff --git a/drivers/crypto/cavium/cpt/cpt_common.h b/drivers/crypto/cavium/cpt/cpt_common.h new file mode 100644 index 000000000..eb3361518 --- /dev/null +++ b/drivers/crypto/cavium/cpt/cpt_common.h @@ -0,0 +1,153 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2016 Cavium, Inc. + */ + +#ifndef __CPT_COMMON_H +#define __CPT_COMMON_H + +#include <asm/byteorder.h> +#include <linux/delay.h> +#include <linux/pci.h> + +#include "cpt_hw_types.h" + +/* Device ID */ +#define CPT_81XX_PCI_PF_DEVICE_ID 0xa040 +#define CPT_81XX_PCI_VF_DEVICE_ID 0xa041 + +/* flags to indicate the features supported */ +#define CPT_FLAG_SRIOV_ENABLED BIT(1) +#define CPT_FLAG_VF_DRIVER BIT(2) +#define CPT_FLAG_DEVICE_READY BIT(3) + +#define cpt_sriov_enabled(cpt) ((cpt)->flags & CPT_FLAG_SRIOV_ENABLED) +#define cpt_vf_driver(cpt) ((cpt)->flags & CPT_FLAG_VF_DRIVER) +#define cpt_device_ready(cpt) ((cpt)->flags & CPT_FLAG_DEVICE_READY) + +#define CPT_MBOX_MSG_TYPE_ACK 1 +#define CPT_MBOX_MSG_TYPE_NACK 2 +#define CPT_MBOX_MSG_TIMEOUT 2000 +#define VF_STATE_DOWN 0 +#define VF_STATE_UP 1 + +/* + * CPT Registers map for 81xx + */ + +/* PF registers */ +#define CPTX_PF_CONSTANTS(a) (0x0ll + ((u64)(a) << 36)) +#define CPTX_PF_RESET(a) (0x100ll + ((u64)(a) << 36)) +#define CPTX_PF_DIAG(a) (0x120ll + ((u64)(a) << 36)) +#define CPTX_PF_BIST_STATUS(a) (0x160ll + ((u64)(a) << 36)) +#define CPTX_PF_ECC0_CTL(a) (0x200ll + ((u64)(a) << 36)) +#define CPTX_PF_ECC0_FLIP(a) (0x210ll + ((u64)(a) << 36)) +#define CPTX_PF_ECC0_INT(a) (0x220ll + ((u64)(a) << 36)) +#define CPTX_PF_ECC0_INT_W1S(a) (0x230ll + ((u64)(a) << 36)) +#define CPTX_PF_ECC0_ENA_W1S(a) (0x240ll + ((u64)(a) << 36)) +#define CPTX_PF_ECC0_ENA_W1C(a) (0x250ll + ((u64)(a) << 36)) +#define CPTX_PF_MBOX_INTX(a, b) \ + (0x400ll + ((u64)(a) << 36) + ((b) << 3)) +#define CPTX_PF_MBOX_INT_W1SX(a, b) \ + (0x420ll + ((u64)(a) << 36) + ((b) << 3)) +#define CPTX_PF_MBOX_ENA_W1CX(a, b) \ + (0x440ll + ((u64)(a) << 36) + ((b) << 3)) +#define CPTX_PF_MBOX_ENA_W1SX(a, b) \ + (0x460ll + ((u64)(a) << 36) + ((b) << 3)) +#define CPTX_PF_EXEC_INT(a) (0x500ll + 0x1000000000ll * ((a) & 0x1)) +#define CPTX_PF_EXEC_INT_W1S(a) (0x520ll + ((u64)(a) << 36)) +#define CPTX_PF_EXEC_ENA_W1C(a) (0x540ll + ((u64)(a) << 36)) +#define CPTX_PF_EXEC_ENA_W1S(a) (0x560ll + ((u64)(a) << 36)) +#define CPTX_PF_GX_EN(a, b) \ + (0x600ll + ((u64)(a) << 36) + ((b) << 3)) +#define CPTX_PF_EXEC_INFO(a) (0x700ll + ((u64)(a) << 36)) +#define CPTX_PF_EXEC_BUSY(a) (0x800ll + ((u64)(a) << 36)) +#define CPTX_PF_EXEC_INFO0(a) (0x900ll + ((u64)(a) << 36)) +#define CPTX_PF_EXEC_INFO1(a) (0x910ll + ((u64)(a) << 36)) +#define CPTX_PF_INST_REQ_PC(a) (0x10000ll + ((u64)(a) << 36)) +#define CPTX_PF_INST_LATENCY_PC(a) \ + (0x10020ll + ((u64)(a) << 36)) +#define CPTX_PF_RD_REQ_PC(a) (0x10040ll + ((u64)(a) << 36)) +#define CPTX_PF_RD_LATENCY_PC(a) (0x10060ll + ((u64)(a) << 36)) +#define CPTX_PF_RD_UC_PC(a) (0x10080ll + ((u64)(a) << 36)) +#define CPTX_PF_ACTIVE_CYCLES_PC(a) (0x10100ll + ((u64)(a) << 36)) +#define CPTX_PF_EXE_CTL(a) (0x4000000ll + ((u64)(a) << 36)) +#define CPTX_PF_EXE_STATUS(a) (0x4000008ll + ((u64)(a) << 36)) +#define CPTX_PF_EXE_CLK(a) (0x4000010ll + ((u64)(a) << 36)) +#define CPTX_PF_EXE_DBG_CTL(a) (0x4000018ll + ((u64)(a) << 36)) +#define CPTX_PF_EXE_DBG_DATA(a) (0x4000020ll + ((u64)(a) << 36)) +#define CPTX_PF_EXE_BIST_STATUS(a) (0x4000028ll + ((u64)(a) << 36)) +#define CPTX_PF_EXE_REQ_TIMER(a) (0x4000030ll + ((u64)(a) << 36)) +#define CPTX_PF_EXE_MEM_CTL(a) (0x4000038ll + ((u64)(a) << 36)) +#define CPTX_PF_EXE_PERF_CTL(a) (0x4001000ll + ((u64)(a) << 36)) +#define CPTX_PF_EXE_DBG_CNTX(a, b) \ + (0x4001100ll + ((u64)(a) << 36) + ((b) << 3)) +#define CPTX_PF_EXE_PERF_EVENT_CNT(a) (0x4001180ll + ((u64)(a) << 36)) +#define CPTX_PF_EXE_EPCI_INBX_CNT(a, b) \ + (0x4001200ll + ((u64)(a) << 36) + ((b) << 3)) +#define CPTX_PF_EXE_EPCI_OUTBX_CNT(a, b) \ + (0x4001240ll + ((u64)(a) << 36) + ((b) << 3)) +#define CPTX_PF_ENGX_UCODE_BASE(a, b) \ + (0x4002000ll + ((u64)(a) << 36) + ((b) << 3)) +#define CPTX_PF_QX_CTL(a, b) \ + (0x8000000ll + ((u64)(a) << 36) + ((b) << 20)) +#define CPTX_PF_QX_GMCTL(a, b) \ + (0x8000020ll + ((u64)(a) << 36) + ((b) << 20)) +#define CPTX_PF_QX_CTL2(a, b) \ + (0x8000100ll + ((u64)(a) << 36) + ((b) << 20)) +#define CPTX_PF_VFX_MBOXX(a, b, c) \ + (0x8001000ll + ((u64)(a) << 36) + ((b) << 20) + ((c) << 8)) + +/* VF registers */ +#define CPTX_VQX_CTL(a, b) (0x100ll + ((u64)(a) << 36) + ((b) << 20)) +#define CPTX_VQX_SADDR(a, b) (0x200ll + ((u64)(a) << 36) + ((b) << 20)) +#define CPTX_VQX_DONE_WAIT(a, b) (0x400ll + ((u64)(a) << 36) + ((b) << 20)) +#define CPTX_VQX_INPROG(a, b) (0x410ll + ((u64)(a) << 36) + ((b) << 20)) +#define CPTX_VQX_DONE(a, b) (0x420ll + ((u64)(a) << 36) + ((b) << 20)) +#define CPTX_VQX_DONE_ACK(a, b) (0x440ll + ((u64)(a) << 36) + ((b) << 20)) +#define CPTX_VQX_DONE_INT_W1S(a, b) (0x460ll + ((u64)(a) << 36) + ((b) << 20)) +#define CPTX_VQX_DONE_INT_W1C(a, b) (0x468ll + ((u64)(a) << 36) + ((b) << 20)) +#define CPTX_VQX_DONE_ENA_W1S(a, b) (0x470ll + ((u64)(a) << 36) + ((b) << 20)) +#define CPTX_VQX_DONE_ENA_W1C(a, b) (0x478ll + ((u64)(a) << 36) + ((b) << 20)) +#define CPTX_VQX_MISC_INT(a, b) (0x500ll + ((u64)(a) << 36) + ((b) << 20)) +#define CPTX_VQX_MISC_INT_W1S(a, b) (0x508ll + ((u64)(a) << 36) + ((b) << 20)) +#define CPTX_VQX_MISC_ENA_W1S(a, b) (0x510ll + ((u64)(a) << 36) + ((b) << 20)) +#define CPTX_VQX_MISC_ENA_W1C(a, b) (0x518ll + ((u64)(a) << 36) + ((b) << 20)) +#define CPTX_VQX_DOORBELL(a, b) (0x600ll + ((u64)(a) << 36) + ((b) << 20)) +#define CPTX_VFX_PF_MBOXX(a, b, c) \ + (0x1000ll + ((u64)(a) << 36) + ((b) << 20) + ((c) << 3)) + +enum vftype { + AE_TYPES = 1, + SE_TYPES = 2, + BAD_CPT_TYPES, +}; + +/* Max CPT devices supported */ +enum cpt_mbox_opcode { + CPT_MSG_VF_UP = 1, + CPT_MSG_VF_DOWN, + CPT_MSG_READY, + CPT_MSG_QLEN, + CPT_MSG_QBIND_GRP, + CPT_MSG_VQ_PRIORITY, +}; + +/* CPT mailbox structure */ +struct cpt_mbox { + u64 msg; /* Message type MBOX[0] */ + u64 data;/* Data MBOX[1] */ +}; + +/* Register read/write APIs */ +static inline void cpt_write_csr64(u8 __iomem *hw_addr, u64 offset, + u64 val) +{ + writeq(val, hw_addr + offset); +} + +static inline u64 cpt_read_csr64(u8 __iomem *hw_addr, u64 offset) +{ + return readq(hw_addr + offset); +} +#endif /* __CPT_COMMON_H */ diff --git a/drivers/crypto/cavium/cpt/cpt_hw_types.h b/drivers/crypto/cavium/cpt/cpt_hw_types.h new file mode 100644 index 000000000..ae4791a8e --- /dev/null +++ b/drivers/crypto/cavium/cpt/cpt_hw_types.h @@ -0,0 +1,655 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2016 Cavium, Inc. + */ + +#ifndef __CPT_HW_TYPES_H +#define __CPT_HW_TYPES_H + +#include "cpt_common.h" + +/** + * Enumeration cpt_comp_e + * + * CPT Completion Enumeration + * Enumerates the values of CPT_RES_S[COMPCODE]. + */ +enum cpt_comp_e { + CPT_COMP_E_NOTDONE = 0x00, + CPT_COMP_E_GOOD = 0x01, + CPT_COMP_E_FAULT = 0x02, + CPT_COMP_E_SWERR = 0x03, + CPT_COMP_E_LAST_ENTRY = 0xFF +}; + +/** + * Structure cpt_inst_s + * + * CPT Instruction Structure + * This structure specifies the instruction layout. Instructions are + * stored in memory as little-endian unless CPT()_PF_Q()_CTL[INST_BE] is set. + * cpt_inst_s_s + * Word 0 + * doneint:1 Done interrupt. + * 0 = No interrupts related to this instruction. + * 1 = When the instruction completes, CPT()_VQ()_DONE[DONE] will be + * incremented,and based on the rules described there an interrupt may + * occur. + * Word 1 + * res_addr [127: 64] Result IOVA. + * If nonzero, specifies where to write CPT_RES_S. + * If zero, no result structure will be written. + * Address must be 16-byte aligned. + * Bits <63:49> are ignored by hardware; software should use a + * sign-extended bit <48> for forward compatibility. + * Word 2 + * grp:10 [171:162] If [WQ_PTR] is nonzero, the SSO guest-group to use when + * CPT submits work SSO. + * For the SSO to not discard the add-work request, FPA_PF_MAP() must map + * [GRP] and CPT()_PF_Q()_GMCTL[GMID] as valid. + * tt:2 [161:160] If [WQ_PTR] is nonzero, the SSO tag type to use when CPT + * submits work to SSO + * tag:32 [159:128] If [WQ_PTR] is nonzero, the SSO tag to use when CPT + * submits work to SSO. + * Word 3 + * wq_ptr [255:192] If [WQ_PTR] is nonzero, it is a pointer to a + * work-queue entry that CPT submits work to SSO after all context, + * output data, and result write operations are visible to other + * CNXXXX units and the cores. Bits <2:0> must be zero. + * Bits <63:49> are ignored by hardware; software should + * use a sign-extended bit <48> for forward compatibility. + * Internal: + * Bits <63:49>, <2:0> are ignored by hardware, treated as always 0x0. + * Word 4 + * ei0; [319:256] Engine instruction word 0. Passed to the AE/SE. + * Word 5 + * ei1; [383:320] Engine instruction word 1. Passed to the AE/SE. + * Word 6 + * ei2; [447:384] Engine instruction word 1. Passed to the AE/SE. + * Word 7 + * ei3; [511:448] Engine instruction word 1. Passed to the AE/SE. + * + */ +union cpt_inst_s { + u64 u[8]; + struct cpt_inst_s_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_17_63:47; + u64 doneint:1; + u64 reserved_0_1:16; +#else /* Word 0 - Little Endian */ + u64 reserved_0_15:16; + u64 doneint:1; + u64 reserved_17_63:47; +#endif /* Word 0 - End */ + u64 res_addr; +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 2 - Big Endian */ + u64 reserved_172_19:20; + u64 grp:10; + u64 tt:2; + u64 tag:32; +#else /* Word 2 - Little Endian */ + u64 tag:32; + u64 tt:2; + u64 grp:10; + u64 reserved_172_191:20; +#endif /* Word 2 - End */ + u64 wq_ptr; + u64 ei0; + u64 ei1; + u64 ei2; + u64 ei3; + } s; +}; + +/** + * Structure cpt_res_s + * + * CPT Result Structure + * The CPT coprocessor writes the result structure after it completes a + * CPT_INST_S instruction. The result structure is exactly 16 bytes, and + * each instruction completion produces exactly one result structure. + * + * This structure is stored in memory as little-endian unless + * CPT()_PF_Q()_CTL[INST_BE] is set. + * cpt_res_s_s + * Word 0 + * doneint:1 [16:16] Done interrupt. This bit is copied from the + * corresponding instruction's CPT_INST_S[DONEINT]. + * compcode:8 [7:0] Indicates completion/error status of the CPT coprocessor + * for the associated instruction, as enumerated by CPT_COMP_E. + * Core software may write the memory location containing [COMPCODE] to + * 0x0 before ringing the doorbell, and then poll for completion by + * checking for a nonzero value. + * Once the core observes a nonzero [COMPCODE] value in this case,the CPT + * coprocessor will have also completed L2/DRAM write operations. + * Word 1 + * reserved + * + */ +union cpt_res_s { + u64 u[2]; + struct cpt_res_s_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_17_63:47; + u64 doneint:1; + u64 reserved_8_15:8; + u64 compcode:8; +#else /* Word 0 - Little Endian */ + u64 compcode:8; + u64 reserved_8_15:8; + u64 doneint:1; + u64 reserved_17_63:47; +#endif /* Word 0 - End */ + u64 reserved_64_127; + } s; +}; + +/** + * Register (NCB) cpt#_pf_bist_status + * + * CPT PF Control Bist Status Register + * This register has the BIST status of memories. Each bit is the BIST result + * of an individual memory (per bit, 0 = pass and 1 = fail). + * cptx_pf_bist_status_s + * Word0 + * bstatus [29:0](RO/H) BIST status. One bit per memory, enumerated by + * CPT_RAMS_E. + */ +union cptx_pf_bist_status { + u64 u; + struct cptx_pf_bist_status_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_30_63:34; + u64 bstatus:30; +#else /* Word 0 - Little Endian */ + u64 bstatus:30; + u64 reserved_30_63:34; +#endif /* Word 0 - End */ + } s; +}; + +/** + * Register (NCB) cpt#_pf_constants + * + * CPT PF Constants Register + * This register contains implementation-related parameters of CPT in CNXXXX. + * cptx_pf_constants_s + * Word 0 + * reserved_40_63:24 [63:40] Reserved. + * epcis:8 [39:32](RO) Number of EPCI busses. + * grps:8 [31:24](RO) Number of engine groups implemented. + * ae:8 [23:16](RO/H) Number of AEs. In CNXXXX, for CPT0 returns 0x0, + * for CPT1 returns 0x18, or less if there are fuse-disables. + * se:8 [15:8](RO/H) Number of SEs. In CNXXXX, for CPT0 returns 0x30, + * or less if there are fuse-disables, for CPT1 returns 0x0. + * vq:8 [7:0](RO) Number of VQs. + */ +union cptx_pf_constants { + u64 u; + struct cptx_pf_constants_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_40_63:24; + u64 epcis:8; + u64 grps:8; + u64 ae:8; + u64 se:8; + u64 vq:8; +#else /* Word 0 - Little Endian */ + u64 vq:8; + u64 se:8; + u64 ae:8; + u64 grps:8; + u64 epcis:8; + u64 reserved_40_63:24; +#endif /* Word 0 - End */ + } s; +}; + +/** + * Register (NCB) cpt#_pf_exe_bist_status + * + * CPT PF Engine Bist Status Register + * This register has the BIST status of each engine. Each bit is the + * BIST result of an individual engine (per bit, 0 = pass and 1 = fail). + * cptx_pf_exe_bist_status_s + * Word0 + * reserved_48_63:16 [63:48] reserved + * bstatus:48 [47:0](RO/H) BIST status. One bit per engine. + * + */ +union cptx_pf_exe_bist_status { + u64 u; + struct cptx_pf_exe_bist_status_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_48_63:16; + u64 bstatus:48; +#else /* Word 0 - Little Endian */ + u64 bstatus:48; + u64 reserved_48_63:16; +#endif /* Word 0 - End */ + } s; +}; + +/** + * Register (NCB) cpt#_pf_q#_ctl + * + * CPT Queue Control Register + * This register configures queues. This register should be changed only + * when quiescent (see CPT()_VQ()_INPROG[INFLIGHT]). + * cptx_pf_qx_ctl_s + * Word0 + * reserved_60_63:4 [63:60] reserved. + * aura:12; [59:48](R/W) Guest-aura for returning this queue's + * instruction-chunk buffers to FPA. Only used when [INST_FREE] is set. + * For the FPA to not discard the request, FPA_PF_MAP() must map + * [AURA] and CPT()_PF_Q()_GMCTL[GMID] as valid. + * reserved_45_47:3 [47:45] reserved. + * size:13 [44:32](R/W) Command-buffer size, in number of 64-bit words per + * command buffer segment. Must be 8*n + 1, where n is the number of + * instructions per buffer segment. + * reserved_11_31:21 [31:11] Reserved. + * cont_err:1 [10:10](R/W) Continue on error. + * 0 = When CPT()_VQ()_MISC_INT[NWRP], CPT()_VQ()_MISC_INT[IRDE] or + * CPT()_VQ()_MISC_INT[DOVF] are set by hardware or software via + * CPT()_VQ()_MISC_INT_W1S, then CPT()_VQ()_CTL[ENA] is cleared. Due to + * pipelining, additional instructions may have been processed between the + * instruction causing the error and the next instruction in the disabled + * queue (the instruction at CPT()_VQ()_SADDR). + * 1 = Ignore errors and continue processing instructions. + * For diagnostic use only. + * inst_free:1 [9:9](R/W) Instruction FPA free. When set, when CPT reaches the + * end of an instruction chunk, that chunk will be freed to the FPA. + * inst_be:1 [8:8](R/W) Instruction big-endian control. When set, instructions, + * instruction next chunk pointers, and result structures are stored in + * big-endian format in memory. + * iqb_ldwb:1 [7:7](R/W) Instruction load don't write back. + * 0 = The hardware issues NCB transient load (LDT) towards the cache, + * which if the line hits and it is dirty will cause the line to be + * written back before being replaced. + * 1 = The hardware issues NCB LDWB read-and-invalidate command towards + * the cache when fetching the last word of instructions; as a result the + * line will not be written back when replaced. This improves + * performance, but software must not read the instructions after they are + * posted to the hardware. Reads that do not consume the last word of a + * cache line always use LDI. + * reserved_4_6:3 [6:4] Reserved. + * grp:3; [3:1](R/W) Engine group. + * pri:1; [0:0](R/W) Queue priority. + * 1 = This queue has higher priority. Round-robin between higher + * priority queues. + * 0 = This queue has lower priority. Round-robin between lower + * priority queues. + */ +union cptx_pf_qx_ctl { + u64 u; + struct cptx_pf_qx_ctl_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_60_63:4; + u64 aura:12; + u64 reserved_45_47:3; + u64 size:13; + u64 reserved_11_31:21; + u64 cont_err:1; + u64 inst_free:1; + u64 inst_be:1; + u64 iqb_ldwb:1; + u64 reserved_4_6:3; + u64 grp:3; + u64 pri:1; +#else /* Word 0 - Little Endian */ + u64 pri:1; + u64 grp:3; + u64 reserved_4_6:3; + u64 iqb_ldwb:1; + u64 inst_be:1; + u64 inst_free:1; + u64 cont_err:1; + u64 reserved_11_31:21; + u64 size:13; + u64 reserved_45_47:3; + u64 aura:12; + u64 reserved_60_63:4; +#endif /* Word 0 - End */ + } s; +}; + +/** + * Register (NCB) cpt#_vq#_saddr + * + * CPT Queue Starting Buffer Address Registers + * These registers set the instruction buffer starting address. + * cptx_vqx_saddr_s + * Word0 + * reserved_49_63:15 [63:49] Reserved. + * ptr:43 [48:6](R/W/H) Instruction buffer IOVA <48:6> (64-byte aligned). + * When written, it is the initial buffer starting address; when read, + * it is the next read pointer to be requested from L2C. The PTR field + * is overwritten with the next pointer each time that the command buffer + * segment is exhausted. New commands will then be read from the newly + * specified command buffer pointer. + * reserved_0_5:6 [5:0] Reserved. + * + */ +union cptx_vqx_saddr { + u64 u; + struct cptx_vqx_saddr_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_49_63:15; + u64 ptr:43; + u64 reserved_0_5:6; +#else /* Word 0 - Little Endian */ + u64 reserved_0_5:6; + u64 ptr:43; + u64 reserved_49_63:15; +#endif /* Word 0 - End */ + } s; +}; + +/** + * Register (NCB) cpt#_vq#_misc_ena_w1s + * + * CPT Queue Misc Interrupt Enable Set Register + * This register sets interrupt enable bits. + * cptx_vqx_misc_ena_w1s_s + * Word0 + * reserved_5_63:59 [63:5] Reserved. + * swerr:1 [4:4](R/W1S/H) Reads or sets enable for + * CPT(0..1)_VQ(0..63)_MISC_INT[SWERR]. + * nwrp:1 [3:3](R/W1S/H) Reads or sets enable for + * CPT(0..1)_VQ(0..63)_MISC_INT[NWRP]. + * irde:1 [2:2](R/W1S/H) Reads or sets enable for + * CPT(0..1)_VQ(0..63)_MISC_INT[IRDE]. + * dovf:1 [1:1](R/W1S/H) Reads or sets enable for + * CPT(0..1)_VQ(0..63)_MISC_INT[DOVF]. + * mbox:1 [0:0](R/W1S/H) Reads or sets enable for + * CPT(0..1)_VQ(0..63)_MISC_INT[MBOX]. + * + */ +union cptx_vqx_misc_ena_w1s { + u64 u; + struct cptx_vqx_misc_ena_w1s_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_5_63:59; + u64 swerr:1; + u64 nwrp:1; + u64 irde:1; + u64 dovf:1; + u64 mbox:1; +#else /* Word 0 - Little Endian */ + u64 mbox:1; + u64 dovf:1; + u64 irde:1; + u64 nwrp:1; + u64 swerr:1; + u64 reserved_5_63:59; +#endif /* Word 0 - End */ + } s; +}; + +/** + * Register (NCB) cpt#_vq#_doorbell + * + * CPT Queue Doorbell Registers + * Doorbells for the CPT instruction queues. + * cptx_vqx_doorbell_s + * Word0 + * reserved_20_63:44 [63:20] Reserved. + * dbell_cnt:20 [19:0](R/W/H) Number of instruction queue 64-bit words to add + * to the CPT instruction doorbell count. Readback value is the + * current number of pending doorbell requests. If counter overflows + * CPT()_VQ()_MISC_INT[DBELL_DOVF] is set. To reset the count back to + * zero, write one to clear CPT()_VQ()_MISC_INT_ENA_W1C[DBELL_DOVF], + * then write a value of 2^20 minus the read [DBELL_CNT], then write one + * to CPT()_VQ()_MISC_INT_W1C[DBELL_DOVF] and + * CPT()_VQ()_MISC_INT_ENA_W1S[DBELL_DOVF]. Must be a multiple of 8. + * All CPT instructions are 8 words and require a doorbell count of + * multiple of 8. + */ +union cptx_vqx_doorbell { + u64 u; + struct cptx_vqx_doorbell_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_20_63:44; + u64 dbell_cnt:20; +#else /* Word 0 - Little Endian */ + u64 dbell_cnt:20; + u64 reserved_20_63:44; +#endif /* Word 0 - End */ + } s; +}; + +/** + * Register (NCB) cpt#_vq#_inprog + * + * CPT Queue In Progress Count Registers + * These registers contain the per-queue instruction in flight registers. + * cptx_vqx_inprog_s + * Word0 + * reserved_8_63:56 [63:8] Reserved. + * inflight:8 [7:0](RO/H) Inflight count. Counts the number of instructions + * for the VF for which CPT is fetching, executing or responding to + * instructions. However this does not include any interrupts that are + * awaiting software handling (CPT()_VQ()_DONE[DONE] != 0x0). + * A queue may not be reconfigured until: + * 1. CPT()_VQ()_CTL[ENA] is cleared by software. + * 2. [INFLIGHT] is polled until equals to zero. + */ +union cptx_vqx_inprog { + u64 u; + struct cptx_vqx_inprog_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_8_63:56; + u64 inflight:8; +#else /* Word 0 - Little Endian */ + u64 inflight:8; + u64 reserved_8_63:56; +#endif /* Word 0 - End */ + } s; +}; + +/** + * Register (NCB) cpt#_vq#_misc_int + * + * CPT Queue Misc Interrupt Register + * These registers contain the per-queue miscellaneous interrupts. + * cptx_vqx_misc_int_s + * Word 0 + * reserved_5_63:59 [63:5] Reserved. + * swerr:1 [4:4](R/W1C/H) Software error from engines. + * nwrp:1 [3:3](R/W1C/H) NCB result write response error. + * irde:1 [2:2](R/W1C/H) Instruction NCB read response error. + * dovf:1 [1:1](R/W1C/H) Doorbell overflow. + * mbox:1 [0:0](R/W1C/H) PF to VF mailbox interrupt. Set when + * CPT()_VF()_PF_MBOX(0) is written. + * + */ +union cptx_vqx_misc_int { + u64 u; + struct cptx_vqx_misc_int_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_5_63:59; + u64 swerr:1; + u64 nwrp:1; + u64 irde:1; + u64 dovf:1; + u64 mbox:1; +#else /* Word 0 - Little Endian */ + u64 mbox:1; + u64 dovf:1; + u64 irde:1; + u64 nwrp:1; + u64 swerr:1; + u64 reserved_5_63:59; +#endif /* Word 0 - End */ + } s; +}; + +/** + * Register (NCB) cpt#_vq#_done_ack + * + * CPT Queue Done Count Ack Registers + * This register is written by software to acknowledge interrupts. + * cptx_vqx_done_ack_s + * Word0 + * reserved_20_63:44 [63:20] Reserved. + * done_ack:20 [19:0](R/W/H) Number of decrements to CPT()_VQ()_DONE[DONE]. + * Reads CPT()_VQ()_DONE[DONE]. Written by software to acknowledge + * interrupts. If CPT()_VQ()_DONE[DONE] is still nonzero the interrupt + * will be re-sent if the conditions described in CPT()_VQ()_DONE[DONE] + * are satisfied. + * + */ +union cptx_vqx_done_ack { + u64 u; + struct cptx_vqx_done_ack_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_20_63:44; + u64 done_ack:20; +#else /* Word 0 - Little Endian */ + u64 done_ack:20; + u64 reserved_20_63:44; +#endif /* Word 0 - End */ + } s; +}; + +/** + * Register (NCB) cpt#_vq#_done + * + * CPT Queue Done Count Registers + * These registers contain the per-queue instruction done count. + * cptx_vqx_done_s + * Word0 + * reserved_20_63:44 [63:20] Reserved. + * done:20 [19:0](R/W/H) Done count. When CPT_INST_S[DONEINT] set and that + * instruction completes, CPT()_VQ()_DONE[DONE] is incremented when the + * instruction finishes. Write to this field are for diagnostic use only; + * instead software writes CPT()_VQ()_DONE_ACK with the number of + * decrements for this field. + * Interrupts are sent as follows: + * * When CPT()_VQ()_DONE[DONE] = 0, then no results are pending, the + * interrupt coalescing timer is held to zero, and an interrupt is not + * sent. + * * When CPT()_VQ()_DONE[DONE] != 0, then the interrupt coalescing timer + * counts. If the counter is >= CPT()_VQ()_DONE_WAIT[TIME_WAIT]*1024, or + * CPT()_VQ()_DONE[DONE] >= CPT()_VQ()_DONE_WAIT[NUM_WAIT], i.e. enough + * time has passed or enough results have arrived, then the interrupt is + * sent. + * * When CPT()_VQ()_DONE_ACK is written (or CPT()_VQ()_DONE is written + * but this is not typical), the interrupt coalescing timer restarts. + * Note after decrementing this interrupt equation is recomputed, + * for example if CPT()_VQ()_DONE[DONE] >= CPT()_VQ()_DONE_WAIT[NUM_WAIT] + * and because the timer is zero, the interrupt will be resent immediately. + * (This covers the race case between software acknowledging an interrupt + * and a result returning.) + * * When CPT()_VQ()_DONE_ENA_W1S[DONE] = 0, interrupts are not sent, + * but the counting described above still occurs. + * Since CPT instructions complete out-of-order, if software is using + * completion interrupts the suggested scheme is to request a DONEINT on + * each request, and when an interrupt arrives perform a "greedy" scan for + * completions; even if a later command is acknowledged first this will + * not result in missing a completion. + * Software is responsible for making sure [DONE] does not overflow; + * for example by insuring there are not more than 2^20-1 instructions in + * flight that may request interrupts. + * + */ +union cptx_vqx_done { + u64 u; + struct cptx_vqx_done_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_20_63:44; + u64 done:20; +#else /* Word 0 - Little Endian */ + u64 done:20; + u64 reserved_20_63:44; +#endif /* Word 0 - End */ + } s; +}; + +/** + * Register (NCB) cpt#_vq#_done_wait + * + * CPT Queue Done Interrupt Coalescing Wait Registers + * Specifies the per queue interrupt coalescing settings. + * cptx_vqx_done_wait_s + * Word0 + * reserved_48_63:16 [63:48] Reserved. + * time_wait:16; [47:32](R/W) Time hold-off. When CPT()_VQ()_DONE[DONE] = 0 + * or CPT()_VQ()_DONE_ACK is written a timer is cleared. When the timer + * reaches [TIME_WAIT]*1024 then interrupt coalescing ends. + * see CPT()_VQ()_DONE[DONE]. If 0x0, time coalescing is disabled. + * reserved_20_31:12 [31:20] Reserved. + * num_wait:20 [19:0](R/W) Number of messages hold-off. + * When CPT()_VQ()_DONE[DONE] >= [NUM_WAIT] then interrupt coalescing ends + * see CPT()_VQ()_DONE[DONE]. If 0x0, same behavior as 0x1. + * + */ +union cptx_vqx_done_wait { + u64 u; + struct cptx_vqx_done_wait_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_48_63:16; + u64 time_wait:16; + u64 reserved_20_31:12; + u64 num_wait:20; +#else /* Word 0 - Little Endian */ + u64 num_wait:20; + u64 reserved_20_31:12; + u64 time_wait:16; + u64 reserved_48_63:16; +#endif /* Word 0 - End */ + } s; +}; + +/** + * Register (NCB) cpt#_vq#_done_ena_w1s + * + * CPT Queue Done Interrupt Enable Set Registers + * Write 1 to these registers will enable the DONEINT interrupt for the queue. + * cptx_vqx_done_ena_w1s_s + * Word0 + * reserved_1_63:63 [63:1] Reserved. + * done:1 [0:0](R/W1S/H) Write 1 will enable DONEINT for this queue. + * Write 0 has no effect. Read will return the enable bit. + */ +union cptx_vqx_done_ena_w1s { + u64 u; + struct cptx_vqx_done_ena_w1s_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_1_63:63; + u64 done:1; +#else /* Word 0 - Little Endian */ + u64 done:1; + u64 reserved_1_63:63; +#endif /* Word 0 - End */ + } s; +}; + +/** + * Register (NCB) cpt#_vq#_ctl + * + * CPT VF Queue Control Registers + * This register configures queues. This register should be changed (other than + * clearing [ENA]) only when quiescent (see CPT()_VQ()_INPROG[INFLIGHT]). + * cptx_vqx_ctl_s + * Word0 + * reserved_1_63:63 [63:1] Reserved. + * ena:1 [0:0](R/W/H) Enables the logical instruction queue. + * See also CPT()_PF_Q()_CTL[CONT_ERR] and CPT()_VQ()_INPROG[INFLIGHT]. + * 1 = Queue is enabled. + * 0 = Queue is disabled. + */ +union cptx_vqx_ctl { + u64 u; + struct cptx_vqx_ctl_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_1_63:63; + u64 ena:1; +#else /* Word 0 - Little Endian */ + u64 ena:1; + u64 reserved_1_63:63; +#endif /* Word 0 - End */ + } s; +}; +#endif /*__CPT_HW_TYPES_H*/ diff --git a/drivers/crypto/cavium/cpt/cptpf.h b/drivers/crypto/cavium/cpt/cptpf.h new file mode 100644 index 000000000..c88e177a3 --- /dev/null +++ b/drivers/crypto/cavium/cpt/cptpf.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2016 Cavium, Inc. + */ + +#ifndef __CPTPF_H +#define __CPTPF_H + +#include "cpt_common.h" + +#define CSR_DELAY 30 +#define CPT_MAX_CORE_GROUPS 8 +#define CPT_MAX_SE_CORES 10 +#define CPT_MAX_AE_CORES 6 +#define CPT_MAX_TOTAL_CORES (CPT_MAX_SE_CORES + CPT_MAX_AE_CORES) +#define CPT_MAX_VF_NUM 16 +#define CPT_PF_MSIX_VECTORS 3 +#define CPT_PF_INT_VEC_E_MBOXX(a) (0x02 + (a)) +#define CPT_UCODE_VERSION_SZ 32 +struct cpt_device; + +struct microcode { + u8 is_mc_valid; + u8 is_ae; + u8 group; + u8 num_cores; + u32 code_size; + u64 core_mask; + u8 version[CPT_UCODE_VERSION_SZ]; + /* Base info */ + dma_addr_t phys_base; + void *code; +}; + +struct cpt_vf_info { + u8 state; + u8 priority; + u8 id; + u32 qlen; +}; + +/** + * cpt device structure + */ +struct cpt_device { + u16 flags; /* Flags to hold device status bits */ + u8 num_vf_en; /* Number of VFs enabled (0...CPT_MAX_VF_NUM) */ + struct cpt_vf_info vfinfo[CPT_MAX_VF_NUM]; /* Per VF info */ + + void __iomem *reg_base; /* Register start address */ + struct pci_dev *pdev; /* pci device handle */ + + struct microcode mcode[CPT_MAX_CORE_GROUPS]; + u8 next_mc_idx; /* next microcode index */ + u8 next_group; + u8 max_se_cores; + u8 max_ae_cores; +}; + +void cpt_mbox_intr_handler(struct cpt_device *cpt, int mbx); +#endif /* __CPTPF_H */ diff --git a/drivers/crypto/cavium/cpt/cptpf_main.c b/drivers/crypto/cavium/cpt/cptpf_main.c new file mode 100644 index 000000000..6872ac344 --- /dev/null +++ b/drivers/crypto/cavium/cpt/cptpf_main.c @@ -0,0 +1,667 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2016 Cavium, Inc. + */ + +#include <linux/device.h> +#include <linux/firmware.h> +#include <linux/interrupt.h> +#include <linux/module.h> +#include <linux/moduleparam.h> +#include <linux/pci.h> +#include <linux/printk.h> + +#include "cptpf.h" + +#define DRV_NAME "thunder-cpt" +#define DRV_VERSION "1.0" + +static u32 num_vfs = 4; /* Default 4 VF enabled */ +module_param(num_vfs, uint, 0444); +MODULE_PARM_DESC(num_vfs, "Number of VFs to enable(1-16)"); + +/* + * Disable cores specified by coremask + */ +static void cpt_disable_cores(struct cpt_device *cpt, u64 coremask, + u8 type, u8 grp) +{ + u64 pf_exe_ctl; + u32 timeout = 100; + u64 grpmask = 0; + struct device *dev = &cpt->pdev->dev; + + if (type == AE_TYPES) + coremask = (coremask << cpt->max_se_cores); + + /* Disengage the cores from groups */ + grpmask = cpt_read_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp)); + cpt_write_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp), + (grpmask & ~coremask)); + udelay(CSR_DELAY); + grp = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXEC_BUSY(0)); + while (grp & coremask) { + dev_err(dev, "Cores still busy %llx", coremask); + grp = cpt_read_csr64(cpt->reg_base, + CPTX_PF_EXEC_BUSY(0)); + if (timeout--) + break; + + udelay(CSR_DELAY); + } + + /* Disable the cores */ + pf_exe_ctl = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0)); + cpt_write_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0), + (pf_exe_ctl & ~coremask)); + udelay(CSR_DELAY); +} + +/* + * Enable cores specified by coremask + */ +static void cpt_enable_cores(struct cpt_device *cpt, u64 coremask, + u8 type) +{ + u64 pf_exe_ctl; + + if (type == AE_TYPES) + coremask = (coremask << cpt->max_se_cores); + + pf_exe_ctl = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0)); + cpt_write_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0), + (pf_exe_ctl | coremask)); + udelay(CSR_DELAY); +} + +static void cpt_configure_group(struct cpt_device *cpt, u8 grp, + u64 coremask, u8 type) +{ + u64 pf_gx_en = 0; + + if (type == AE_TYPES) + coremask = (coremask << cpt->max_se_cores); + + pf_gx_en = cpt_read_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp)); + cpt_write_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp), + (pf_gx_en | coremask)); + udelay(CSR_DELAY); +} + +static void cpt_disable_mbox_interrupts(struct cpt_device *cpt) +{ + /* Clear mbox(0) interupts for all vfs */ + cpt_write_csr64(cpt->reg_base, CPTX_PF_MBOX_ENA_W1CX(0, 0), ~0ull); +} + +static void cpt_disable_ecc_interrupts(struct cpt_device *cpt) +{ + /* Clear ecc(0) interupts for all vfs */ + cpt_write_csr64(cpt->reg_base, CPTX_PF_ECC0_ENA_W1C(0), ~0ull); +} + +static void cpt_disable_exec_interrupts(struct cpt_device *cpt) +{ + /* Clear exec interupts for all vfs */ + cpt_write_csr64(cpt->reg_base, CPTX_PF_EXEC_ENA_W1C(0), ~0ull); +} + +static void cpt_disable_all_interrupts(struct cpt_device *cpt) +{ + cpt_disable_mbox_interrupts(cpt); + cpt_disable_ecc_interrupts(cpt); + cpt_disable_exec_interrupts(cpt); +} + +static void cpt_enable_mbox_interrupts(struct cpt_device *cpt) +{ + /* Set mbox(0) interupts for all vfs */ + cpt_write_csr64(cpt->reg_base, CPTX_PF_MBOX_ENA_W1SX(0, 0), ~0ull); +} + +static int cpt_load_microcode(struct cpt_device *cpt, struct microcode *mcode) +{ + int ret = 0, core = 0, shift = 0; + u32 total_cores = 0; + struct device *dev = &cpt->pdev->dev; + + if (!mcode || !mcode->code) { + dev_err(dev, "Either the mcode is null or data is NULL\n"); + return -EINVAL; + } + + if (mcode->code_size == 0) { + dev_err(dev, "microcode size is 0\n"); + return -EINVAL; + } + + /* Assumes 0-9 are SE cores for UCODE_BASE registers and + * AE core bases follow + */ + if (mcode->is_ae) { + core = CPT_MAX_SE_CORES; /* start couting from 10 */ + total_cores = CPT_MAX_TOTAL_CORES; /* upto 15 */ + } else { + core = 0; /* start couting from 0 */ + total_cores = CPT_MAX_SE_CORES; /* upto 9 */ + } + + /* Point to microcode for each core of the group */ + for (; core < total_cores ; core++, shift++) { + if (mcode->core_mask & (1 << shift)) { + cpt_write_csr64(cpt->reg_base, + CPTX_PF_ENGX_UCODE_BASE(0, core), + (u64)mcode->phys_base); + } + } + return ret; +} + +static int do_cpt_init(struct cpt_device *cpt, struct microcode *mcode) +{ + int ret = 0; + struct device *dev = &cpt->pdev->dev; + + /* Make device not ready */ + cpt->flags &= ~CPT_FLAG_DEVICE_READY; + /* Disable All PF interrupts */ + cpt_disable_all_interrupts(cpt); + /* Calculate mcode group and coremasks */ + if (mcode->is_ae) { + if (mcode->num_cores > cpt->max_ae_cores) { + dev_err(dev, "Requested for more cores than available AE cores\n"); + ret = -EINVAL; + goto cpt_init_fail; + } + + if (cpt->next_group >= CPT_MAX_CORE_GROUPS) { + dev_err(dev, "Can't load, all eight microcode groups in use"); + return -ENFILE; + } + + mcode->group = cpt->next_group; + /* Convert requested cores to mask */ + mcode->core_mask = GENMASK(mcode->num_cores, 0); + cpt_disable_cores(cpt, mcode->core_mask, AE_TYPES, + mcode->group); + /* Load microcode for AE engines */ + ret = cpt_load_microcode(cpt, mcode); + if (ret) { + dev_err(dev, "Microcode load Failed for %s\n", + mcode->version); + goto cpt_init_fail; + } + cpt->next_group++; + /* Configure group mask for the mcode */ + cpt_configure_group(cpt, mcode->group, mcode->core_mask, + AE_TYPES); + /* Enable AE cores for the group mask */ + cpt_enable_cores(cpt, mcode->core_mask, AE_TYPES); + } else { + if (mcode->num_cores > cpt->max_se_cores) { + dev_err(dev, "Requested for more cores than available SE cores\n"); + ret = -EINVAL; + goto cpt_init_fail; + } + if (cpt->next_group >= CPT_MAX_CORE_GROUPS) { + dev_err(dev, "Can't load, all eight microcode groups in use"); + return -ENFILE; + } + + mcode->group = cpt->next_group; + /* Covert requested cores to mask */ + mcode->core_mask = GENMASK(mcode->num_cores, 0); + cpt_disable_cores(cpt, mcode->core_mask, SE_TYPES, + mcode->group); + /* Load microcode for SE engines */ + ret = cpt_load_microcode(cpt, mcode); + if (ret) { + dev_err(dev, "Microcode load Failed for %s\n", + mcode->version); + goto cpt_init_fail; + } + cpt->next_group++; + /* Configure group mask for the mcode */ + cpt_configure_group(cpt, mcode->group, mcode->core_mask, + SE_TYPES); + /* Enable SE cores for the group mask */ + cpt_enable_cores(cpt, mcode->core_mask, SE_TYPES); + } + + /* Enabled PF mailbox interrupts */ + cpt_enable_mbox_interrupts(cpt); + cpt->flags |= CPT_FLAG_DEVICE_READY; + + return ret; + +cpt_init_fail: + /* Enabled PF mailbox interrupts */ + cpt_enable_mbox_interrupts(cpt); + + return ret; +} + +struct ucode_header { + u8 version[CPT_UCODE_VERSION_SZ]; + __be32 code_length; + u32 data_length; + u64 sram_address; +}; + +static int cpt_ucode_load_fw(struct cpt_device *cpt, const u8 *fw, bool is_ae) +{ + const struct firmware *fw_entry; + struct device *dev = &cpt->pdev->dev; + struct ucode_header *ucode; + unsigned int code_length; + struct microcode *mcode; + int j, ret = 0; + + ret = request_firmware(&fw_entry, fw, dev); + if (ret) + return ret; + + ucode = (struct ucode_header *)fw_entry->data; + mcode = &cpt->mcode[cpt->next_mc_idx]; + memcpy(mcode->version, (u8 *)fw_entry->data, CPT_UCODE_VERSION_SZ); + code_length = ntohl(ucode->code_length); + if (code_length == 0 || code_length >= INT_MAX / 2) { + ret = -EINVAL; + goto fw_release; + } + mcode->code_size = code_length * 2; + + mcode->is_ae = is_ae; + mcode->core_mask = 0ULL; + mcode->num_cores = is_ae ? 6 : 10; + + /* Allocate DMAable space */ + mcode->code = dma_alloc_coherent(&cpt->pdev->dev, mcode->code_size, + &mcode->phys_base, GFP_KERNEL); + if (!mcode->code) { + dev_err(dev, "Unable to allocate space for microcode"); + ret = -ENOMEM; + goto fw_release; + } + + memcpy((void *)mcode->code, (void *)(fw_entry->data + sizeof(*ucode)), + mcode->code_size); + + /* Byte swap 64-bit */ + for (j = 0; j < (mcode->code_size / 8); j++) + ((__be64 *)mcode->code)[j] = cpu_to_be64(((u64 *)mcode->code)[j]); + /* MC needs 16-bit swap */ + for (j = 0; j < (mcode->code_size / 2); j++) + ((__be16 *)mcode->code)[j] = cpu_to_be16(((u16 *)mcode->code)[j]); + + dev_dbg(dev, "mcode->code_size = %u\n", mcode->code_size); + dev_dbg(dev, "mcode->is_ae = %u\n", mcode->is_ae); + dev_dbg(dev, "mcode->num_cores = %u\n", mcode->num_cores); + dev_dbg(dev, "mcode->code = %llx\n", (u64)mcode->code); + dev_dbg(dev, "mcode->phys_base = %llx\n", mcode->phys_base); + + ret = do_cpt_init(cpt, mcode); + if (ret) { + dev_err(dev, "do_cpt_init failed with ret: %d\n", ret); + goto fw_release; + } + + dev_info(dev, "Microcode Loaded %s\n", mcode->version); + mcode->is_mc_valid = 1; + cpt->next_mc_idx++; + +fw_release: + release_firmware(fw_entry); + + return ret; +} + +static int cpt_ucode_load(struct cpt_device *cpt) +{ + int ret = 0; + struct device *dev = &cpt->pdev->dev; + + ret = cpt_ucode_load_fw(cpt, "cpt8x-mc-ae.out", true); + if (ret) { + dev_err(dev, "ae:cpt_ucode_load failed with ret: %d\n", ret); + return ret; + } + ret = cpt_ucode_load_fw(cpt, "cpt8x-mc-se.out", false); + if (ret) { + dev_err(dev, "se:cpt_ucode_load failed with ret: %d\n", ret); + return ret; + } + + return ret; +} + +static irqreturn_t cpt_mbx0_intr_handler(int irq, void *cpt_irq) +{ + struct cpt_device *cpt = (struct cpt_device *)cpt_irq; + + cpt_mbox_intr_handler(cpt, 0); + + return IRQ_HANDLED; +} + +static void cpt_reset(struct cpt_device *cpt) +{ + cpt_write_csr64(cpt->reg_base, CPTX_PF_RESET(0), 1); +} + +static void cpt_find_max_enabled_cores(struct cpt_device *cpt) +{ + union cptx_pf_constants pf_cnsts = {0}; + + pf_cnsts.u = cpt_read_csr64(cpt->reg_base, CPTX_PF_CONSTANTS(0)); + cpt->max_se_cores = pf_cnsts.s.se; + cpt->max_ae_cores = pf_cnsts.s.ae; +} + +static u32 cpt_check_bist_status(struct cpt_device *cpt) +{ + union cptx_pf_bist_status bist_sts = {0}; + + bist_sts.u = cpt_read_csr64(cpt->reg_base, + CPTX_PF_BIST_STATUS(0)); + + return bist_sts.u; +} + +static u64 cpt_check_exe_bist_status(struct cpt_device *cpt) +{ + union cptx_pf_exe_bist_status bist_sts = {0}; + + bist_sts.u = cpt_read_csr64(cpt->reg_base, + CPTX_PF_EXE_BIST_STATUS(0)); + + return bist_sts.u; +} + +static void cpt_disable_all_cores(struct cpt_device *cpt) +{ + u32 grp, timeout = 100; + struct device *dev = &cpt->pdev->dev; + + /* Disengage the cores from groups */ + for (grp = 0; grp < CPT_MAX_CORE_GROUPS; grp++) { + cpt_write_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp), 0); + udelay(CSR_DELAY); + } + + grp = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXEC_BUSY(0)); + while (grp) { + dev_err(dev, "Cores still busy"); + grp = cpt_read_csr64(cpt->reg_base, + CPTX_PF_EXEC_BUSY(0)); + if (timeout--) + break; + + udelay(CSR_DELAY); + } + /* Disable the cores */ + cpt_write_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0), 0); +} + +/* + * Ensure all cores are disengaged from all groups by + * calling cpt_disable_all_cores() before calling this + * function. + */ +static void cpt_unload_microcode(struct cpt_device *cpt) +{ + u32 grp = 0, core; + + /* Free microcode bases and reset group masks */ + for (grp = 0; grp < CPT_MAX_CORE_GROUPS; grp++) { + struct microcode *mcode = &cpt->mcode[grp]; + + if (cpt->mcode[grp].code) + dma_free_coherent(&cpt->pdev->dev, mcode->code_size, + mcode->code, mcode->phys_base); + mcode->code = NULL; + } + /* Clear UCODE_BASE registers for all engines */ + for (core = 0; core < CPT_MAX_TOTAL_CORES; core++) + cpt_write_csr64(cpt->reg_base, + CPTX_PF_ENGX_UCODE_BASE(0, core), 0ull); +} + +static int cpt_device_init(struct cpt_device *cpt) +{ + u64 bist; + struct device *dev = &cpt->pdev->dev; + + /* Reset the PF when probed first */ + cpt_reset(cpt); + msleep(100); + + /*Check BIST status*/ + bist = (u64)cpt_check_bist_status(cpt); + if (bist) { + dev_err(dev, "RAM BIST failed with code 0x%llx", bist); + return -ENODEV; + } + + bist = cpt_check_exe_bist_status(cpt); + if (bist) { + dev_err(dev, "Engine BIST failed with code 0x%llx", bist); + return -ENODEV; + } + + /*Get CLK frequency*/ + /*Get max enabled cores */ + cpt_find_max_enabled_cores(cpt); + /*Disable all cores*/ + cpt_disable_all_cores(cpt); + /*Reset device parameters*/ + cpt->next_mc_idx = 0; + cpt->next_group = 0; + /* PF is ready */ + cpt->flags |= CPT_FLAG_DEVICE_READY; + + return 0; +} + +static int cpt_register_interrupts(struct cpt_device *cpt) +{ + int ret; + struct device *dev = &cpt->pdev->dev; + + /* Enable MSI-X */ + ret = pci_alloc_irq_vectors(cpt->pdev, CPT_PF_MSIX_VECTORS, + CPT_PF_MSIX_VECTORS, PCI_IRQ_MSIX); + if (ret < 0) { + dev_err(&cpt->pdev->dev, "Request for #%d msix vectors failed\n", + CPT_PF_MSIX_VECTORS); + return ret; + } + + /* Register mailbox interrupt handlers */ + ret = request_irq(pci_irq_vector(cpt->pdev, CPT_PF_INT_VEC_E_MBOXX(0)), + cpt_mbx0_intr_handler, 0, "CPT Mbox0", cpt); + if (ret) + goto fail; + + /* Enable mailbox interrupt */ + cpt_enable_mbox_interrupts(cpt); + return 0; + +fail: + dev_err(dev, "Request irq failed\n"); + pci_disable_msix(cpt->pdev); + return ret; +} + +static void cpt_unregister_interrupts(struct cpt_device *cpt) +{ + free_irq(pci_irq_vector(cpt->pdev, CPT_PF_INT_VEC_E_MBOXX(0)), cpt); + pci_disable_msix(cpt->pdev); +} + +static int cpt_sriov_init(struct cpt_device *cpt, int num_vfs) +{ + int pos = 0; + int err; + u16 total_vf_cnt; + struct pci_dev *pdev = cpt->pdev; + + pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV); + if (!pos) { + dev_err(&pdev->dev, "SRIOV capability is not found in PCIe config space\n"); + return -ENODEV; + } + + cpt->num_vf_en = num_vfs; /* User requested VFs */ + pci_read_config_word(pdev, (pos + PCI_SRIOV_TOTAL_VF), &total_vf_cnt); + if (total_vf_cnt < cpt->num_vf_en) + cpt->num_vf_en = total_vf_cnt; + + if (!total_vf_cnt) + return 0; + + /*Enabled the available VFs */ + err = pci_enable_sriov(pdev, cpt->num_vf_en); + if (err) { + dev_err(&pdev->dev, "SRIOV enable failed, num VF is %d\n", + cpt->num_vf_en); + cpt->num_vf_en = 0; + return err; + } + + /* TODO: Optionally enable static VQ priorities feature */ + + dev_info(&pdev->dev, "SRIOV enabled, number of VF available %d\n", + cpt->num_vf_en); + + cpt->flags |= CPT_FLAG_SRIOV_ENABLED; + + return 0; +} + +static int cpt_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct device *dev = &pdev->dev; + struct cpt_device *cpt; + int err; + + if (num_vfs > 16 || num_vfs < 4) { + dev_warn(dev, "Invalid vf count %d, Resetting it to 4(default)\n", + num_vfs); + num_vfs = 4; + } + + cpt = devm_kzalloc(dev, sizeof(*cpt), GFP_KERNEL); + if (!cpt) + return -ENOMEM; + + pci_set_drvdata(pdev, cpt); + cpt->pdev = pdev; + err = pci_enable_device(pdev); + if (err) { + dev_err(dev, "Failed to enable PCI device\n"); + pci_set_drvdata(pdev, NULL); + return err; + } + + err = pci_request_regions(pdev, DRV_NAME); + if (err) { + dev_err(dev, "PCI request regions failed 0x%x\n", err); + goto cpt_err_disable_device; + } + + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); + if (err) { + dev_err(dev, "Unable to get usable 48-bit DMA configuration\n"); + goto cpt_err_release_regions; + } + + /* MAP PF's configuration registers */ + cpt->reg_base = pcim_iomap(pdev, 0, 0); + if (!cpt->reg_base) { + dev_err(dev, "Cannot map config register space, aborting\n"); + err = -ENOMEM; + goto cpt_err_release_regions; + } + + /* CPT device HW initialization */ + cpt_device_init(cpt); + + /* Register interrupts */ + err = cpt_register_interrupts(cpt); + if (err) + goto cpt_err_release_regions; + + err = cpt_ucode_load(cpt); + if (err) + goto cpt_err_unregister_interrupts; + + /* Configure SRIOV */ + err = cpt_sriov_init(cpt, num_vfs); + if (err) + goto cpt_err_unregister_interrupts; + + return 0; + +cpt_err_unregister_interrupts: + cpt_unregister_interrupts(cpt); +cpt_err_release_regions: + pci_release_regions(pdev); +cpt_err_disable_device: + pci_disable_device(pdev); + pci_set_drvdata(pdev, NULL); + return err; +} + +static void cpt_remove(struct pci_dev *pdev) +{ + struct cpt_device *cpt = pci_get_drvdata(pdev); + + /* Disengage SE and AE cores from all groups*/ + cpt_disable_all_cores(cpt); + /* Unload microcodes */ + cpt_unload_microcode(cpt); + cpt_unregister_interrupts(cpt); + pci_disable_sriov(pdev); + pci_release_regions(pdev); + pci_disable_device(pdev); + pci_set_drvdata(pdev, NULL); +} + +static void cpt_shutdown(struct pci_dev *pdev) +{ + struct cpt_device *cpt = pci_get_drvdata(pdev); + + if (!cpt) + return; + + dev_info(&pdev->dev, "Shutdown device %x:%x.\n", + (u32)pdev->vendor, (u32)pdev->device); + + cpt_unregister_interrupts(cpt); + pci_release_regions(pdev); + pci_disable_device(pdev); + pci_set_drvdata(pdev, NULL); +} + +/* Supported devices */ +static const struct pci_device_id cpt_id_table[] = { + { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, CPT_81XX_PCI_PF_DEVICE_ID) }, + { 0, } /* end of table */ +}; + +static struct pci_driver cpt_pci_driver = { + .name = DRV_NAME, + .id_table = cpt_id_table, + .probe = cpt_probe, + .remove = cpt_remove, + .shutdown = cpt_shutdown, +}; + +module_pci_driver(cpt_pci_driver); + +MODULE_AUTHOR("George Cherian <george.cherian@cavium.com>"); +MODULE_DESCRIPTION("Cavium Thunder CPT Physical Function Driver"); +MODULE_LICENSE("GPL v2"); +MODULE_VERSION(DRV_VERSION); +MODULE_DEVICE_TABLE(pci, cpt_id_table); diff --git a/drivers/crypto/cavium/cpt/cptpf_mbox.c b/drivers/crypto/cavium/cpt/cptpf_mbox.c new file mode 100644 index 000000000..f01b863d6 --- /dev/null +++ b/drivers/crypto/cavium/cpt/cptpf_mbox.c @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2016 Cavium, Inc. + */ +#include <linux/module.h> +#include "cptpf.h" + +static void cpt_send_msg_to_vf(struct cpt_device *cpt, int vf, + struct cpt_mbox *mbx) +{ + /* Writing mbox(0) causes interrupt */ + cpt_write_csr64(cpt->reg_base, CPTX_PF_VFX_MBOXX(0, vf, 1), + mbx->data); + cpt_write_csr64(cpt->reg_base, CPTX_PF_VFX_MBOXX(0, vf, 0), mbx->msg); +} + +/* ACKs VF's mailbox message + * @vf: VF to which ACK to be sent + */ +static void cpt_mbox_send_ack(struct cpt_device *cpt, int vf, + struct cpt_mbox *mbx) +{ + mbx->data = 0ull; + mbx->msg = CPT_MBOX_MSG_TYPE_ACK; + cpt_send_msg_to_vf(cpt, vf, mbx); +} + +static void cpt_clear_mbox_intr(struct cpt_device *cpt, u32 vf) +{ + /* W1C for the VF */ + cpt_write_csr64(cpt->reg_base, CPTX_PF_MBOX_INTX(0, 0), (1 << vf)); +} + +/* + * Configure QLEN/Chunk sizes for VF + */ +static void cpt_cfg_qlen_for_vf(struct cpt_device *cpt, int vf, u32 size) +{ + union cptx_pf_qx_ctl pf_qx_ctl; + + pf_qx_ctl.u = cpt_read_csr64(cpt->reg_base, CPTX_PF_QX_CTL(0, vf)); + pf_qx_ctl.s.size = size; + pf_qx_ctl.s.cont_err = true; + cpt_write_csr64(cpt->reg_base, CPTX_PF_QX_CTL(0, vf), pf_qx_ctl.u); +} + +/* + * Configure VQ priority + */ +static void cpt_cfg_vq_priority(struct cpt_device *cpt, int vf, u32 pri) +{ + union cptx_pf_qx_ctl pf_qx_ctl; + + pf_qx_ctl.u = cpt_read_csr64(cpt->reg_base, CPTX_PF_QX_CTL(0, vf)); + pf_qx_ctl.s.pri = pri; + cpt_write_csr64(cpt->reg_base, CPTX_PF_QX_CTL(0, vf), pf_qx_ctl.u); +} + +static int cpt_bind_vq_to_grp(struct cpt_device *cpt, u8 q, u8 grp) +{ + struct microcode *mcode = cpt->mcode; + union cptx_pf_qx_ctl pf_qx_ctl; + struct device *dev = &cpt->pdev->dev; + + if (q >= CPT_MAX_VF_NUM) { + dev_err(dev, "Queues are more than cores in the group"); + return -EINVAL; + } + if (grp >= CPT_MAX_CORE_GROUPS) { + dev_err(dev, "Request group is more than possible groups"); + return -EINVAL; + } + if (grp >= cpt->next_mc_idx) { + dev_err(dev, "Request group is higher than available functional groups"); + return -EINVAL; + } + pf_qx_ctl.u = cpt_read_csr64(cpt->reg_base, CPTX_PF_QX_CTL(0, q)); + pf_qx_ctl.s.grp = mcode[grp].group; + cpt_write_csr64(cpt->reg_base, CPTX_PF_QX_CTL(0, q), pf_qx_ctl.u); + dev_dbg(dev, "VF %d TYPE %s", q, (mcode[grp].is_ae ? "AE" : "SE")); + + return mcode[grp].is_ae ? AE_TYPES : SE_TYPES; +} + +/* Interrupt handler to handle mailbox messages from VFs */ +static void cpt_handle_mbox_intr(struct cpt_device *cpt, int vf) +{ + struct cpt_vf_info *vfx = &cpt->vfinfo[vf]; + struct cpt_mbox mbx = {}; + int vftype; + struct device *dev = &cpt->pdev->dev; + /* + * MBOX[0] contains msg + * MBOX[1] contains data + */ + mbx.msg = cpt_read_csr64(cpt->reg_base, CPTX_PF_VFX_MBOXX(0, vf, 0)); + mbx.data = cpt_read_csr64(cpt->reg_base, CPTX_PF_VFX_MBOXX(0, vf, 1)); + dev_dbg(dev, "%s: Mailbox msg 0x%llx from VF%d", __func__, mbx.msg, vf); + switch (mbx.msg) { + case CPT_MSG_VF_UP: + vfx->state = VF_STATE_UP; + try_module_get(THIS_MODULE); + cpt_mbox_send_ack(cpt, vf, &mbx); + break; + case CPT_MSG_READY: + mbx.msg = CPT_MSG_READY; + mbx.data = vf; + cpt_send_msg_to_vf(cpt, vf, &mbx); + break; + case CPT_MSG_VF_DOWN: + /* First msg in VF teardown sequence */ + vfx->state = VF_STATE_DOWN; + module_put(THIS_MODULE); + cpt_mbox_send_ack(cpt, vf, &mbx); + break; + case CPT_MSG_QLEN: + vfx->qlen = mbx.data; + cpt_cfg_qlen_for_vf(cpt, vf, vfx->qlen); + cpt_mbox_send_ack(cpt, vf, &mbx); + break; + case CPT_MSG_QBIND_GRP: + vftype = cpt_bind_vq_to_grp(cpt, vf, (u8)mbx.data); + if ((vftype != AE_TYPES) && (vftype != SE_TYPES)) + dev_err(dev, "Queue %d binding to group %llu failed", + vf, mbx.data); + else { + dev_dbg(dev, "Queue %d binding to group %llu successful", + vf, mbx.data); + mbx.msg = CPT_MSG_QBIND_GRP; + mbx.data = vftype; + cpt_send_msg_to_vf(cpt, vf, &mbx); + } + break; + case CPT_MSG_VQ_PRIORITY: + vfx->priority = mbx.data; + cpt_cfg_vq_priority(cpt, vf, vfx->priority); + cpt_mbox_send_ack(cpt, vf, &mbx); + break; + default: + dev_err(&cpt->pdev->dev, "Invalid msg from VF%d, msg 0x%llx\n", + vf, mbx.msg); + break; + } +} + +void cpt_mbox_intr_handler (struct cpt_device *cpt, int mbx) +{ + u64 intr; + u8 vf; + + intr = cpt_read_csr64(cpt->reg_base, CPTX_PF_MBOX_INTX(0, 0)); + dev_dbg(&cpt->pdev->dev, "PF interrupt Mbox%d 0x%llx\n", mbx, intr); + for (vf = 0; vf < CPT_MAX_VF_NUM; vf++) { + if (intr & (1ULL << vf)) { + dev_dbg(&cpt->pdev->dev, "Intr from VF %d\n", vf); + cpt_handle_mbox_intr(cpt, vf); + cpt_clear_mbox_intr(cpt, vf); + } + } +} diff --git a/drivers/crypto/cavium/cpt/cptvf.h b/drivers/crypto/cavium/cpt/cptvf.h new file mode 100644 index 000000000..c695676ad --- /dev/null +++ b/drivers/crypto/cavium/cpt/cptvf.h @@ -0,0 +1,129 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2016 Cavium, Inc. + */ + +#ifndef __CPTVF_H +#define __CPTVF_H + +#include <linux/list.h> +#include "cpt_common.h" + +/* Default command queue length */ +#define CPT_CMD_QLEN 2046 +#define CPT_CMD_QCHUNK_SIZE 1023 + +/* Default command timeout in seconds */ +#define CPT_COMMAND_TIMEOUT 4 +#define CPT_TIMER_THOLD 0xFFFF +#define CPT_NUM_QS_PER_VF 1 +#define CPT_INST_SIZE 64 +#define CPT_NEXT_CHUNK_PTR_SIZE 8 + +#define CPT_VF_MSIX_VECTORS 2 +#define CPT_VF_INTR_MBOX_MASK BIT(0) +#define CPT_VF_INTR_DOVF_MASK BIT(1) +#define CPT_VF_INTR_IRDE_MASK BIT(2) +#define CPT_VF_INTR_NWRP_MASK BIT(3) +#define CPT_VF_INTR_SERR_MASK BIT(4) +#define DMA_DIRECT_DIRECT 0 /* Input DIRECT, Output DIRECT */ +#define DMA_GATHER_SCATTER 1 +#define FROM_DPTR 1 + +/** + * Enumeration cpt_vf_int_vec_e + * + * CPT VF MSI-X Vector Enumeration + * Enumerates the MSI-X interrupt vectors. + */ +enum cpt_vf_int_vec_e { + CPT_VF_INT_VEC_E_MISC = 0x00, + CPT_VF_INT_VEC_E_DONE = 0x01 +}; + +struct command_chunk { + u8 *head; + dma_addr_t dma_addr; + u32 size; /* Chunk size, max CPT_INST_CHUNK_MAX_SIZE */ + struct hlist_node nextchunk; +}; + +struct command_queue { + spinlock_t lock; /* command queue lock */ + u32 idx; /* Command queue host write idx */ + u32 nchunks; /* Number of command chunks */ + struct command_chunk *qhead; /* Command queue head, instructions + * are inserted here + */ + struct hlist_head chead; +}; + +struct command_qinfo { + u32 cmd_size; + u32 qchunksize; /* Command queue chunk size */ + struct command_queue queue[CPT_NUM_QS_PER_VF]; +}; + +struct pending_entry { + u8 busy; /* Entry status (free/busy) */ + + volatile u64 *completion_addr; /* Completion address */ + void *post_arg; + void (*callback)(int, void *); /* Kernel ASYNC request callabck */ + void *callback_arg; /* Kernel ASYNC request callabck arg */ +}; + +struct pending_queue { + struct pending_entry *head; /* head of the queue */ + u32 front; /* Process work from here */ + u32 rear; /* Append new work here */ + atomic64_t pending_count; + spinlock_t lock; /* Queue lock */ +}; + +struct pending_qinfo { + u32 nr_queues; /* Number of queues supported */ + u32 qlen; /* Queue length */ + struct pending_queue queue[CPT_NUM_QS_PER_VF]; +}; + +#define for_each_pending_queue(qinfo, q, i) \ + for (i = 0, q = &qinfo->queue[i]; i < qinfo->nr_queues; i++, \ + q = &qinfo->queue[i]) + +struct cpt_vf { + u16 flags; /* Flags to hold device status bits */ + u8 vfid; /* Device Index 0...CPT_MAX_VF_NUM */ + u8 vftype; /* VF type of SE_TYPE(1) or AE_TYPE(1) */ + u8 vfgrp; /* VF group (0 - 8) */ + u8 node; /* Operating node: Bits (46:44) in BAR0 address */ + u8 priority; /* VF priority ring: 1-High proirity round + * robin ring;0-Low priority round robin ring; + */ + struct pci_dev *pdev; /* pci device handle */ + void __iomem *reg_base; /* Register start address */ + void *wqe_info; /* BH worker info */ + /* MSI-X */ + cpumask_var_t affinity_mask[CPT_VF_MSIX_VECTORS]; + /* Command and Pending queues */ + u32 qsize; + u32 nr_queues; + struct command_qinfo cqinfo; /* Command queue information */ + struct pending_qinfo pqinfo; /* Pending queue information */ + /* VF-PF mailbox communication */ + bool pf_acked; + bool pf_nacked; +}; + +int cptvf_send_vf_up(struct cpt_vf *cptvf); +int cptvf_send_vf_down(struct cpt_vf *cptvf); +int cptvf_send_vf_to_grp_msg(struct cpt_vf *cptvf); +int cptvf_send_vf_priority_msg(struct cpt_vf *cptvf); +int cptvf_send_vq_size_msg(struct cpt_vf *cptvf); +int cptvf_check_pf_ready(struct cpt_vf *cptvf); +void cptvf_handle_mbox_intr(struct cpt_vf *cptvf); +void cvm_crypto_exit(void); +int cvm_crypto_init(struct cpt_vf *cptvf); +void vq_post_process(struct cpt_vf *cptvf, u32 qno); +void cptvf_write_vq_doorbell(struct cpt_vf *cptvf, u32 val); +#endif /* __CPTVF_H */ diff --git a/drivers/crypto/cavium/cpt/cptvf_algs.c b/drivers/crypto/cavium/cpt/cptvf_algs.c new file mode 100644 index 000000000..ce3b91c61 --- /dev/null +++ b/drivers/crypto/cavium/cpt/cptvf_algs.c @@ -0,0 +1,488 @@ +// SPDX-License-Identifier: GPL-2.0-only + +/* + * Copyright (C) 2016 Cavium, Inc. + */ + +#include <crypto/aes.h> +#include <crypto/algapi.h> +#include <crypto/authenc.h> +#include <crypto/internal/des.h> +#include <crypto/xts.h> +#include <linux/crypto.h> +#include <linux/err.h> +#include <linux/list.h> +#include <linux/scatterlist.h> + +#include "cptvf.h" +#include "cptvf_algs.h" + +struct cpt_device_handle { + void *cdev[MAX_DEVICES]; + u32 dev_count; +}; + +static struct cpt_device_handle dev_handle; + +static void cvm_callback(u32 status, void *arg) +{ + struct crypto_async_request *req = (struct crypto_async_request *)arg; + + req->complete(req, !status); +} + +static inline void update_input_iv(struct cpt_request_info *req_info, + u8 *iv, u32 enc_iv_len, + u32 *argcnt) +{ + /* Setting the iv information */ + req_info->in[*argcnt].vptr = (void *)iv; + req_info->in[*argcnt].size = enc_iv_len; + req_info->req.dlen += enc_iv_len; + + ++(*argcnt); +} + +static inline void update_output_iv(struct cpt_request_info *req_info, + u8 *iv, u32 enc_iv_len, + u32 *argcnt) +{ + /* Setting the iv information */ + req_info->out[*argcnt].vptr = (void *)iv; + req_info->out[*argcnt].size = enc_iv_len; + req_info->rlen += enc_iv_len; + + ++(*argcnt); +} + +static inline void update_input_data(struct cpt_request_info *req_info, + struct scatterlist *inp_sg, + u32 nbytes, u32 *argcnt) +{ + req_info->req.dlen += nbytes; + + while (nbytes) { + u32 len = min(nbytes, inp_sg->length); + u8 *ptr = sg_virt(inp_sg); + + req_info->in[*argcnt].vptr = (void *)ptr; + req_info->in[*argcnt].size = len; + nbytes -= len; + + ++(*argcnt); + ++inp_sg; + } +} + +static inline void update_output_data(struct cpt_request_info *req_info, + struct scatterlist *outp_sg, + u32 nbytes, u32 *argcnt) +{ + req_info->rlen += nbytes; + + while (nbytes) { + u32 len = min(nbytes, outp_sg->length); + u8 *ptr = sg_virt(outp_sg); + + req_info->out[*argcnt].vptr = (void *)ptr; + req_info->out[*argcnt].size = len; + nbytes -= len; + ++(*argcnt); + ++outp_sg; + } +} + +static inline u32 create_ctx_hdr(struct skcipher_request *req, u32 enc, + u32 *argcnt) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct cvm_enc_ctx *ctx = crypto_skcipher_ctx(tfm); + struct cvm_req_ctx *rctx = skcipher_request_ctx(req); + struct fc_context *fctx = &rctx->fctx; + u32 enc_iv_len = crypto_skcipher_ivsize(tfm); + struct cpt_request_info *req_info = &rctx->cpt_req; + __be64 *ctrl_flags = NULL; + __be64 *offset_control; + + req_info->ctrl.s.grp = 0; + req_info->ctrl.s.dma_mode = DMA_GATHER_SCATTER; + req_info->ctrl.s.se_req = SE_CORE_REQ; + + req_info->req.opcode.s.major = MAJOR_OP_FC | + DMA_MODE_FLAG(DMA_GATHER_SCATTER); + if (enc) + req_info->req.opcode.s.minor = 2; + else + req_info->req.opcode.s.minor = 3; + + req_info->req.param1 = req->cryptlen; /* Encryption Data length */ + req_info->req.param2 = 0; /*Auth data length */ + + fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type; + fctx->enc.enc_ctrl.e.aes_key = ctx->key_type; + fctx->enc.enc_ctrl.e.iv_source = FROM_DPTR; + + if (ctx->cipher_type == AES_XTS) + memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2); + else + memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len); + ctrl_flags = (__be64 *)&fctx->enc.enc_ctrl.flags; + *ctrl_flags = cpu_to_be64(fctx->enc.enc_ctrl.flags); + + offset_control = (__be64 *)&rctx->control_word; + *offset_control = cpu_to_be64(((u64)(enc_iv_len) << 16)); + /* Storing Packet Data Information in offset + * Control Word First 8 bytes + */ + req_info->in[*argcnt].vptr = (u8 *)offset_control; + req_info->in[*argcnt].size = CONTROL_WORD_LEN; + req_info->req.dlen += CONTROL_WORD_LEN; + ++(*argcnt); + + req_info->in[*argcnt].vptr = (u8 *)fctx; + req_info->in[*argcnt].size = sizeof(struct fc_context); + req_info->req.dlen += sizeof(struct fc_context); + + ++(*argcnt); + + return 0; +} + +static inline u32 create_input_list(struct skcipher_request *req, u32 enc, + u32 enc_iv_len) +{ + struct cvm_req_ctx *rctx = skcipher_request_ctx(req); + struct cpt_request_info *req_info = &rctx->cpt_req; + u32 argcnt = 0; + + create_ctx_hdr(req, enc, &argcnt); + update_input_iv(req_info, req->iv, enc_iv_len, &argcnt); + update_input_data(req_info, req->src, req->cryptlen, &argcnt); + req_info->incnt = argcnt; + + return 0; +} + +static inline void store_cb_info(struct skcipher_request *req, + struct cpt_request_info *req_info) +{ + req_info->callback = (void *)cvm_callback; + req_info->callback_arg = (void *)&req->base; +} + +static inline void create_output_list(struct skcipher_request *req, + u32 enc_iv_len) +{ + struct cvm_req_ctx *rctx = skcipher_request_ctx(req); + struct cpt_request_info *req_info = &rctx->cpt_req; + u32 argcnt = 0; + + /* OUTPUT Buffer Processing + * AES encryption/decryption output would be + * received in the following format + * + * ------IV--------|------ENCRYPTED/DECRYPTED DATA-----| + * [ 16 Bytes/ [ Request Enc/Dec/ DATA Len AES CBC ] + */ + /* Reading IV information */ + update_output_iv(req_info, req->iv, enc_iv_len, &argcnt); + update_output_data(req_info, req->dst, req->cryptlen, &argcnt); + req_info->outcnt = argcnt; +} + +static inline int cvm_enc_dec(struct skcipher_request *req, u32 enc) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct cvm_req_ctx *rctx = skcipher_request_ctx(req); + u32 enc_iv_len = crypto_skcipher_ivsize(tfm); + struct fc_context *fctx = &rctx->fctx; + struct cpt_request_info *req_info = &rctx->cpt_req; + void *cdev = NULL; + int status; + + memset(req_info, 0, sizeof(struct cpt_request_info)); + req_info->may_sleep = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) != 0; + memset(fctx, 0, sizeof(struct fc_context)); + create_input_list(req, enc, enc_iv_len); + create_output_list(req, enc_iv_len); + store_cb_info(req, req_info); + cdev = dev_handle.cdev[smp_processor_id()]; + status = cptvf_do_request(cdev, req_info); + /* We perform an asynchronous send and once + * the request is completed the driver would + * intimate through registered call back functions + */ + + if (status) + return status; + else + return -EINPROGRESS; +} + +static int cvm_encrypt(struct skcipher_request *req) +{ + return cvm_enc_dec(req, true); +} + +static int cvm_decrypt(struct skcipher_request *req) +{ + return cvm_enc_dec(req, false); +} + +static int cvm_xts_setkey(struct crypto_skcipher *cipher, const u8 *key, + u32 keylen) +{ + struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); + struct cvm_enc_ctx *ctx = crypto_tfm_ctx(tfm); + int err; + const u8 *key1 = key; + const u8 *key2 = key + (keylen / 2); + + err = xts_check_key(tfm, key, keylen); + if (err) + return err; + ctx->key_len = keylen; + memcpy(ctx->enc_key, key1, keylen / 2); + memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2); + ctx->cipher_type = AES_XTS; + switch (ctx->key_len) { + case 32: + ctx->key_type = AES_128_BIT; + break; + case 64: + ctx->key_type = AES_256_BIT; + break; + default: + return -EINVAL; + } + + return 0; +} + +static int cvm_validate_keylen(struct cvm_enc_ctx *ctx, u32 keylen) +{ + if ((keylen == 16) || (keylen == 24) || (keylen == 32)) { + ctx->key_len = keylen; + switch (ctx->key_len) { + case 16: + ctx->key_type = AES_128_BIT; + break; + case 24: + ctx->key_type = AES_192_BIT; + break; + case 32: + ctx->key_type = AES_256_BIT; + break; + default: + return -EINVAL; + } + + if (ctx->cipher_type == DES3_CBC) + ctx->key_type = 0; + + return 0; + } + + return -EINVAL; +} + +static int cvm_setkey(struct crypto_skcipher *cipher, const u8 *key, + u32 keylen, u8 cipher_type) +{ + struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); + struct cvm_enc_ctx *ctx = crypto_tfm_ctx(tfm); + + ctx->cipher_type = cipher_type; + if (!cvm_validate_keylen(ctx, keylen)) { + memcpy(ctx->enc_key, key, keylen); + return 0; + } else { + return -EINVAL; + } +} + +static int cvm_cbc_aes_setkey(struct crypto_skcipher *cipher, const u8 *key, + u32 keylen) +{ + return cvm_setkey(cipher, key, keylen, AES_CBC); +} + +static int cvm_ecb_aes_setkey(struct crypto_skcipher *cipher, const u8 *key, + u32 keylen) +{ + return cvm_setkey(cipher, key, keylen, AES_ECB); +} + +static int cvm_cfb_aes_setkey(struct crypto_skcipher *cipher, const u8 *key, + u32 keylen) +{ + return cvm_setkey(cipher, key, keylen, AES_CFB); +} + +static int cvm_cbc_des3_setkey(struct crypto_skcipher *cipher, const u8 *key, + u32 keylen) +{ + return verify_skcipher_des3_key(cipher, key) ?: + cvm_setkey(cipher, key, keylen, DES3_CBC); +} + +static int cvm_ecb_des3_setkey(struct crypto_skcipher *cipher, const u8 *key, + u32 keylen) +{ + return verify_skcipher_des3_key(cipher, key) ?: + cvm_setkey(cipher, key, keylen, DES3_ECB); +} + +static int cvm_enc_dec_init(struct crypto_skcipher *tfm) +{ + crypto_skcipher_set_reqsize(tfm, sizeof(struct cvm_req_ctx)); + + return 0; +} + +static struct skcipher_alg algs[] = { { + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct cvm_enc_ctx), + .base.cra_alignmask = 7, + .base.cra_priority = 4001, + .base.cra_name = "xts(aes)", + .base.cra_driver_name = "cavium-xts-aes", + .base.cra_module = THIS_MODULE, + + .ivsize = AES_BLOCK_SIZE, + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .setkey = cvm_xts_setkey, + .encrypt = cvm_encrypt, + .decrypt = cvm_decrypt, + .init = cvm_enc_dec_init, +}, { + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct cvm_enc_ctx), + .base.cra_alignmask = 7, + .base.cra_priority = 4001, + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "cavium-cbc-aes", + .base.cra_module = THIS_MODULE, + + .ivsize = AES_BLOCK_SIZE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = cvm_cbc_aes_setkey, + .encrypt = cvm_encrypt, + .decrypt = cvm_decrypt, + .init = cvm_enc_dec_init, +}, { + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct cvm_enc_ctx), + .base.cra_alignmask = 7, + .base.cra_priority = 4001, + .base.cra_name = "ecb(aes)", + .base.cra_driver_name = "cavium-ecb-aes", + .base.cra_module = THIS_MODULE, + + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = cvm_ecb_aes_setkey, + .encrypt = cvm_encrypt, + .decrypt = cvm_decrypt, + .init = cvm_enc_dec_init, +}, { + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct cvm_enc_ctx), + .base.cra_alignmask = 7, + .base.cra_priority = 4001, + .base.cra_name = "cfb(aes)", + .base.cra_driver_name = "cavium-cfb-aes", + .base.cra_module = THIS_MODULE, + + .ivsize = AES_BLOCK_SIZE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = cvm_cfb_aes_setkey, + .encrypt = cvm_encrypt, + .decrypt = cvm_decrypt, + .init = cvm_enc_dec_init, +}, { + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct cvm_des3_ctx), + .base.cra_alignmask = 7, + .base.cra_priority = 4001, + .base.cra_name = "cbc(des3_ede)", + .base.cra_driver_name = "cavium-cbc-des3_ede", + .base.cra_module = THIS_MODULE, + + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = cvm_cbc_des3_setkey, + .encrypt = cvm_encrypt, + .decrypt = cvm_decrypt, + .init = cvm_enc_dec_init, +}, { + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct cvm_des3_ctx), + .base.cra_alignmask = 7, + .base.cra_priority = 4001, + .base.cra_name = "ecb(des3_ede)", + .base.cra_driver_name = "cavium-ecb-des3_ede", + .base.cra_module = THIS_MODULE, + + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = cvm_ecb_des3_setkey, + .encrypt = cvm_encrypt, + .decrypt = cvm_decrypt, + .init = cvm_enc_dec_init, +} }; + +static inline int cav_register_algs(void) +{ + return crypto_register_skciphers(algs, ARRAY_SIZE(algs)); +} + +static inline void cav_unregister_algs(void) +{ + crypto_unregister_skciphers(algs, ARRAY_SIZE(algs)); +} + +int cvm_crypto_init(struct cpt_vf *cptvf) +{ + struct pci_dev *pdev = cptvf->pdev; + u32 dev_count; + + dev_count = dev_handle.dev_count; + dev_handle.cdev[dev_count] = cptvf; + dev_handle.dev_count++; + + if (dev_count == 3) { + if (cav_register_algs()) { + dev_err(&pdev->dev, "Error in registering crypto algorithms\n"); + return -EINVAL; + } + } + + return 0; +} + +void cvm_crypto_exit(void) +{ + u32 dev_count; + + dev_count = --dev_handle.dev_count; + if (!dev_count) + cav_unregister_algs(); +} diff --git a/drivers/crypto/cavium/cpt/cptvf_algs.h b/drivers/crypto/cavium/cpt/cptvf_algs.h new file mode 100644 index 000000000..61a149fe8 --- /dev/null +++ b/drivers/crypto/cavium/cpt/cptvf_algs.h @@ -0,0 +1,117 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2016 Cavium, Inc. + */ + +#ifndef _CPTVF_ALGS_H_ +#define _CPTVF_ALGS_H_ + +#include "request_manager.h" + +#define MAX_DEVICES 16 +#define MAJOR_OP_FC 0x33 +#define MAX_ENC_KEY_SIZE 32 +#define MAX_HASH_KEY_SIZE 64 +#define MAX_KEY_SIZE (MAX_ENC_KEY_SIZE + MAX_HASH_KEY_SIZE) +#define CONTROL_WORD_LEN 8 +#define KEY2_OFFSET 48 + +#define DMA_MODE_FLAG(dma_mode) \ + (((dma_mode) == DMA_GATHER_SCATTER) ? (1 << 7) : 0) + +enum req_type { + AE_CORE_REQ, + SE_CORE_REQ, +}; + +enum cipher_type { + DES3_CBC = 0x1, + DES3_ECB = 0x2, + AES_CBC = 0x3, + AES_ECB = 0x4, + AES_CFB = 0x5, + AES_CTR = 0x6, + AES_GCM = 0x7, + AES_XTS = 0x8 +}; + +enum aes_type { + AES_128_BIT = 0x1, + AES_192_BIT = 0x2, + AES_256_BIT = 0x3 +}; + +union encr_ctrl { + u64 flags; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 enc_cipher:4; + u64 reserved1:1; + u64 aes_key:2; + u64 iv_source:1; + u64 hash_type:4; + u64 reserved2:3; + u64 auth_input_type:1; + u64 mac_len:8; + u64 reserved3:8; + u64 encr_offset:16; + u64 iv_offset:8; + u64 auth_offset:8; +#else + u64 auth_offset:8; + u64 iv_offset:8; + u64 encr_offset:16; + u64 reserved3:8; + u64 mac_len:8; + u64 auth_input_type:1; + u64 reserved2:3; + u64 hash_type:4; + u64 iv_source:1; + u64 aes_key:2; + u64 reserved1:1; + u64 enc_cipher:4; +#endif + } e; +}; + +struct cvm_cipher { + const char *name; + u8 value; +}; + +struct enc_context { + union encr_ctrl enc_ctrl; + u8 encr_key[32]; + u8 encr_iv[16]; +}; + +struct fchmac_context { + u8 ipad[64]; + u8 opad[64]; /* or OPAD */ +}; + +struct fc_context { + struct enc_context enc; + struct fchmac_context hmac; +}; + +struct cvm_enc_ctx { + u32 key_len; + u8 enc_key[MAX_KEY_SIZE]; + u8 cipher_type:4; + u8 key_type:2; +}; + +struct cvm_des3_ctx { + u32 key_len; + u8 des3_key[MAX_KEY_SIZE]; +}; + +struct cvm_req_ctx { + struct cpt_request_info cpt_req; + u64 control_word; + struct fc_context fctx; +}; + +int cptvf_do_request(void *cptvf, struct cpt_request_info *req); +#endif /*_CPTVF_ALGS_H_*/ diff --git a/drivers/crypto/cavium/cpt/cptvf_main.c b/drivers/crypto/cavium/cpt/cptvf_main.c new file mode 100644 index 000000000..c246920e6 --- /dev/null +++ b/drivers/crypto/cavium/cpt/cptvf_main.c @@ -0,0 +1,855 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2016 Cavium, Inc. + */ + +#include <linux/interrupt.h> +#include <linux/module.h> + +#include "cptvf.h" + +#define DRV_NAME "thunder-cptvf" +#define DRV_VERSION "1.0" + +struct cptvf_wqe { + struct tasklet_struct twork; + void *cptvf; + u32 qno; +}; + +struct cptvf_wqe_info { + struct cptvf_wqe vq_wqe[CPT_NUM_QS_PER_VF]; +}; + +static void vq_work_handler(unsigned long data) +{ + struct cptvf_wqe_info *cwqe_info = (struct cptvf_wqe_info *)data; + struct cptvf_wqe *cwqe = &cwqe_info->vq_wqe[0]; + + vq_post_process(cwqe->cptvf, cwqe->qno); +} + +static int init_worker_threads(struct cpt_vf *cptvf) +{ + struct pci_dev *pdev = cptvf->pdev; + struct cptvf_wqe_info *cwqe_info; + int i; + + cwqe_info = kzalloc(sizeof(*cwqe_info), GFP_KERNEL); + if (!cwqe_info) + return -ENOMEM; + + if (cptvf->nr_queues) { + dev_info(&pdev->dev, "Creating VQ worker threads (%d)\n", + cptvf->nr_queues); + } + + for (i = 0; i < cptvf->nr_queues; i++) { + tasklet_init(&cwqe_info->vq_wqe[i].twork, vq_work_handler, + (u64)cwqe_info); + cwqe_info->vq_wqe[i].qno = i; + cwqe_info->vq_wqe[i].cptvf = cptvf; + } + + cptvf->wqe_info = cwqe_info; + + return 0; +} + +static void cleanup_worker_threads(struct cpt_vf *cptvf) +{ + struct cptvf_wqe_info *cwqe_info; + struct pci_dev *pdev = cptvf->pdev; + int i; + + cwqe_info = (struct cptvf_wqe_info *)cptvf->wqe_info; + if (!cwqe_info) + return; + + if (cptvf->nr_queues) { + dev_info(&pdev->dev, "Cleaning VQ worker threads (%u)\n", + cptvf->nr_queues); + } + + for (i = 0; i < cptvf->nr_queues; i++) + tasklet_kill(&cwqe_info->vq_wqe[i].twork); + + kfree_sensitive(cwqe_info); + cptvf->wqe_info = NULL; +} + +static void free_pending_queues(struct pending_qinfo *pqinfo) +{ + int i; + struct pending_queue *queue; + + for_each_pending_queue(pqinfo, queue, i) { + if (!queue->head) + continue; + + /* free single queue */ + kfree_sensitive((queue->head)); + + queue->front = 0; + queue->rear = 0; + + return; + } + + pqinfo->qlen = 0; + pqinfo->nr_queues = 0; +} + +static int alloc_pending_queues(struct pending_qinfo *pqinfo, u32 qlen, + u32 nr_queues) +{ + u32 i; + int ret; + struct pending_queue *queue = NULL; + + pqinfo->nr_queues = nr_queues; + pqinfo->qlen = qlen; + + for_each_pending_queue(pqinfo, queue, i) { + queue->head = kcalloc(qlen, sizeof(*queue->head), GFP_KERNEL); + if (!queue->head) { + ret = -ENOMEM; + goto pending_qfail; + } + + queue->front = 0; + queue->rear = 0; + atomic64_set((&queue->pending_count), (0)); + + /* init queue spin lock */ + spin_lock_init(&queue->lock); + } + + return 0; + +pending_qfail: + free_pending_queues(pqinfo); + + return ret; +} + +static int init_pending_queues(struct cpt_vf *cptvf, u32 qlen, u32 nr_queues) +{ + struct pci_dev *pdev = cptvf->pdev; + int ret; + + if (!nr_queues) + return 0; + + ret = alloc_pending_queues(&cptvf->pqinfo, qlen, nr_queues); + if (ret) { + dev_err(&pdev->dev, "failed to setup pending queues (%u)\n", + nr_queues); + return ret; + } + + return 0; +} + +static void cleanup_pending_queues(struct cpt_vf *cptvf) +{ + struct pci_dev *pdev = cptvf->pdev; + + if (!cptvf->nr_queues) + return; + + dev_info(&pdev->dev, "Cleaning VQ pending queue (%u)\n", + cptvf->nr_queues); + free_pending_queues(&cptvf->pqinfo); +} + +static void free_command_queues(struct cpt_vf *cptvf, + struct command_qinfo *cqinfo) +{ + int i; + struct command_queue *queue = NULL; + struct command_chunk *chunk = NULL; + struct pci_dev *pdev = cptvf->pdev; + struct hlist_node *node; + + /* clean up for each queue */ + for (i = 0; i < cptvf->nr_queues; i++) { + queue = &cqinfo->queue[i]; + if (hlist_empty(&cqinfo->queue[i].chead)) + continue; + + hlist_for_each_entry_safe(chunk, node, &cqinfo->queue[i].chead, + nextchunk) { + dma_free_coherent(&pdev->dev, chunk->size, + chunk->head, + chunk->dma_addr); + chunk->head = NULL; + chunk->dma_addr = 0; + hlist_del(&chunk->nextchunk); + kfree_sensitive(chunk); + } + + queue->nchunks = 0; + queue->idx = 0; + } + + /* common cleanup */ + cqinfo->cmd_size = 0; +} + +static int alloc_command_queues(struct cpt_vf *cptvf, + struct command_qinfo *cqinfo, size_t cmd_size, + u32 qlen) +{ + int i; + size_t q_size; + struct command_queue *queue = NULL; + struct pci_dev *pdev = cptvf->pdev; + + /* common init */ + cqinfo->cmd_size = cmd_size; + /* Qsize in dwords, needed for SADDR config, 1-next chunk pointer */ + cptvf->qsize = min(qlen, cqinfo->qchunksize) * + CPT_NEXT_CHUNK_PTR_SIZE + 1; + /* Qsize in bytes to create space for alignment */ + q_size = qlen * cqinfo->cmd_size; + + /* per queue initialization */ + for (i = 0; i < cptvf->nr_queues; i++) { + size_t c_size = 0; + size_t rem_q_size = q_size; + struct command_chunk *curr = NULL, *first = NULL, *last = NULL; + u32 qcsize_bytes = cqinfo->qchunksize * cqinfo->cmd_size; + + queue = &cqinfo->queue[i]; + INIT_HLIST_HEAD(&cqinfo->queue[i].chead); + do { + curr = kzalloc(sizeof(*curr), GFP_KERNEL); + if (!curr) + goto cmd_qfail; + + c_size = (rem_q_size > qcsize_bytes) ? qcsize_bytes : + rem_q_size; + curr->head = dma_alloc_coherent(&pdev->dev, + c_size + CPT_NEXT_CHUNK_PTR_SIZE, + &curr->dma_addr, + GFP_KERNEL); + if (!curr->head) { + dev_err(&pdev->dev, "Command Q (%d) chunk (%d) allocation failed\n", + i, queue->nchunks); + kfree(curr); + goto cmd_qfail; + } + + curr->size = c_size; + if (queue->nchunks == 0) { + hlist_add_head(&curr->nextchunk, + &cqinfo->queue[i].chead); + first = curr; + } else { + hlist_add_behind(&curr->nextchunk, + &last->nextchunk); + } + + queue->nchunks++; + rem_q_size -= c_size; + if (last) + *((u64 *)(&last->head[last->size])) = (u64)curr->dma_addr; + + last = curr; + } while (rem_q_size); + + /* Make the queue circular */ + /* Tie back last chunk entry to head */ + curr = first; + *((u64 *)(&last->head[last->size])) = (u64)curr->dma_addr; + queue->qhead = curr; + spin_lock_init(&queue->lock); + } + return 0; + +cmd_qfail: + free_command_queues(cptvf, cqinfo); + return -ENOMEM; +} + +static int init_command_queues(struct cpt_vf *cptvf, u32 qlen) +{ + struct pci_dev *pdev = cptvf->pdev; + int ret; + + /* setup AE command queues */ + ret = alloc_command_queues(cptvf, &cptvf->cqinfo, CPT_INST_SIZE, + qlen); + if (ret) { + dev_err(&pdev->dev, "failed to allocate AE command queues (%u)\n", + cptvf->nr_queues); + return ret; + } + + return ret; +} + +static void cleanup_command_queues(struct cpt_vf *cptvf) +{ + struct pci_dev *pdev = cptvf->pdev; + + if (!cptvf->nr_queues) + return; + + dev_info(&pdev->dev, "Cleaning VQ command queue (%u)\n", + cptvf->nr_queues); + free_command_queues(cptvf, &cptvf->cqinfo); +} + +static void cptvf_sw_cleanup(struct cpt_vf *cptvf) +{ + cleanup_worker_threads(cptvf); + cleanup_pending_queues(cptvf); + cleanup_command_queues(cptvf); +} + +static int cptvf_sw_init(struct cpt_vf *cptvf, u32 qlen, u32 nr_queues) +{ + struct pci_dev *pdev = cptvf->pdev; + int ret = 0; + u32 max_dev_queues = 0; + + max_dev_queues = CPT_NUM_QS_PER_VF; + /* possible cpus */ + nr_queues = min_t(u32, nr_queues, max_dev_queues); + cptvf->nr_queues = nr_queues; + + ret = init_command_queues(cptvf, qlen); + if (ret) { + dev_err(&pdev->dev, "Failed to setup command queues (%u)\n", + nr_queues); + return ret; + } + + ret = init_pending_queues(cptvf, qlen, nr_queues); + if (ret) { + dev_err(&pdev->dev, "Failed to setup pending queues (%u)\n", + nr_queues); + goto setup_pqfail; + } + + /* Create worker threads for BH processing */ + ret = init_worker_threads(cptvf); + if (ret) { + dev_err(&pdev->dev, "Failed to setup worker threads\n"); + goto init_work_fail; + } + + return 0; + +init_work_fail: + cleanup_worker_threads(cptvf); + cleanup_pending_queues(cptvf); + +setup_pqfail: + cleanup_command_queues(cptvf); + + return ret; +} + +static void cptvf_free_irq_affinity(struct cpt_vf *cptvf, int vec) +{ + irq_set_affinity_hint(pci_irq_vector(cptvf->pdev, vec), NULL); + free_cpumask_var(cptvf->affinity_mask[vec]); +} + +static void cptvf_write_vq_ctl(struct cpt_vf *cptvf, bool val) +{ + union cptx_vqx_ctl vqx_ctl; + + vqx_ctl.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_CTL(0, 0)); + vqx_ctl.s.ena = val; + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_CTL(0, 0), vqx_ctl.u); +} + +void cptvf_write_vq_doorbell(struct cpt_vf *cptvf, u32 val) +{ + union cptx_vqx_doorbell vqx_dbell; + + vqx_dbell.u = cpt_read_csr64(cptvf->reg_base, + CPTX_VQX_DOORBELL(0, 0)); + vqx_dbell.s.dbell_cnt = val * 8; /* Num of Instructions * 8 words */ + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DOORBELL(0, 0), + vqx_dbell.u); +} + +static void cptvf_write_vq_inprog(struct cpt_vf *cptvf, u8 val) +{ + union cptx_vqx_inprog vqx_inprg; + + vqx_inprg.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_INPROG(0, 0)); + vqx_inprg.s.inflight = val; + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_INPROG(0, 0), vqx_inprg.u); +} + +static void cptvf_write_vq_done_numwait(struct cpt_vf *cptvf, u32 val) +{ + union cptx_vqx_done_wait vqx_dwait; + + vqx_dwait.u = cpt_read_csr64(cptvf->reg_base, + CPTX_VQX_DONE_WAIT(0, 0)); + vqx_dwait.s.num_wait = val; + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_WAIT(0, 0), + vqx_dwait.u); +} + +static void cptvf_write_vq_done_timewait(struct cpt_vf *cptvf, u16 time) +{ + union cptx_vqx_done_wait vqx_dwait; + + vqx_dwait.u = cpt_read_csr64(cptvf->reg_base, + CPTX_VQX_DONE_WAIT(0, 0)); + vqx_dwait.s.time_wait = time; + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_WAIT(0, 0), + vqx_dwait.u); +} + +static void cptvf_enable_swerr_interrupts(struct cpt_vf *cptvf) +{ + union cptx_vqx_misc_ena_w1s vqx_misc_ena; + + vqx_misc_ena.u = cpt_read_csr64(cptvf->reg_base, + CPTX_VQX_MISC_ENA_W1S(0, 0)); + /* Set mbox(0) interupts for the requested vf */ + vqx_misc_ena.s.swerr = 1; + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_ENA_W1S(0, 0), + vqx_misc_ena.u); +} + +static void cptvf_enable_mbox_interrupts(struct cpt_vf *cptvf) +{ + union cptx_vqx_misc_ena_w1s vqx_misc_ena; + + vqx_misc_ena.u = cpt_read_csr64(cptvf->reg_base, + CPTX_VQX_MISC_ENA_W1S(0, 0)); + /* Set mbox(0) interupts for the requested vf */ + vqx_misc_ena.s.mbox = 1; + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_ENA_W1S(0, 0), + vqx_misc_ena.u); +} + +static void cptvf_enable_done_interrupts(struct cpt_vf *cptvf) +{ + union cptx_vqx_done_ena_w1s vqx_done_ena; + + vqx_done_ena.u = cpt_read_csr64(cptvf->reg_base, + CPTX_VQX_DONE_ENA_W1S(0, 0)); + /* Set DONE interrupt for the requested vf */ + vqx_done_ena.s.done = 1; + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_ENA_W1S(0, 0), + vqx_done_ena.u); +} + +static void cptvf_clear_dovf_intr(struct cpt_vf *cptvf) +{ + union cptx_vqx_misc_int vqx_misc_int; + + vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base, + CPTX_VQX_MISC_INT(0, 0)); + /* W1C for the VF */ + vqx_misc_int.s.dovf = 1; + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0), + vqx_misc_int.u); +} + +static void cptvf_clear_irde_intr(struct cpt_vf *cptvf) +{ + union cptx_vqx_misc_int vqx_misc_int; + + vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base, + CPTX_VQX_MISC_INT(0, 0)); + /* W1C for the VF */ + vqx_misc_int.s.irde = 1; + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0), + vqx_misc_int.u); +} + +static void cptvf_clear_nwrp_intr(struct cpt_vf *cptvf) +{ + union cptx_vqx_misc_int vqx_misc_int; + + vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base, + CPTX_VQX_MISC_INT(0, 0)); + /* W1C for the VF */ + vqx_misc_int.s.nwrp = 1; + cpt_write_csr64(cptvf->reg_base, + CPTX_VQX_MISC_INT(0, 0), vqx_misc_int.u); +} + +static void cptvf_clear_mbox_intr(struct cpt_vf *cptvf) +{ + union cptx_vqx_misc_int vqx_misc_int; + + vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base, + CPTX_VQX_MISC_INT(0, 0)); + /* W1C for the VF */ + vqx_misc_int.s.mbox = 1; + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0), + vqx_misc_int.u); +} + +static void cptvf_clear_swerr_intr(struct cpt_vf *cptvf) +{ + union cptx_vqx_misc_int vqx_misc_int; + + vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base, + CPTX_VQX_MISC_INT(0, 0)); + /* W1C for the VF */ + vqx_misc_int.s.swerr = 1; + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0), + vqx_misc_int.u); +} + +static u64 cptvf_read_vf_misc_intr_status(struct cpt_vf *cptvf) +{ + return cpt_read_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0)); +} + +static irqreturn_t cptvf_misc_intr_handler(int irq, void *cptvf_irq) +{ + struct cpt_vf *cptvf = (struct cpt_vf *)cptvf_irq; + struct pci_dev *pdev = cptvf->pdev; + u64 intr; + + intr = cptvf_read_vf_misc_intr_status(cptvf); + /*Check for MISC interrupt types*/ + if (likely(intr & CPT_VF_INTR_MBOX_MASK)) { + dev_dbg(&pdev->dev, "Mailbox interrupt 0x%llx on CPT VF %d\n", + intr, cptvf->vfid); + cptvf_handle_mbox_intr(cptvf); + cptvf_clear_mbox_intr(cptvf); + } else if (unlikely(intr & CPT_VF_INTR_DOVF_MASK)) { + cptvf_clear_dovf_intr(cptvf); + /*Clear doorbell count*/ + cptvf_write_vq_doorbell(cptvf, 0); + dev_err(&pdev->dev, "Doorbell overflow error interrupt 0x%llx on CPT VF %d\n", + intr, cptvf->vfid); + } else if (unlikely(intr & CPT_VF_INTR_IRDE_MASK)) { + cptvf_clear_irde_intr(cptvf); + dev_err(&pdev->dev, "Instruction NCB read error interrupt 0x%llx on CPT VF %d\n", + intr, cptvf->vfid); + } else if (unlikely(intr & CPT_VF_INTR_NWRP_MASK)) { + cptvf_clear_nwrp_intr(cptvf); + dev_err(&pdev->dev, "NCB response write error interrupt 0x%llx on CPT VF %d\n", + intr, cptvf->vfid); + } else if (unlikely(intr & CPT_VF_INTR_SERR_MASK)) { + cptvf_clear_swerr_intr(cptvf); + dev_err(&pdev->dev, "Software error interrupt 0x%llx on CPT VF %d\n", + intr, cptvf->vfid); + } else { + dev_err(&pdev->dev, "Unhandled interrupt in CPT VF %d\n", + cptvf->vfid); + } + + return IRQ_HANDLED; +} + +static inline struct cptvf_wqe *get_cptvf_vq_wqe(struct cpt_vf *cptvf, + int qno) +{ + struct cptvf_wqe_info *nwqe_info; + + if (unlikely(qno >= cptvf->nr_queues)) + return NULL; + nwqe_info = (struct cptvf_wqe_info *)cptvf->wqe_info; + + return &nwqe_info->vq_wqe[qno]; +} + +static inline u32 cptvf_read_vq_done_count(struct cpt_vf *cptvf) +{ + union cptx_vqx_done vqx_done; + + vqx_done.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_DONE(0, 0)); + return vqx_done.s.done; +} + +static inline void cptvf_write_vq_done_ack(struct cpt_vf *cptvf, + u32 ackcnt) +{ + union cptx_vqx_done_ack vqx_dack_cnt; + + vqx_dack_cnt.u = cpt_read_csr64(cptvf->reg_base, + CPTX_VQX_DONE_ACK(0, 0)); + vqx_dack_cnt.s.done_ack = ackcnt; + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_ACK(0, 0), + vqx_dack_cnt.u); +} + +static irqreturn_t cptvf_done_intr_handler(int irq, void *cptvf_irq) +{ + struct cpt_vf *cptvf = (struct cpt_vf *)cptvf_irq; + struct pci_dev *pdev = cptvf->pdev; + /* Read the number of completions */ + u32 intr = cptvf_read_vq_done_count(cptvf); + + if (intr) { + struct cptvf_wqe *wqe; + + /* Acknowledge the number of + * scheduled completions for processing + */ + cptvf_write_vq_done_ack(cptvf, intr); + wqe = get_cptvf_vq_wqe(cptvf, 0); + if (unlikely(!wqe)) { + dev_err(&pdev->dev, "No work to schedule for VF (%d)", + cptvf->vfid); + return IRQ_NONE; + } + tasklet_hi_schedule(&wqe->twork); + } + + return IRQ_HANDLED; +} + +static void cptvf_set_irq_affinity(struct cpt_vf *cptvf, int vec) +{ + struct pci_dev *pdev = cptvf->pdev; + int cpu; + + if (!zalloc_cpumask_var(&cptvf->affinity_mask[vec], + GFP_KERNEL)) { + dev_err(&pdev->dev, "Allocation failed for affinity_mask for VF %d", + cptvf->vfid); + return; + } + + cpu = cptvf->vfid % num_online_cpus(); + cpumask_set_cpu(cpumask_local_spread(cpu, cptvf->node), + cptvf->affinity_mask[vec]); + irq_set_affinity_hint(pci_irq_vector(pdev, vec), + cptvf->affinity_mask[vec]); +} + +static void cptvf_write_vq_saddr(struct cpt_vf *cptvf, u64 val) +{ + union cptx_vqx_saddr vqx_saddr; + + vqx_saddr.u = val; + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_SADDR(0, 0), vqx_saddr.u); +} + +static void cptvf_device_init(struct cpt_vf *cptvf) +{ + u64 base_addr = 0; + + /* Disable the VQ */ + cptvf_write_vq_ctl(cptvf, 0); + /* Reset the doorbell */ + cptvf_write_vq_doorbell(cptvf, 0); + /* Clear inflight */ + cptvf_write_vq_inprog(cptvf, 0); + /* Write VQ SADDR */ + /* TODO: for now only one queue, so hard coded */ + base_addr = (u64)(cptvf->cqinfo.queue[0].qhead->dma_addr); + cptvf_write_vq_saddr(cptvf, base_addr); + /* Configure timerhold / coalescence */ + cptvf_write_vq_done_timewait(cptvf, CPT_TIMER_THOLD); + cptvf_write_vq_done_numwait(cptvf, 1); + /* Enable the VQ */ + cptvf_write_vq_ctl(cptvf, 1); + /* Flag the VF ready */ + cptvf->flags |= CPT_FLAG_DEVICE_READY; +} + +static int cptvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct device *dev = &pdev->dev; + struct cpt_vf *cptvf; + int err; + + cptvf = devm_kzalloc(dev, sizeof(*cptvf), GFP_KERNEL); + if (!cptvf) + return -ENOMEM; + + pci_set_drvdata(pdev, cptvf); + cptvf->pdev = pdev; + err = pci_enable_device(pdev); + if (err) { + dev_err(dev, "Failed to enable PCI device\n"); + pci_set_drvdata(pdev, NULL); + return err; + } + + err = pci_request_regions(pdev, DRV_NAME); + if (err) { + dev_err(dev, "PCI request regions failed 0x%x\n", err); + goto cptvf_err_disable_device; + } + /* Mark as VF driver */ + cptvf->flags |= CPT_FLAG_VF_DRIVER; + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); + if (err) { + dev_err(dev, "Unable to get usable 48-bit DMA configuration\n"); + goto cptvf_err_release_regions; + } + + /* MAP PF's configuration registers */ + cptvf->reg_base = pcim_iomap(pdev, 0, 0); + if (!cptvf->reg_base) { + dev_err(dev, "Cannot map config register space, aborting\n"); + err = -ENOMEM; + goto cptvf_err_release_regions; + } + + cptvf->node = dev_to_node(&pdev->dev); + err = pci_alloc_irq_vectors(pdev, CPT_VF_MSIX_VECTORS, + CPT_VF_MSIX_VECTORS, PCI_IRQ_MSIX); + if (err < 0) { + dev_err(dev, "Request for #%d msix vectors failed\n", + CPT_VF_MSIX_VECTORS); + goto cptvf_err_release_regions; + } + + err = request_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC), + cptvf_misc_intr_handler, 0, "CPT VF misc intr", + cptvf); + if (err) { + dev_err(dev, "Request misc irq failed"); + goto cptvf_free_vectors; + } + + /* Enable mailbox interrupt */ + cptvf_enable_mbox_interrupts(cptvf); + cptvf_enable_swerr_interrupts(cptvf); + + /* Check ready with PF */ + /* Gets chip ID / device Id from PF if ready */ + err = cptvf_check_pf_ready(cptvf); + if (err) { + dev_err(dev, "PF not responding to READY msg"); + goto cptvf_free_misc_irq; + } + + /* CPT VF software resources initialization */ + cptvf->cqinfo.qchunksize = CPT_CMD_QCHUNK_SIZE; + err = cptvf_sw_init(cptvf, CPT_CMD_QLEN, CPT_NUM_QS_PER_VF); + if (err) { + dev_err(dev, "cptvf_sw_init() failed"); + goto cptvf_free_misc_irq; + } + /* Convey VQ LEN to PF */ + err = cptvf_send_vq_size_msg(cptvf); + if (err) { + dev_err(dev, "PF not responding to QLEN msg"); + goto cptvf_free_misc_irq; + } + + /* CPT VF device initialization */ + cptvf_device_init(cptvf); + /* Send msg to PF to assign currnet Q to required group */ + cptvf->vfgrp = 1; + err = cptvf_send_vf_to_grp_msg(cptvf); + if (err) { + dev_err(dev, "PF not responding to VF_GRP msg"); + goto cptvf_free_misc_irq; + } + + cptvf->priority = 1; + err = cptvf_send_vf_priority_msg(cptvf); + if (err) { + dev_err(dev, "PF not responding to VF_PRIO msg"); + goto cptvf_free_misc_irq; + } + + err = request_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_DONE), + cptvf_done_intr_handler, 0, "CPT VF done intr", + cptvf); + if (err) { + dev_err(dev, "Request done irq failed\n"); + goto cptvf_free_misc_irq; + } + + /* Enable mailbox interrupt */ + cptvf_enable_done_interrupts(cptvf); + + /* Set irq affinity masks */ + cptvf_set_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC); + cptvf_set_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE); + + err = cptvf_send_vf_up(cptvf); + if (err) { + dev_err(dev, "PF not responding to UP msg"); + goto cptvf_free_irq_affinity; + } + err = cvm_crypto_init(cptvf); + if (err) { + dev_err(dev, "Algorithm register failed\n"); + goto cptvf_free_irq_affinity; + } + return 0; + +cptvf_free_irq_affinity: + cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE); + cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC); +cptvf_free_misc_irq: + free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC), cptvf); +cptvf_free_vectors: + pci_free_irq_vectors(cptvf->pdev); +cptvf_err_release_regions: + pci_release_regions(pdev); +cptvf_err_disable_device: + pci_disable_device(pdev); + pci_set_drvdata(pdev, NULL); + + return err; +} + +static void cptvf_remove(struct pci_dev *pdev) +{ + struct cpt_vf *cptvf = pci_get_drvdata(pdev); + + if (!cptvf) { + dev_err(&pdev->dev, "Invalid CPT-VF device\n"); + return; + } + + /* Convey DOWN to PF */ + if (cptvf_send_vf_down(cptvf)) { + dev_err(&pdev->dev, "PF not responding to DOWN msg"); + } else { + cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE); + cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC); + free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_DONE), cptvf); + free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC), cptvf); + pci_free_irq_vectors(cptvf->pdev); + cptvf_sw_cleanup(cptvf); + pci_set_drvdata(pdev, NULL); + pci_release_regions(pdev); + pci_disable_device(pdev); + cvm_crypto_exit(); + } +} + +static void cptvf_shutdown(struct pci_dev *pdev) +{ + cptvf_remove(pdev); +} + +/* Supported devices */ +static const struct pci_device_id cptvf_id_table[] = { + {PCI_VDEVICE(CAVIUM, CPT_81XX_PCI_VF_DEVICE_ID), 0}, + { 0, } /* end of table */ +}; + +static struct pci_driver cptvf_pci_driver = { + .name = DRV_NAME, + .id_table = cptvf_id_table, + .probe = cptvf_probe, + .remove = cptvf_remove, + .shutdown = cptvf_shutdown, +}; + +module_pci_driver(cptvf_pci_driver); + +MODULE_AUTHOR("George Cherian <george.cherian@cavium.com>"); +MODULE_DESCRIPTION("Cavium Thunder CPT Virtual Function Driver"); +MODULE_LICENSE("GPL v2"); +MODULE_VERSION(DRV_VERSION); +MODULE_DEVICE_TABLE(pci, cptvf_id_table); diff --git a/drivers/crypto/cavium/cpt/cptvf_mbox.c b/drivers/crypto/cavium/cpt/cptvf_mbox.c new file mode 100644 index 000000000..1267e1eba --- /dev/null +++ b/drivers/crypto/cavium/cpt/cptvf_mbox.c @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2016 Cavium, Inc. + */ + +#include "cptvf.h" + +static void cptvf_send_msg_to_pf(struct cpt_vf *cptvf, struct cpt_mbox *mbx) +{ + /* Writing mbox(1) causes interrupt */ + cpt_write_csr64(cptvf->reg_base, CPTX_VFX_PF_MBOXX(0, 0, 0), + mbx->msg); + cpt_write_csr64(cptvf->reg_base, CPTX_VFX_PF_MBOXX(0, 0, 1), + mbx->data); +} + +/* Interrupt handler to handle mailbox messages from VFs */ +void cptvf_handle_mbox_intr(struct cpt_vf *cptvf) +{ + struct cpt_mbox mbx = {}; + + /* + * MBOX[0] contains msg + * MBOX[1] contains data + */ + mbx.msg = cpt_read_csr64(cptvf->reg_base, CPTX_VFX_PF_MBOXX(0, 0, 0)); + mbx.data = cpt_read_csr64(cptvf->reg_base, CPTX_VFX_PF_MBOXX(0, 0, 1)); + dev_dbg(&cptvf->pdev->dev, "%s: Mailbox msg 0x%llx from PF\n", + __func__, mbx.msg); + switch (mbx.msg) { + case CPT_MSG_READY: + { + cptvf->pf_acked = true; + cptvf->vfid = mbx.data; + dev_dbg(&cptvf->pdev->dev, "Received VFID %d\n", cptvf->vfid); + break; + } + case CPT_MSG_QBIND_GRP: + cptvf->pf_acked = true; + cptvf->vftype = mbx.data; + dev_dbg(&cptvf->pdev->dev, "VF %d type %s group %d\n", + cptvf->vfid, ((mbx.data == SE_TYPES) ? "SE" : "AE"), + cptvf->vfgrp); + break; + case CPT_MBOX_MSG_TYPE_ACK: + cptvf->pf_acked = true; + break; + case CPT_MBOX_MSG_TYPE_NACK: + cptvf->pf_nacked = true; + break; + default: + dev_err(&cptvf->pdev->dev, "Invalid msg from PF, msg 0x%llx\n", + mbx.msg); + break; + } +} + +static int cptvf_send_msg_to_pf_timeout(struct cpt_vf *cptvf, + struct cpt_mbox *mbx) +{ + int timeout = CPT_MBOX_MSG_TIMEOUT; + int sleep = 10; + + cptvf->pf_acked = false; + cptvf->pf_nacked = false; + cptvf_send_msg_to_pf(cptvf, mbx); + /* Wait for previous message to be acked, timeout 2sec */ + while (!cptvf->pf_acked) { + if (cptvf->pf_nacked) + return -EINVAL; + msleep(sleep); + if (cptvf->pf_acked) + break; + timeout -= sleep; + if (!timeout) { + dev_err(&cptvf->pdev->dev, "PF didn't ack to mbox msg %llx from VF%u\n", + (mbx->msg & 0xFF), cptvf->vfid); + return -EBUSY; + } + } + + return 0; +} + +/* + * Checks if VF is able to comminicate with PF + * and also gets the CPT number this VF is associated to. + */ +int cptvf_check_pf_ready(struct cpt_vf *cptvf) +{ + struct pci_dev *pdev = cptvf->pdev; + struct cpt_mbox mbx = {}; + + mbx.msg = CPT_MSG_READY; + if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) { + dev_err(&pdev->dev, "PF didn't respond to READY msg\n"); + return -EBUSY; + } + + return 0; +} + +/* + * Communicate VQs size to PF to program CPT(0)_PF_Q(0-15)_CTL of the VF. + * Must be ACKed. + */ +int cptvf_send_vq_size_msg(struct cpt_vf *cptvf) +{ + struct pci_dev *pdev = cptvf->pdev; + struct cpt_mbox mbx = {}; + + mbx.msg = CPT_MSG_QLEN; + mbx.data = cptvf->qsize; + if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) { + dev_err(&pdev->dev, "PF didn't respond to vq_size msg\n"); + return -EBUSY; + } + + return 0; +} + +/* + * Communicate VF group required to PF and get the VQ binded to that group + */ +int cptvf_send_vf_to_grp_msg(struct cpt_vf *cptvf) +{ + struct pci_dev *pdev = cptvf->pdev; + struct cpt_mbox mbx = {}; + + mbx.msg = CPT_MSG_QBIND_GRP; + /* Convey group of the VF */ + mbx.data = cptvf->vfgrp; + if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) { + dev_err(&pdev->dev, "PF didn't respond to vf_type msg\n"); + return -EBUSY; + } + + return 0; +} + +/* + * Communicate VF group required to PF and get the VQ binded to that group + */ +int cptvf_send_vf_priority_msg(struct cpt_vf *cptvf) +{ + struct pci_dev *pdev = cptvf->pdev; + struct cpt_mbox mbx = {}; + + mbx.msg = CPT_MSG_VQ_PRIORITY; + /* Convey group of the VF */ + mbx.data = cptvf->priority; + if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) { + dev_err(&pdev->dev, "PF didn't respond to vf_type msg\n"); + return -EBUSY; + } + return 0; +} + +/* + * Communicate to PF that VF is UP and running + */ +int cptvf_send_vf_up(struct cpt_vf *cptvf) +{ + struct pci_dev *pdev = cptvf->pdev; + struct cpt_mbox mbx = {}; + + mbx.msg = CPT_MSG_VF_UP; + if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) { + dev_err(&pdev->dev, "PF didn't respond to UP msg\n"); + return -EBUSY; + } + + return 0; +} + +/* + * Communicate to PF that VF is DOWN and running + */ +int cptvf_send_vf_down(struct cpt_vf *cptvf) +{ + struct pci_dev *pdev = cptvf->pdev; + struct cpt_mbox mbx = {}; + + mbx.msg = CPT_MSG_VF_DOWN; + if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) { + dev_err(&pdev->dev, "PF didn't respond to DOWN msg\n"); + return -EBUSY; + } + + return 0; +} diff --git a/drivers/crypto/cavium/cpt/cptvf_reqmanager.c b/drivers/crypto/cavium/cpt/cptvf_reqmanager.c new file mode 100644 index 000000000..153004bdf --- /dev/null +++ b/drivers/crypto/cavium/cpt/cptvf_reqmanager.c @@ -0,0 +1,575 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2016 Cavium, Inc. + */ + +#include "cptvf.h" +#include "cptvf_algs.h" +#include "request_manager.h" + +/** + * get_free_pending_entry - get free entry from pending queue + * @q: pending queue + * @qlen: queue length + */ +static struct pending_entry *get_free_pending_entry(struct pending_queue *q, + int qlen) +{ + struct pending_entry *ent = NULL; + + ent = &q->head[q->rear]; + if (unlikely(ent->busy)) { + ent = NULL; + goto no_free_entry; + } + + q->rear++; + if (unlikely(q->rear == qlen)) + q->rear = 0; + +no_free_entry: + return ent; +} + +static inline void pending_queue_inc_front(struct pending_qinfo *pqinfo, + int qno) +{ + struct pending_queue *queue = &pqinfo->queue[qno]; + + queue->front++; + if (unlikely(queue->front == pqinfo->qlen)) + queue->front = 0; +} + +static int setup_sgio_components(struct cpt_vf *cptvf, struct buf_ptr *list, + int buf_count, u8 *buffer) +{ + int ret = 0, i, j; + int components; + struct sglist_component *sg_ptr = NULL; + struct pci_dev *pdev = cptvf->pdev; + + if (unlikely(!list)) { + dev_err(&pdev->dev, "Input List pointer is NULL\n"); + return -EFAULT; + } + + for (i = 0; i < buf_count; i++) { + if (likely(list[i].vptr)) { + list[i].dma_addr = dma_map_single(&pdev->dev, + list[i].vptr, + list[i].size, + DMA_BIDIRECTIONAL); + if (unlikely(dma_mapping_error(&pdev->dev, + list[i].dma_addr))) { + dev_err(&pdev->dev, "DMA map kernel buffer failed for component: %d\n", + i); + ret = -EIO; + goto sg_cleanup; + } + } + } + + components = buf_count / 4; + sg_ptr = (struct sglist_component *)buffer; + for (i = 0; i < components; i++) { + sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size); + sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size); + sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size); + sg_ptr->u.s.len3 = cpu_to_be16(list[i * 4 + 3].size); + sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr); + sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr); + sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr); + sg_ptr->ptr3 = cpu_to_be64(list[i * 4 + 3].dma_addr); + sg_ptr++; + } + + components = buf_count % 4; + + switch (components) { + case 3: + sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size); + sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr); + fallthrough; + case 2: + sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size); + sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr); + fallthrough; + case 1: + sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size); + sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr); + break; + default: + break; + } + + return ret; + +sg_cleanup: + for (j = 0; j < i; j++) { + if (list[j].dma_addr) { + dma_unmap_single(&pdev->dev, list[i].dma_addr, + list[i].size, DMA_BIDIRECTIONAL); + } + + list[j].dma_addr = 0; + } + + return ret; +} + +static inline int setup_sgio_list(struct cpt_vf *cptvf, + struct cpt_info_buffer *info, + struct cpt_request_info *req) +{ + u16 g_sz_bytes = 0, s_sz_bytes = 0; + int ret = 0; + struct pci_dev *pdev = cptvf->pdev; + + if (req->incnt > MAX_SG_IN_CNT || req->outcnt > MAX_SG_OUT_CNT) { + dev_err(&pdev->dev, "Request SG components are higher than supported\n"); + ret = -EINVAL; + goto scatter_gather_clean; + } + + /* Setup gather (input) components */ + g_sz_bytes = ((req->incnt + 3) / 4) * sizeof(struct sglist_component); + info->gather_components = kzalloc(g_sz_bytes, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC); + if (!info->gather_components) { + ret = -ENOMEM; + goto scatter_gather_clean; + } + + ret = setup_sgio_components(cptvf, req->in, + req->incnt, + info->gather_components); + if (ret) { + dev_err(&pdev->dev, "Failed to setup gather list\n"); + ret = -EFAULT; + goto scatter_gather_clean; + } + + /* Setup scatter (output) components */ + s_sz_bytes = ((req->outcnt + 3) / 4) * sizeof(struct sglist_component); + info->scatter_components = kzalloc(s_sz_bytes, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC); + if (!info->scatter_components) { + ret = -ENOMEM; + goto scatter_gather_clean; + } + + ret = setup_sgio_components(cptvf, req->out, + req->outcnt, + info->scatter_components); + if (ret) { + dev_err(&pdev->dev, "Failed to setup gather list\n"); + ret = -EFAULT; + goto scatter_gather_clean; + } + + /* Create and initialize DPTR */ + info->dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE; + info->in_buffer = kzalloc(info->dlen, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC); + if (!info->in_buffer) { + ret = -ENOMEM; + goto scatter_gather_clean; + } + + ((__be16 *)info->in_buffer)[0] = cpu_to_be16(req->outcnt); + ((__be16 *)info->in_buffer)[1] = cpu_to_be16(req->incnt); + ((__be16 *)info->in_buffer)[2] = 0; + ((__be16 *)info->in_buffer)[3] = 0; + + memcpy(&info->in_buffer[8], info->gather_components, + g_sz_bytes); + memcpy(&info->in_buffer[8 + g_sz_bytes], + info->scatter_components, s_sz_bytes); + + info->dptr_baddr = dma_map_single(&pdev->dev, + (void *)info->in_buffer, + info->dlen, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(&pdev->dev, info->dptr_baddr)) { + dev_err(&pdev->dev, "Mapping DPTR Failed %d\n", info->dlen); + ret = -EIO; + goto scatter_gather_clean; + } + + /* Create and initialize RPTR */ + info->out_buffer = kzalloc(COMPLETION_CODE_SIZE, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC); + if (!info->out_buffer) { + ret = -ENOMEM; + goto scatter_gather_clean; + } + + *((u64 *)info->out_buffer) = ~((u64)COMPLETION_CODE_INIT); + info->alternate_caddr = (u64 *)info->out_buffer; + info->rptr_baddr = dma_map_single(&pdev->dev, + (void *)info->out_buffer, + COMPLETION_CODE_SIZE, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(&pdev->dev, info->rptr_baddr)) { + dev_err(&pdev->dev, "Mapping RPTR Failed %d\n", + COMPLETION_CODE_SIZE); + ret = -EIO; + goto scatter_gather_clean; + } + + return 0; + +scatter_gather_clean: + return ret; +} + +static int send_cpt_command(struct cpt_vf *cptvf, union cpt_inst_s *cmd, + u32 qno) +{ + struct pci_dev *pdev = cptvf->pdev; + struct command_qinfo *qinfo = NULL; + struct command_queue *queue; + struct command_chunk *chunk; + u8 *ent; + int ret = 0; + + if (unlikely(qno >= cptvf->nr_queues)) { + dev_err(&pdev->dev, "Invalid queue (qno: %d, nr_queues: %d)\n", + qno, cptvf->nr_queues); + return -EINVAL; + } + + qinfo = &cptvf->cqinfo; + queue = &qinfo->queue[qno]; + /* lock commad queue */ + spin_lock(&queue->lock); + ent = &queue->qhead->head[queue->idx * qinfo->cmd_size]; + memcpy(ent, (void *)cmd, qinfo->cmd_size); + + if (++queue->idx >= queue->qhead->size / 64) { + hlist_for_each_entry(chunk, &queue->chead, nextchunk) { + if (chunk == queue->qhead) { + continue; + } else { + queue->qhead = chunk; + break; + } + } + queue->idx = 0; + } + /* make sure all memory stores are done before ringing doorbell */ + smp_wmb(); + cptvf_write_vq_doorbell(cptvf, 1); + /* unlock command queue */ + spin_unlock(&queue->lock); + + return ret; +} + +static void do_request_cleanup(struct cpt_vf *cptvf, + struct cpt_info_buffer *info) +{ + int i; + struct pci_dev *pdev = cptvf->pdev; + struct cpt_request_info *req; + + if (info->dptr_baddr) + dma_unmap_single(&pdev->dev, info->dptr_baddr, + info->dlen, DMA_BIDIRECTIONAL); + + if (info->rptr_baddr) + dma_unmap_single(&pdev->dev, info->rptr_baddr, + COMPLETION_CODE_SIZE, DMA_BIDIRECTIONAL); + + if (info->comp_baddr) + dma_unmap_single(&pdev->dev, info->comp_baddr, + sizeof(union cpt_res_s), DMA_BIDIRECTIONAL); + + if (info->req) { + req = info->req; + for (i = 0; i < req->outcnt; i++) { + if (req->out[i].dma_addr) + dma_unmap_single(&pdev->dev, + req->out[i].dma_addr, + req->out[i].size, + DMA_BIDIRECTIONAL); + } + + for (i = 0; i < req->incnt; i++) { + if (req->in[i].dma_addr) + dma_unmap_single(&pdev->dev, + req->in[i].dma_addr, + req->in[i].size, + DMA_BIDIRECTIONAL); + } + } + + kfree_sensitive(info->scatter_components); + kfree_sensitive(info->gather_components); + kfree_sensitive(info->out_buffer); + kfree_sensitive(info->in_buffer); + kfree_sensitive((void *)info->completion_addr); + kfree_sensitive(info); +} + +static void do_post_process(struct cpt_vf *cptvf, struct cpt_info_buffer *info) +{ + struct pci_dev *pdev = cptvf->pdev; + + if (!info) { + dev_err(&pdev->dev, "incorrect cpt_info_buffer for post processing\n"); + return; + } + + do_request_cleanup(cptvf, info); +} + +static inline void process_pending_queue(struct cpt_vf *cptvf, + struct pending_qinfo *pqinfo, + int qno) +{ + struct pci_dev *pdev = cptvf->pdev; + struct pending_queue *pqueue = &pqinfo->queue[qno]; + struct pending_entry *pentry = NULL; + struct cpt_info_buffer *info = NULL; + union cpt_res_s *status = NULL; + unsigned char ccode; + + while (1) { + spin_lock_bh(&pqueue->lock); + pentry = &pqueue->head[pqueue->front]; + if (unlikely(!pentry->busy)) { + spin_unlock_bh(&pqueue->lock); + break; + } + + info = (struct cpt_info_buffer *)pentry->post_arg; + if (unlikely(!info)) { + dev_err(&pdev->dev, "Pending Entry post arg NULL\n"); + pending_queue_inc_front(pqinfo, qno); + spin_unlock_bh(&pqueue->lock); + continue; + } + + status = (union cpt_res_s *)pentry->completion_addr; + ccode = status->s.compcode; + if ((status->s.compcode == CPT_COMP_E_FAULT) || + (status->s.compcode == CPT_COMP_E_SWERR)) { + dev_err(&pdev->dev, "Request failed with %s\n", + (status->s.compcode == CPT_COMP_E_FAULT) ? + "DMA Fault" : "Software error"); + pentry->completion_addr = NULL; + pentry->busy = false; + atomic64_dec((&pqueue->pending_count)); + pentry->post_arg = NULL; + pending_queue_inc_front(pqinfo, qno); + do_request_cleanup(cptvf, info); + spin_unlock_bh(&pqueue->lock); + break; + } else if (status->s.compcode == COMPLETION_CODE_INIT) { + /* check for timeout */ + if (time_after_eq(jiffies, + (info->time_in + + (CPT_COMMAND_TIMEOUT * HZ)))) { + dev_err(&pdev->dev, "Request timed out"); + pentry->completion_addr = NULL; + pentry->busy = false; + atomic64_dec((&pqueue->pending_count)); + pentry->post_arg = NULL; + pending_queue_inc_front(pqinfo, qno); + do_request_cleanup(cptvf, info); + spin_unlock_bh(&pqueue->lock); + break; + } else if ((*info->alternate_caddr == + (~COMPLETION_CODE_INIT)) && + (info->extra_time < TIME_IN_RESET_COUNT)) { + info->time_in = jiffies; + info->extra_time++; + spin_unlock_bh(&pqueue->lock); + break; + } + } + + pentry->completion_addr = NULL; + pentry->busy = false; + pentry->post_arg = NULL; + atomic64_dec((&pqueue->pending_count)); + pending_queue_inc_front(pqinfo, qno); + spin_unlock_bh(&pqueue->lock); + + do_post_process(info->cptvf, info); + /* + * Calling callback after we find + * that the request has been serviced + */ + pentry->callback(ccode, pentry->callback_arg); + } +} + +int process_request(struct cpt_vf *cptvf, struct cpt_request_info *req) +{ + int ret = 0, clear = 0, queue = 0; + struct cpt_info_buffer *info = NULL; + struct cptvf_request *cpt_req = NULL; + union ctrl_info *ctrl = NULL; + union cpt_res_s *result = NULL; + struct pending_entry *pentry = NULL; + struct pending_queue *pqueue = NULL; + struct pci_dev *pdev = cptvf->pdev; + u8 group = 0; + struct cpt_vq_command vq_cmd; + union cpt_inst_s cptinst; + + info = kzalloc(sizeof(*info), req->may_sleep ? GFP_KERNEL : GFP_ATOMIC); + if (unlikely(!info)) { + dev_err(&pdev->dev, "Unable to allocate memory for info_buffer\n"); + return -ENOMEM; + } + + cpt_req = (struct cptvf_request *)&req->req; + ctrl = (union ctrl_info *)&req->ctrl; + + info->cptvf = cptvf; + group = ctrl->s.grp; + ret = setup_sgio_list(cptvf, info, req); + if (ret) { + dev_err(&pdev->dev, "Setting up SG list failed"); + goto request_cleanup; + } + + cpt_req->dlen = info->dlen; + /* + * Get buffer for union cpt_res_s response + * structure and its physical address + */ + info->completion_addr = kzalloc(sizeof(union cpt_res_s), req->may_sleep ? GFP_KERNEL : GFP_ATOMIC); + if (unlikely(!info->completion_addr)) { + dev_err(&pdev->dev, "Unable to allocate memory for completion_addr\n"); + ret = -ENOMEM; + goto request_cleanup; + } + + result = (union cpt_res_s *)info->completion_addr; + result->s.compcode = COMPLETION_CODE_INIT; + info->comp_baddr = dma_map_single(&pdev->dev, + (void *)info->completion_addr, + sizeof(union cpt_res_s), + DMA_BIDIRECTIONAL); + if (dma_mapping_error(&pdev->dev, info->comp_baddr)) { + dev_err(&pdev->dev, "mapping compptr Failed %lu\n", + sizeof(union cpt_res_s)); + ret = -EFAULT; + goto request_cleanup; + } + + /* Fill the VQ command */ + vq_cmd.cmd.u64 = 0; + vq_cmd.cmd.s.opcode = cpu_to_be16(cpt_req->opcode.flags); + vq_cmd.cmd.s.param1 = cpu_to_be16(cpt_req->param1); + vq_cmd.cmd.s.param2 = cpu_to_be16(cpt_req->param2); + vq_cmd.cmd.s.dlen = cpu_to_be16(cpt_req->dlen); + + vq_cmd.dptr = info->dptr_baddr; + vq_cmd.rptr = info->rptr_baddr; + vq_cmd.cptr.u64 = 0; + vq_cmd.cptr.s.grp = group; + /* Get Pending Entry to submit command */ + /* Always queue 0, because 1 queue per VF */ + queue = 0; + pqueue = &cptvf->pqinfo.queue[queue]; + + if (atomic64_read(&pqueue->pending_count) > PENDING_THOLD) { + dev_err(&pdev->dev, "pending threshold reached\n"); + process_pending_queue(cptvf, &cptvf->pqinfo, queue); + } + +get_pending_entry: + spin_lock_bh(&pqueue->lock); + pentry = get_free_pending_entry(pqueue, cptvf->pqinfo.qlen); + if (unlikely(!pentry)) { + spin_unlock_bh(&pqueue->lock); + if (clear == 0) { + process_pending_queue(cptvf, &cptvf->pqinfo, queue); + clear = 1; + goto get_pending_entry; + } + dev_err(&pdev->dev, "Get free entry failed\n"); + dev_err(&pdev->dev, "queue: %d, rear: %d, front: %d\n", + queue, pqueue->rear, pqueue->front); + ret = -EFAULT; + goto request_cleanup; + } + + pentry->completion_addr = info->completion_addr; + pentry->post_arg = (void *)info; + pentry->callback = req->callback; + pentry->callback_arg = req->callback_arg; + info->pentry = pentry; + pentry->busy = true; + atomic64_inc(&pqueue->pending_count); + + /* Send CPT command */ + info->pentry = pentry; + info->time_in = jiffies; + info->req = req; + + /* Create the CPT_INST_S type command for HW intrepretation */ + cptinst.s.doneint = true; + cptinst.s.res_addr = (u64)info->comp_baddr; + cptinst.s.tag = 0; + cptinst.s.grp = 0; + cptinst.s.wq_ptr = 0; + cptinst.s.ei0 = vq_cmd.cmd.u64; + cptinst.s.ei1 = vq_cmd.dptr; + cptinst.s.ei2 = vq_cmd.rptr; + cptinst.s.ei3 = vq_cmd.cptr.u64; + + ret = send_cpt_command(cptvf, &cptinst, queue); + spin_unlock_bh(&pqueue->lock); + if (unlikely(ret)) { + dev_err(&pdev->dev, "Send command failed for AE\n"); + ret = -EFAULT; + goto request_cleanup; + } + + return 0; + +request_cleanup: + dev_dbg(&pdev->dev, "Failed to submit CPT command\n"); + do_request_cleanup(cptvf, info); + + return ret; +} + +void vq_post_process(struct cpt_vf *cptvf, u32 qno) +{ + struct pci_dev *pdev = cptvf->pdev; + + if (unlikely(qno > cptvf->nr_queues)) { + dev_err(&pdev->dev, "Request for post processing on invalid pending queue: %u\n", + qno); + return; + } + + process_pending_queue(cptvf, &cptvf->pqinfo, qno); +} + +int cptvf_do_request(void *vfdev, struct cpt_request_info *req) +{ + struct cpt_vf *cptvf = (struct cpt_vf *)vfdev; + struct pci_dev *pdev = cptvf->pdev; + + if (!cpt_device_ready(cptvf)) { + dev_err(&pdev->dev, "CPT Device is not ready"); + return -ENODEV; + } + + if ((cptvf->vftype == SE_TYPES) && (!req->ctrl.s.se_req)) { + dev_err(&pdev->dev, "CPTVF-%d of SE TYPE got AE request", + cptvf->vfid); + return -EINVAL; + } else if ((cptvf->vftype == AE_TYPES) && (req->ctrl.s.se_req)) { + dev_err(&pdev->dev, "CPTVF-%d of AE TYPE got SE request", + cptvf->vfid); + return -EINVAL; + } + + return process_request(cptvf, req); +} diff --git a/drivers/crypto/cavium/cpt/request_manager.h b/drivers/crypto/cavium/cpt/request_manager.h new file mode 100644 index 000000000..8d40e4ba3 --- /dev/null +++ b/drivers/crypto/cavium/cpt/request_manager.h @@ -0,0 +1,146 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2016 Cavium, Inc. + */ + +#ifndef __REQUEST_MANAGER_H +#define __REQUEST_MANAGER_H + +#include "cpt_common.h" + +#define TIME_IN_RESET_COUNT 5 +#define COMPLETION_CODE_SIZE 8 +#define COMPLETION_CODE_INIT 0 +#define PENDING_THOLD 100 +#define MAX_SG_IN_CNT 12 +#define MAX_SG_OUT_CNT 13 +#define SG_LIST_HDR_SIZE 8 +#define MAX_BUF_CNT 16 + +union ctrl_info { + u32 flags; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u32 reserved0:26; + u32 grp:3; /* Group bits */ + u32 dma_mode:2; /* DMA mode */ + u32 se_req:1;/* To SE core */ +#else + u32 se_req:1; /* To SE core */ + u32 dma_mode:2; /* DMA mode */ + u32 grp:3; /* Group bits */ + u32 reserved0:26; +#endif + } s; +}; + +union opcode_info { + u16 flags; + struct { + u8 major; + u8 minor; + } s; +}; + +struct cptvf_request { + union opcode_info opcode; + u16 param1; + u16 param2; + u16 dlen; +}; + +struct buf_ptr { + u8 *vptr; + dma_addr_t dma_addr; + u16 size; +}; + +struct cpt_request_info { + u8 incnt; /* Number of input buffers */ + u8 outcnt; /* Number of output buffers */ + u16 rlen; /* Output length */ + union ctrl_info ctrl; /* User control information */ + struct cptvf_request req; /* Request Information (Core specific) */ + + bool may_sleep; + + struct buf_ptr in[MAX_BUF_CNT]; + struct buf_ptr out[MAX_BUF_CNT]; + + void (*callback)(int, void *); /* Kernel ASYNC request callabck */ + void *callback_arg; /* Kernel ASYNC request callabck arg */ +}; + +struct sglist_component { + union { + u64 len; + struct { + __be16 len0; + __be16 len1; + __be16 len2; + __be16 len3; + } s; + } u; + __be64 ptr0; + __be64 ptr1; + __be64 ptr2; + __be64 ptr3; +}; + +struct cpt_info_buffer { + struct cpt_vf *cptvf; + unsigned long time_in; + u8 extra_time; + + struct cpt_request_info *req; + dma_addr_t dptr_baddr; + u32 dlen; + dma_addr_t rptr_baddr; + dma_addr_t comp_baddr; + u8 *in_buffer; + u8 *out_buffer; + u8 *gather_components; + u8 *scatter_components; + + struct pending_entry *pentry; + volatile u64 *completion_addr; + volatile u64 *alternate_caddr; +}; + +/* + * CPT_INST_S software command definitions + * Words EI (0-3) + */ +union vq_cmd_word0 { + u64 u64; + struct { + __be16 opcode; + __be16 param1; + __be16 param2; + __be16 dlen; + } s; +}; + +union vq_cmd_word3 { + u64 u64; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 grp:3; + u64 cptr:61; +#else + u64 cptr:61; + u64 grp:3; +#endif + } s; +}; + +struct cpt_vq_command { + union vq_cmd_word0 cmd; + u64 dptr; + u64 rptr; + union vq_cmd_word3 cptr; +}; + +void vq_post_process(struct cpt_vf *cptvf, u32 qno); +int process_request(struct cpt_vf *cptvf, struct cpt_request_info *req); +#endif /* __REQUEST_MANAGER_H */ diff --git a/drivers/crypto/cavium/nitrox/Kconfig b/drivers/crypto/cavium/nitrox/Kconfig new file mode 100644 index 000000000..7dc008332 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/Kconfig @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Cavium NITROX Crypto Device configuration +# +config CRYPTO_DEV_NITROX + tristate + select CRYPTO_SKCIPHER + select CRYPTO_AES + select CRYPTO_LIB_DES + select FW_LOADER + +config CRYPTO_DEV_NITROX_CNN55XX + tristate "Support for Cavium CNN55XX driver" + depends on PCI_MSI && 64BIT + select CRYPTO_DEV_NITROX + help + Support for Cavium NITROX family CNN55XX driver + for accelerating crypto workloads. + + To compile this as a module, choose M here: the module + will be called n5pf. diff --git a/drivers/crypto/cavium/nitrox/Makefile b/drivers/crypto/cavium/nitrox/Makefile new file mode 100644 index 000000000..f83991aaf --- /dev/null +++ b/drivers/crypto/cavium/nitrox/Makefile @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_CRYPTO_DEV_NITROX_CNN55XX) += n5pf.o + +n5pf-objs := nitrox_main.o \ + nitrox_isr.o \ + nitrox_lib.o \ + nitrox_hal.o \ + nitrox_reqmgr.o \ + nitrox_algs.o \ + nitrox_mbx.o \ + nitrox_skcipher.o \ + nitrox_aead.o + +n5pf-$(CONFIG_PCI_IOV) += nitrox_sriov.o +n5pf-$(CONFIG_DEBUG_FS) += nitrox_debugfs.o diff --git a/drivers/crypto/cavium/nitrox/nitrox_aead.c b/drivers/crypto/cavium/nitrox/nitrox_aead.c new file mode 100644 index 000000000..c93c4e41d --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_aead.c @@ -0,0 +1,567 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/kernel.h> +#include <linux/printk.h> +#include <linux/crypto.h> +#include <linux/rtnetlink.h> + +#include <crypto/aead.h> +#include <crypto/authenc.h> +#include <crypto/des.h> +#include <crypto/internal/aead.h> +#include <crypto/scatterwalk.h> +#include <crypto/gcm.h> + +#include "nitrox_dev.h" +#include "nitrox_common.h" +#include "nitrox_req.h" + +#define GCM_AES_SALT_SIZE 4 + +union gph_p3 { + struct { +#ifdef __BIG_ENDIAN_BITFIELD + u16 iv_offset : 8; + u16 auth_offset : 8; +#else + u16 auth_offset : 8; + u16 iv_offset : 8; +#endif + }; + u16 param; +}; + +static int nitrox_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key, + unsigned int keylen) +{ + int aes_keylen; + struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); + struct flexi_crypto_context *fctx; + union fc_ctx_flags flags; + + aes_keylen = flexi_aes_keylen(keylen); + if (aes_keylen < 0) + return -EINVAL; + + /* fill crypto context */ + fctx = nctx->u.fctx; + flags.fu = be64_to_cpu(fctx->flags.f); + flags.w0.aes_keylen = aes_keylen; + fctx->flags.f = cpu_to_be64(flags.fu); + + /* copy enc key to context */ + memset(&fctx->crypto, 0, sizeof(fctx->crypto)); + memcpy(fctx->crypto.u.key, key, keylen); + + return 0; +} + +static int nitrox_aead_setauthsize(struct crypto_aead *aead, + unsigned int authsize) +{ + struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); + struct flexi_crypto_context *fctx = nctx->u.fctx; + union fc_ctx_flags flags; + + flags.fu = be64_to_cpu(fctx->flags.f); + flags.w0.mac_len = authsize; + fctx->flags.f = cpu_to_be64(flags.fu); + + aead->authsize = authsize; + + return 0; +} + +static int nitrox_aes_gcm_setauthsize(struct crypto_aead *aead, + unsigned int authsize) +{ + switch (authsize) { + case 4: + case 8: + case 12: + case 13: + case 14: + case 15: + case 16: + break; + default: + return -EINVAL; + } + + return nitrox_aead_setauthsize(aead, authsize); +} + +static int alloc_src_sglist(struct nitrox_kcrypt_request *nkreq, + struct scatterlist *src, char *iv, int ivsize, + int buflen) +{ + int nents = sg_nents_for_len(src, buflen); + int ret; + + if (nents < 0) + return nents; + + /* IV entry */ + nents += 1; + /* Allocate buffer to hold IV and input scatterlist array */ + ret = alloc_src_req_buf(nkreq, nents, ivsize); + if (ret) + return ret; + + nitrox_creq_copy_iv(nkreq->src, iv, ivsize); + nitrox_creq_set_src_sg(nkreq, nents, ivsize, src, buflen); + + return 0; +} + +static int alloc_dst_sglist(struct nitrox_kcrypt_request *nkreq, + struct scatterlist *dst, int ivsize, int buflen) +{ + int nents = sg_nents_for_len(dst, buflen); + int ret; + + if (nents < 0) + return nents; + + /* IV, ORH, COMPLETION entries */ + nents += 3; + /* Allocate buffer to hold ORH, COMPLETION and output scatterlist + * array + */ + ret = alloc_dst_req_buf(nkreq, nents); + if (ret) + return ret; + + nitrox_creq_set_orh(nkreq); + nitrox_creq_set_comp(nkreq); + nitrox_creq_set_dst_sg(nkreq, nents, ivsize, dst, buflen); + + return 0; +} + +static void free_src_sglist(struct nitrox_kcrypt_request *nkreq) +{ + kfree(nkreq->src); +} + +static void free_dst_sglist(struct nitrox_kcrypt_request *nkreq) +{ + kfree(nkreq->dst); +} + +static int nitrox_set_creq(struct nitrox_aead_rctx *rctx) +{ + struct se_crypto_request *creq = &rctx->nkreq.creq; + union gph_p3 param3; + int ret; + + creq->flags = rctx->flags; + creq->gfp = (rctx->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : + GFP_ATOMIC; + + creq->ctrl.value = 0; + creq->opcode = FLEXI_CRYPTO_ENCRYPT_HMAC; + creq->ctrl.s.arg = rctx->ctrl_arg; + + creq->gph.param0 = cpu_to_be16(rctx->cryptlen); + creq->gph.param1 = cpu_to_be16(rctx->cryptlen + rctx->assoclen); + creq->gph.param2 = cpu_to_be16(rctx->ivsize + rctx->assoclen); + param3.iv_offset = 0; + param3.auth_offset = rctx->ivsize; + creq->gph.param3 = cpu_to_be16(param3.param); + + creq->ctx_handle = rctx->ctx_handle; + creq->ctrl.s.ctxl = sizeof(struct flexi_crypto_context); + + ret = alloc_src_sglist(&rctx->nkreq, rctx->src, rctx->iv, rctx->ivsize, + rctx->srclen); + if (ret) + return ret; + + ret = alloc_dst_sglist(&rctx->nkreq, rctx->dst, rctx->ivsize, + rctx->dstlen); + if (ret) { + free_src_sglist(&rctx->nkreq); + return ret; + } + + return 0; +} + +static void nitrox_aead_callback(void *arg, int err) +{ + struct aead_request *areq = arg; + struct nitrox_aead_rctx *rctx = aead_request_ctx(areq); + + free_src_sglist(&rctx->nkreq); + free_dst_sglist(&rctx->nkreq); + if (err) { + pr_err_ratelimited("request failed status 0x%0x\n", err); + err = -EINVAL; + } + + areq->base.complete(&areq->base, err); +} + +static inline bool nitrox_aes_gcm_assoclen_supported(unsigned int assoclen) +{ + if (assoclen <= 512) + return true; + + return false; +} + +static int nitrox_aes_gcm_enc(struct aead_request *areq) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(areq); + struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); + struct nitrox_aead_rctx *rctx = aead_request_ctx(areq); + struct se_crypto_request *creq = &rctx->nkreq.creq; + struct flexi_crypto_context *fctx = nctx->u.fctx; + int ret; + + if (!nitrox_aes_gcm_assoclen_supported(areq->assoclen)) + return -EINVAL; + + memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE); + + rctx->cryptlen = areq->cryptlen; + rctx->assoclen = areq->assoclen; + rctx->srclen = areq->assoclen + areq->cryptlen; + rctx->dstlen = rctx->srclen + aead->authsize; + rctx->iv = &areq->iv[GCM_AES_SALT_SIZE]; + rctx->ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE; + rctx->flags = areq->base.flags; + rctx->ctx_handle = nctx->u.ctx_handle; + rctx->src = areq->src; + rctx->dst = areq->dst; + rctx->ctrl_arg = ENCRYPT; + ret = nitrox_set_creq(rctx); + if (ret) + return ret; + + /* send the crypto request */ + return nitrox_process_se_request(nctx->ndev, creq, nitrox_aead_callback, + areq); +} + +static int nitrox_aes_gcm_dec(struct aead_request *areq) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(areq); + struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); + struct nitrox_aead_rctx *rctx = aead_request_ctx(areq); + struct se_crypto_request *creq = &rctx->nkreq.creq; + struct flexi_crypto_context *fctx = nctx->u.fctx; + int ret; + + if (!nitrox_aes_gcm_assoclen_supported(areq->assoclen)) + return -EINVAL; + + memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE); + + rctx->cryptlen = areq->cryptlen - aead->authsize; + rctx->assoclen = areq->assoclen; + rctx->srclen = areq->cryptlen + areq->assoclen; + rctx->dstlen = rctx->srclen - aead->authsize; + rctx->iv = &areq->iv[GCM_AES_SALT_SIZE]; + rctx->ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE; + rctx->flags = areq->base.flags; + rctx->ctx_handle = nctx->u.ctx_handle; + rctx->src = areq->src; + rctx->dst = areq->dst; + rctx->ctrl_arg = DECRYPT; + ret = nitrox_set_creq(rctx); + if (ret) + return ret; + + /* send the crypto request */ + return nitrox_process_se_request(nctx->ndev, creq, nitrox_aead_callback, + areq); +} + +static int nitrox_aead_init(struct crypto_aead *aead) +{ + struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); + struct crypto_ctx_hdr *chdr; + + /* get the first device */ + nctx->ndev = nitrox_get_first_device(); + if (!nctx->ndev) + return -ENODEV; + + /* allocate nitrox crypto context */ + chdr = crypto_alloc_context(nctx->ndev); + if (!chdr) { + nitrox_put_device(nctx->ndev); + return -ENOMEM; + } + nctx->chdr = chdr; + nctx->u.ctx_handle = (uintptr_t)((u8 *)chdr->vaddr + + sizeof(struct ctx_hdr)); + nctx->u.fctx->flags.f = 0; + + return 0; +} + +static int nitrox_gcm_common_init(struct crypto_aead *aead) +{ + int ret; + struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); + union fc_ctx_flags *flags; + + ret = nitrox_aead_init(aead); + if (ret) + return ret; + + flags = &nctx->u.fctx->flags; + flags->w0.cipher_type = CIPHER_AES_GCM; + flags->w0.hash_type = AUTH_NULL; + flags->w0.iv_source = IV_FROM_DPTR; + /* ask microcode to calculate ipad/opad */ + flags->w0.auth_input_type = 1; + flags->f = cpu_to_be64(flags->fu); + + return 0; +} + +static int nitrox_aes_gcm_init(struct crypto_aead *aead) +{ + int ret; + + ret = nitrox_gcm_common_init(aead); + if (ret) + return ret; + + crypto_aead_set_reqsize(aead, + sizeof(struct aead_request) + + sizeof(struct nitrox_aead_rctx)); + + return 0; +} + +static void nitrox_aead_exit(struct crypto_aead *aead) +{ + struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); + + /* free the nitrox crypto context */ + if (nctx->u.ctx_handle) { + struct flexi_crypto_context *fctx = nctx->u.fctx; + + memzero_explicit(&fctx->crypto, sizeof(struct crypto_keys)); + memzero_explicit(&fctx->auth, sizeof(struct auth_keys)); + crypto_free_context((void *)nctx->chdr); + } + nitrox_put_device(nctx->ndev); + + nctx->u.ctx_handle = 0; + nctx->ndev = NULL; +} + +static int nitrox_rfc4106_setkey(struct crypto_aead *aead, const u8 *key, + unsigned int keylen) +{ + struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); + struct flexi_crypto_context *fctx = nctx->u.fctx; + int ret; + + if (keylen < GCM_AES_SALT_SIZE) + return -EINVAL; + + keylen -= GCM_AES_SALT_SIZE; + ret = nitrox_aes_gcm_setkey(aead, key, keylen); + if (ret) + return ret; + + memcpy(fctx->crypto.iv, key + keylen, GCM_AES_SALT_SIZE); + return 0; +} + +static int nitrox_rfc4106_setauthsize(struct crypto_aead *aead, + unsigned int authsize) +{ + switch (authsize) { + case 8: + case 12: + case 16: + break; + default: + return -EINVAL; + } + + return nitrox_aead_setauthsize(aead, authsize); +} + +static int nitrox_rfc4106_set_aead_rctx_sglist(struct aead_request *areq) +{ + struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq); + struct nitrox_aead_rctx *aead_rctx = &rctx->base; + unsigned int assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE; + struct scatterlist *sg; + + if (areq->assoclen != 16 && areq->assoclen != 20) + return -EINVAL; + + scatterwalk_map_and_copy(rctx->assoc, areq->src, 0, assoclen, 0); + sg_init_table(rctx->src, 3); + sg_set_buf(rctx->src, rctx->assoc, assoclen); + sg = scatterwalk_ffwd(rctx->src + 1, areq->src, areq->assoclen); + if (sg != rctx->src + 1) + sg_chain(rctx->src, 2, sg); + + if (areq->src != areq->dst) { + sg_init_table(rctx->dst, 3); + sg_set_buf(rctx->dst, rctx->assoc, assoclen); + sg = scatterwalk_ffwd(rctx->dst + 1, areq->dst, areq->assoclen); + if (sg != rctx->dst + 1) + sg_chain(rctx->dst, 2, sg); + } + + aead_rctx->src = rctx->src; + aead_rctx->dst = (areq->src == areq->dst) ? rctx->src : rctx->dst; + + return 0; +} + +static void nitrox_rfc4106_callback(void *arg, int err) +{ + struct aead_request *areq = arg; + struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq); + struct nitrox_kcrypt_request *nkreq = &rctx->base.nkreq; + + free_src_sglist(nkreq); + free_dst_sglist(nkreq); + if (err) { + pr_err_ratelimited("request failed status 0x%0x\n", err); + err = -EINVAL; + } + + areq->base.complete(&areq->base, err); +} + +static int nitrox_rfc4106_enc(struct aead_request *areq) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(areq); + struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); + struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq); + struct nitrox_aead_rctx *aead_rctx = &rctx->base; + struct se_crypto_request *creq = &aead_rctx->nkreq.creq; + int ret; + + aead_rctx->cryptlen = areq->cryptlen; + aead_rctx->assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE; + aead_rctx->srclen = aead_rctx->assoclen + aead_rctx->cryptlen; + aead_rctx->dstlen = aead_rctx->srclen + aead->authsize; + aead_rctx->iv = areq->iv; + aead_rctx->ivsize = GCM_RFC4106_IV_SIZE; + aead_rctx->flags = areq->base.flags; + aead_rctx->ctx_handle = nctx->u.ctx_handle; + aead_rctx->ctrl_arg = ENCRYPT; + + ret = nitrox_rfc4106_set_aead_rctx_sglist(areq); + if (ret) + return ret; + + ret = nitrox_set_creq(aead_rctx); + if (ret) + return ret; + + /* send the crypto request */ + return nitrox_process_se_request(nctx->ndev, creq, + nitrox_rfc4106_callback, areq); +} + +static int nitrox_rfc4106_dec(struct aead_request *areq) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(areq); + struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); + struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq); + struct nitrox_aead_rctx *aead_rctx = &rctx->base; + struct se_crypto_request *creq = &aead_rctx->nkreq.creq; + int ret; + + aead_rctx->cryptlen = areq->cryptlen - aead->authsize; + aead_rctx->assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE; + aead_rctx->srclen = + areq->cryptlen - GCM_RFC4106_IV_SIZE + areq->assoclen; + aead_rctx->dstlen = aead_rctx->srclen - aead->authsize; + aead_rctx->iv = areq->iv; + aead_rctx->ivsize = GCM_RFC4106_IV_SIZE; + aead_rctx->flags = areq->base.flags; + aead_rctx->ctx_handle = nctx->u.ctx_handle; + aead_rctx->ctrl_arg = DECRYPT; + + ret = nitrox_rfc4106_set_aead_rctx_sglist(areq); + if (ret) + return ret; + + ret = nitrox_set_creq(aead_rctx); + if (ret) + return ret; + + /* send the crypto request */ + return nitrox_process_se_request(nctx->ndev, creq, + nitrox_rfc4106_callback, areq); +} + +static int nitrox_rfc4106_init(struct crypto_aead *aead) +{ + int ret; + + ret = nitrox_gcm_common_init(aead); + if (ret) + return ret; + + crypto_aead_set_reqsize(aead, sizeof(struct aead_request) + + sizeof(struct nitrox_rfc4106_rctx)); + + return 0; +} + +static struct aead_alg nitrox_aeads[] = { { + .base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "n5_aes_gcm", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .setkey = nitrox_aes_gcm_setkey, + .setauthsize = nitrox_aes_gcm_setauthsize, + .encrypt = nitrox_aes_gcm_enc, + .decrypt = nitrox_aes_gcm_dec, + .init = nitrox_aes_gcm_init, + .exit = nitrox_aead_exit, + .ivsize = GCM_AES_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, +}, { + .base = { + .cra_name = "rfc4106(gcm(aes))", + .cra_driver_name = "n5_rfc4106", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .setkey = nitrox_rfc4106_setkey, + .setauthsize = nitrox_rfc4106_setauthsize, + .encrypt = nitrox_rfc4106_enc, + .decrypt = nitrox_rfc4106_dec, + .init = nitrox_rfc4106_init, + .exit = nitrox_aead_exit, + .ivsize = GCM_RFC4106_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, +} }; + +int nitrox_register_aeads(void) +{ + return crypto_register_aeads(nitrox_aeads, ARRAY_SIZE(nitrox_aeads)); +} + +void nitrox_unregister_aeads(void) +{ + crypto_unregister_aeads(nitrox_aeads, ARRAY_SIZE(nitrox_aeads)); +} diff --git a/drivers/crypto/cavium/nitrox/nitrox_algs.c b/drivers/crypto/cavium/nitrox/nitrox_algs.c new file mode 100644 index 000000000..d646ae5f2 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_algs.c @@ -0,0 +1,24 @@ +#include "nitrox_common.h" + +int nitrox_crypto_register(void) +{ + int err; + + err = nitrox_register_skciphers(); + if (err) + return err; + + err = nitrox_register_aeads(); + if (err) { + nitrox_unregister_skciphers(); + return err; + } + + return 0; +} + +void nitrox_crypto_unregister(void) +{ + nitrox_unregister_aeads(); + nitrox_unregister_skciphers(); +} diff --git a/drivers/crypto/cavium/nitrox/nitrox_common.h b/drivers/crypto/cavium/nitrox/nitrox_common.h new file mode 100644 index 000000000..e4be69d7e --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_common.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __NITROX_COMMON_H +#define __NITROX_COMMON_H + +#include "nitrox_dev.h" +#include "nitrox_req.h" + +int nitrox_crypto_register(void); +void nitrox_crypto_unregister(void); +int nitrox_register_aeads(void); +void nitrox_unregister_aeads(void); +int nitrox_register_skciphers(void); +void nitrox_unregister_skciphers(void); +void *crypto_alloc_context(struct nitrox_device *ndev); +void crypto_free_context(void *ctx); +struct nitrox_device *nitrox_get_first_device(void); +void nitrox_put_device(struct nitrox_device *ndev); + +int nitrox_common_sw_init(struct nitrox_device *ndev); +void nitrox_common_sw_cleanup(struct nitrox_device *ndev); + +void pkt_slc_resp_tasklet(unsigned long data); +int nitrox_process_se_request(struct nitrox_device *ndev, + struct se_crypto_request *req, + completion_t cb, + void *cb_arg); +void backlog_qflush_work(struct work_struct *work); + + +#endif /* __NITROX_COMMON_H */ diff --git a/drivers/crypto/cavium/nitrox/nitrox_csr.h b/drivers/crypto/cavium/nitrox/nitrox_csr.h new file mode 100644 index 000000000..1c8715ae0 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_csr.h @@ -0,0 +1,1439 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __NITROX_CSR_H +#define __NITROX_CSR_H + +#include <asm/byteorder.h> +#include <linux/types.h> + +/* EMU clusters */ +#define NR_CLUSTERS 4 +/* Maximum cores per cluster, + * varies based on partname + */ +#define AE_CORES_PER_CLUSTER 20 +#define SE_CORES_PER_CLUSTER 16 + +#define AE_MAX_CORES (AE_CORES_PER_CLUSTER * NR_CLUSTERS) +#define SE_MAX_CORES (SE_CORES_PER_CLUSTER * NR_CLUSTERS) +#define ZIP_MAX_CORES 5 + +/* BIST registers */ +#define EMU_BIST_STATUSX(_i) (0x1402700 + ((_i) * 0x40000)) +#define UCD_BIST_STATUS 0x12C0070 +#define NPS_CORE_BIST_REG 0x10000E8 +#define NPS_CORE_NPC_BIST_REG 0x1000128 +#define NPS_PKT_SLC_BIST_REG 0x1040088 +#define NPS_PKT_IN_BIST_REG 0x1040100 +#define POM_BIST_REG 0x11C0100 +#define BMI_BIST_REG 0x1140080 +#define EFL_CORE_BIST_REGX(_i) (0x1240100 + ((_i) * 0x400)) +#define EFL_TOP_BIST_STAT 0x1241090 +#define BMO_BIST_REG 0x1180080 +#define LBC_BIST_STATUS 0x1200020 +#define PEM_BIST_STATUSX(_i) (0x1080468 | ((_i) << 18)) + +/* EMU registers */ +#define EMU_SE_ENABLEX(_i) (0x1400000 + ((_i) * 0x40000)) +#define EMU_AE_ENABLEX(_i) (0x1400008 + ((_i) * 0x40000)) +#define EMU_WD_INT_ENA_W1SX(_i) (0x1402318 + ((_i) * 0x40000)) +#define EMU_GE_INT_ENA_W1SX(_i) (0x1402518 + ((_i) * 0x40000)) +#define EMU_FUSE_MAPX(_i) (0x1402708 + ((_i) * 0x40000)) + +/* UCD registers */ +#define UCD_SE_EID_UCODE_BLOCK_NUMX(_i) (0x12C0000 + ((_i) * 0x1000)) +#define UCD_AE_EID_UCODE_BLOCK_NUMX(_i) (0x12C0008 + ((_i) * 0x800)) +#define UCD_UCODE_LOAD_BLOCK_NUM 0x12C0010 +#define UCD_UCODE_LOAD_IDX_DATAX(_i) (0x12C0018 + ((_i) * 0x20)) +#define UCD_SE_CNTX(_i) (0x12C0040 + ((_i) * 0x1000)) +#define UCD_AE_CNTX(_i) (0x12C0048 + ((_i) * 0x800)) + +/* AQM registers */ +#define AQM_CTL 0x1300000 +#define AQM_INT 0x1300008 +#define AQM_DBELL_OVF_LO 0x1300010 +#define AQM_DBELL_OVF_HI 0x1300018 +#define AQM_DBELL_OVF_LO_W1S 0x1300020 +#define AQM_DBELL_OVF_LO_ENA_W1C 0x1300028 +#define AQM_DBELL_OVF_LO_ENA_W1S 0x1300030 +#define AQM_DBELL_OVF_HI_W1S 0x1300038 +#define AQM_DBELL_OVF_HI_ENA_W1C 0x1300040 +#define AQM_DBELL_OVF_HI_ENA_W1S 0x1300048 +#define AQM_DMA_RD_ERR_LO 0x1300050 +#define AQM_DMA_RD_ERR_HI 0x1300058 +#define AQM_DMA_RD_ERR_LO_W1S 0x1300060 +#define AQM_DMA_RD_ERR_LO_ENA_W1C 0x1300068 +#define AQM_DMA_RD_ERR_LO_ENA_W1S 0x1300070 +#define AQM_DMA_RD_ERR_HI_W1S 0x1300078 +#define AQM_DMA_RD_ERR_HI_ENA_W1C 0x1300080 +#define AQM_DMA_RD_ERR_HI_ENA_W1S 0x1300088 +#define AQM_EXEC_NA_LO 0x1300090 +#define AQM_EXEC_NA_HI 0x1300098 +#define AQM_EXEC_NA_LO_W1S 0x13000A0 +#define AQM_EXEC_NA_LO_ENA_W1C 0x13000A8 +#define AQM_EXEC_NA_LO_ENA_W1S 0x13000B0 +#define AQM_EXEC_NA_HI_W1S 0x13000B8 +#define AQM_EXEC_NA_HI_ENA_W1C 0x13000C0 +#define AQM_EXEC_NA_HI_ENA_W1S 0x13000C8 +#define AQM_EXEC_ERR_LO 0x13000D0 +#define AQM_EXEC_ERR_HI 0x13000D8 +#define AQM_EXEC_ERR_LO_W1S 0x13000E0 +#define AQM_EXEC_ERR_LO_ENA_W1C 0x13000E8 +#define AQM_EXEC_ERR_LO_ENA_W1S 0x13000F0 +#define AQM_EXEC_ERR_HI_W1S 0x13000F8 +#define AQM_EXEC_ERR_HI_ENA_W1C 0x1300100 +#define AQM_EXEC_ERR_HI_ENA_W1S 0x1300108 +#define AQM_ECC_INT 0x1300110 +#define AQM_ECC_INT_W1S 0x1300118 +#define AQM_ECC_INT_ENA_W1C 0x1300120 +#define AQM_ECC_INT_ENA_W1S 0x1300128 +#define AQM_ECC_CTL 0x1300130 +#define AQM_BIST_STATUS 0x1300138 +#define AQM_CMD_INF_THRX(x) (0x1300400 + ((x) * 0x8)) +#define AQM_CMD_INFX(x) (0x1300800 + ((x) * 0x8)) +#define AQM_GRP_EXECMSK_LOX(x) (0x1300C00 + ((x) * 0x10)) +#define AQM_GRP_EXECMSK_HIX(x) (0x1300C08 + ((x) * 0x10)) +#define AQM_ACTIVITY_STAT_LO 0x1300C80 +#define AQM_ACTIVITY_STAT_HI 0x1300C88 +#define AQM_Q_CMD_PROCX(x) (0x1301000 + ((x) * 0x8)) +#define AQM_PERF_CTL_LO 0x1301400 +#define AQM_PERF_CTL_HI 0x1301408 +#define AQM_PERF_CNT 0x1301410 + +#define AQMQ_DRBLX(x) (0x20000 + ((x) * 0x40000)) +#define AQMQ_QSZX(x) (0x20008 + ((x) * 0x40000)) +#define AQMQ_BADRX(x) (0x20010 + ((x) * 0x40000)) +#define AQMQ_NXT_CMDX(x) (0x20018 + ((x) * 0x40000)) +#define AQMQ_CMD_CNTX(x) (0x20020 + ((x) * 0x40000)) +#define AQMQ_CMP_THRX(x) (0x20028 + ((x) * 0x40000)) +#define AQMQ_CMP_CNTX(x) (0x20030 + ((x) * 0x40000)) +#define AQMQ_TIM_LDX(x) (0x20038 + ((x) * 0x40000)) +#define AQMQ_TIMERX(x) (0x20040 + ((x) * 0x40000)) +#define AQMQ_ENX(x) (0x20048 + ((x) * 0x40000)) +#define AQMQ_ACTIVITY_STATX(x) (0x20050 + ((x) * 0x40000)) +#define AQM_VF_CMP_STATX(x) (0x28000 + ((x) * 0x40000)) + +/* NPS core registers */ +#define NPS_CORE_GBL_VFCFG 0x1000000 +#define NPS_CORE_CONTROL 0x1000008 +#define NPS_CORE_INT_ACTIVE 0x1000080 +#define NPS_CORE_INT 0x10000A0 +#define NPS_CORE_INT_ENA_W1S 0x10000B8 +#define NPS_STATS_PKT_DMA_RD_CNT 0x1000180 +#define NPS_STATS_PKT_DMA_WR_CNT 0x1000190 + +/* NPS packet registers */ +#define NPS_PKT_INT 0x1040018 +#define NPS_PKT_MBOX_INT_LO 0x1040020 +#define NPS_PKT_MBOX_INT_LO_ENA_W1C 0x1040030 +#define NPS_PKT_MBOX_INT_LO_ENA_W1S 0x1040038 +#define NPS_PKT_MBOX_INT_HI 0x1040040 +#define NPS_PKT_MBOX_INT_HI_ENA_W1C 0x1040050 +#define NPS_PKT_MBOX_INT_HI_ENA_W1S 0x1040058 +#define NPS_PKT_IN_RERR_HI 0x1040108 +#define NPS_PKT_IN_RERR_HI_ENA_W1S 0x1040120 +#define NPS_PKT_IN_RERR_LO 0x1040128 +#define NPS_PKT_IN_RERR_LO_ENA_W1S 0x1040140 +#define NPS_PKT_IN_ERR_TYPE 0x1040148 +#define NPS_PKT_IN_ERR_TYPE_ENA_W1S 0x1040160 +#define NPS_PKT_IN_INSTR_CTLX(_i) (0x10060 + ((_i) * 0x40000)) +#define NPS_PKT_IN_INSTR_BADDRX(_i) (0x10068 + ((_i) * 0x40000)) +#define NPS_PKT_IN_INSTR_RSIZEX(_i) (0x10070 + ((_i) * 0x40000)) +#define NPS_PKT_IN_DONE_CNTSX(_i) (0x10080 + ((_i) * 0x40000)) +#define NPS_PKT_IN_INSTR_BAOFF_DBELLX(_i) (0x10078 + ((_i) * 0x40000)) +#define NPS_PKT_IN_INT_LEVELSX(_i) (0x10088 + ((_i) * 0x40000)) + +#define NPS_PKT_SLC_RERR_HI 0x1040208 +#define NPS_PKT_SLC_RERR_HI_ENA_W1S 0x1040220 +#define NPS_PKT_SLC_RERR_LO 0x1040228 +#define NPS_PKT_SLC_RERR_LO_ENA_W1S 0x1040240 +#define NPS_PKT_SLC_ERR_TYPE 0x1040248 +#define NPS_PKT_SLC_ERR_TYPE_ENA_W1S 0x1040260 +/* Mailbox PF->VF PF Accessible Data registers */ +#define NPS_PKT_MBOX_PF_VF_PFDATAX(_i) (0x1040800 + ((_i) * 0x8)) +#define NPS_PKT_MBOX_VF_PF_PFDATAX(_i) (0x1040C00 + ((_i) * 0x8)) + +#define NPS_PKT_SLC_CTLX(_i) (0x10000 + ((_i) * 0x40000)) +#define NPS_PKT_SLC_CNTSX(_i) (0x10008 + ((_i) * 0x40000)) +#define NPS_PKT_SLC_INT_LEVELSX(_i) (0x10010 + ((_i) * 0x40000)) + +/* POM registers */ +#define POM_INT_ENA_W1S 0x11C0018 +#define POM_GRP_EXECMASKX(_i) (0x11C1100 | ((_i) * 8)) +#define POM_INT 0x11C0000 +#define POM_PERF_CTL 0x11CC400 + +/* BMI registers */ +#define BMI_INT 0x1140000 +#define BMI_CTL 0x1140020 +#define BMI_INT_ENA_W1S 0x1140018 +#define BMI_NPS_PKT_CNT 0x1140070 + +/* EFL registers */ +#define EFL_CORE_INT_ENA_W1SX(_i) (0x1240018 + ((_i) * 0x400)) +#define EFL_CORE_VF_ERR_INT0X(_i) (0x1240050 + ((_i) * 0x400)) +#define EFL_CORE_VF_ERR_INT0_ENA_W1SX(_i) (0x1240068 + ((_i) * 0x400)) +#define EFL_CORE_VF_ERR_INT1X(_i) (0x1240070 + ((_i) * 0x400)) +#define EFL_CORE_VF_ERR_INT1_ENA_W1SX(_i) (0x1240088 + ((_i) * 0x400)) +#define EFL_CORE_SE_ERR_INTX(_i) (0x12400A0 + ((_i) * 0x400)) +#define EFL_RNM_CTL_STATUS 0x1241800 +#define EFL_CORE_INTX(_i) (0x1240000 + ((_i) * 0x400)) + +/* BMO registers */ +#define BMO_CTL2 0x1180028 +#define BMO_NPS_SLC_PKT_CNT 0x1180078 + +/* LBC registers */ +#define LBC_INT 0x1200000 +#define LBC_INVAL_CTL 0x1201010 +#define LBC_PLM_VF1_64_INT 0x1202008 +#define LBC_INVAL_STATUS 0x1202010 +#define LBC_INT_ENA_W1S 0x1203000 +#define LBC_PLM_VF1_64_INT_ENA_W1S 0x1205008 +#define LBC_PLM_VF65_128_INT 0x1206008 +#define LBC_ELM_VF1_64_INT 0x1208000 +#define LBC_PLM_VF65_128_INT_ENA_W1S 0x1209008 +#define LBC_ELM_VF1_64_INT_ENA_W1S 0x120B000 +#define LBC_ELM_VF65_128_INT 0x120C000 +#define LBC_ELM_VF65_128_INT_ENA_W1S 0x120F000 + +#define RST_BOOT 0x10C1600 +#define FUS_DAT1 0x10C1408 + +/* PEM registers */ +#define PEM0_INT 0x1080428 + +/** + * struct ucd_core_eid_ucode_block_num - Core Eid to Ucode Blk Mapping Registers + * @ucode_len: Ucode length identifier 32KB or 64KB + * @ucode_blk: Ucode Block Number + */ +union ucd_core_eid_ucode_block_num { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_4_63 : 60; + u64 ucode_len : 1; + u64 ucode_blk : 3; +#else + u64 ucode_blk : 3; + u64 ucode_len : 1; + u64 raz_4_63 : 60; +#endif + }; +}; + +/** + * struct aqm_grp_execmsk_lo - Available AE engines for the group + * @exec_0_to_39: AE engines 0 to 39 status + */ +union aqm_grp_execmsk_lo { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_40_63 : 24; + u64 exec_0_to_39 : 40; +#else + u64 exec_0_to_39 : 40; + u64 raz_40_63 : 24; +#endif + }; +}; + +/** + * struct aqm_grp_execmsk_hi - Available AE engines for the group + * @exec_40_to_79: AE engines 40 to 79 status + */ +union aqm_grp_execmsk_hi { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_40_63 : 24; + u64 exec_40_to_79 : 40; +#else + u64 exec_40_to_79 : 40; + u64 raz_40_63 : 24; +#endif + }; +}; + +/** + * struct aqmq_drbl - AQM Queue Doorbell Counter Registers + * @dbell_count: Doorbell Counter + */ +union aqmq_drbl { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_32_63 : 32; + u64 dbell_count : 32; +#else + u64 dbell_count : 32; + u64 raz_32_63 : 32; +#endif + }; +}; + +/** + * struct aqmq_qsz - AQM Queue Host Queue Size Registers + * @host_queue_size: Size, in numbers of 'aqmq_command_s' command + * of the Host Ring. + */ +union aqmq_qsz { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_32_63 : 32; + u64 host_queue_size : 32; +#else + u64 host_queue_size : 32; + u64 raz_32_63 : 32; +#endif + }; +}; + +/** + * struct aqmq_cmp_thr - AQM Queue Commands Completed Threshold Registers + * @commands_completed_threshold: Count of 'aqmq_command_s' commands executed + * by AE engines for which completion interrupt is asserted. + */ +union aqmq_cmp_thr { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_32_63 : 32; + u64 commands_completed_threshold : 32; +#else + u64 commands_completed_threshold : 32; + u64 raz_32_63 : 32; +#endif + }; +}; + +/** + * struct aqmq_cmp_cnt - AQM Queue Commands Completed Count Registers + * @resend: Bit to request completion interrupt Resend. + * @completion_status: Command completion status of the ring. + * @commands_completed_count: Count of 'aqmq_command_s' commands executed by + * AE engines. + */ +union aqmq_cmp_cnt { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_34_63 : 30; + u64 resend : 1; + u64 completion_status : 1; + u64 commands_completed_count : 32; +#else + u64 commands_completed_count : 32; + u64 completion_status : 1; + u64 resend : 1; + u64 raz_34_63 : 30; +#endif + }; +}; + +/** + * struct aqmq_en - AQM Queue Enable Registers + * @queue_status: 1 = AQMQ is enabled, 0 = AQMQ is disabled + */ +union aqmq_en { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_1_63 : 63; + u64 queue_enable : 1; +#else + u64 queue_enable : 1; + u64 raz_1_63 : 63; +#endif + }; +}; + +/** + * struct aqmq_activity_stat - AQM Queue Activity Status Registers + * @queue_active: 1 = AQMQ is active, 0 = AQMQ is quiescent + */ +union aqmq_activity_stat { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_1_63 : 63; + u64 queue_active : 1; +#else + u64 queue_active : 1; + u64 raz_1_63 : 63; +#endif + }; +}; + +/** + * struct emu_fuse_map - EMU Fuse Map Registers + * @ae_fuse: Fuse settings for AE 19..0 + * @se_fuse: Fuse settings for SE 15..0 + * + * A set bit indicates the unit is fuse disabled. + */ +union emu_fuse_map { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 valid : 1; + u64 raz_52_62 : 11; + u64 ae_fuse : 20; + u64 raz_16_31 : 16; + u64 se_fuse : 16; +#else + u64 se_fuse : 16; + u64 raz_16_31 : 16; + u64 ae_fuse : 20; + u64 raz_52_62 : 11; + u64 valid : 1; +#endif + } s; +}; + +/** + * struct emu_se_enable - Symmetric Engine Enable Registers + * @enable: Individual enables for each of the clusters + * 16 symmetric engines. + */ +union emu_se_enable { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz : 48; + u64 enable : 16; +#else + u64 enable : 16; + u64 raz : 48; +#endif + } s; +}; + +/** + * struct emu_ae_enable - EMU Asymmetric engines. + * @enable: Individual enables for each of the cluster's + * 20 Asymmetric Engines. + */ +union emu_ae_enable { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz : 44; + u64 enable : 20; +#else + u64 enable : 20; + u64 raz : 44; +#endif + } s; +}; + +/** + * struct emu_wd_int_ena_w1s - EMU Interrupt Enable Registers + * @ae_wd: Reads or sets enable for EMU(0..3)_WD_INT[AE_WD] + * @se_wd: Reads or sets enable for EMU(0..3)_WD_INT[SE_WD] + */ +union emu_wd_int_ena_w1s { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz2 : 12; + u64 ae_wd : 20; + u64 raz1 : 16; + u64 se_wd : 16; +#else + u64 se_wd : 16; + u64 raz1 : 16; + u64 ae_wd : 20; + u64 raz2 : 12; +#endif + } s; +}; + +/** + * struct emu_ge_int_ena_w1s - EMU Interrupt Enable set registers + * @ae_ge: Reads or sets enable for EMU(0..3)_GE_INT[AE_GE] + * @se_ge: Reads or sets enable for EMU(0..3)_GE_INT[SE_GE] + */ +union emu_ge_int_ena_w1s { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_52_63 : 12; + u64 ae_ge : 20; + u64 raz_16_31: 16; + u64 se_ge : 16; +#else + u64 se_ge : 16; + u64 raz_16_31: 16; + u64 ae_ge : 20; + u64 raz_52_63 : 12; +#endif + } s; +}; + +/** + * struct nps_pkt_slc_ctl - Solicited Packet Out Control Registers + * @rh: Indicates whether to remove or include the response header + * 1 = Include, 0 = Remove + * @z: If set, 8 trailing 0x00 bytes will be added to the end of the + * outgoing packet. + * @enb: Enable for this port. + */ +union nps_pkt_slc_ctl { + u64 value; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 raz : 61; + u64 rh : 1; + u64 z : 1; + u64 enb : 1; +#else + u64 enb : 1; + u64 z : 1; + u64 rh : 1; + u64 raz : 61; +#endif + } s; +}; + +/** + * struct nps_pkt_slc_cnts - Solicited Packet Out Count Registers + * @slc_int: Returns a 1 when: + * NPS_PKT_SLC(i)_CNTS[CNT] > NPS_PKT_SLC(i)_INT_LEVELS[CNT], or + * NPS_PKT_SLC(i)_CNTS[TIMER] > NPS_PKT_SLC(i)_INT_LEVELS[TIMET]. + * To clear the bit, the CNTS register must be written to clear. + * @in_int: Returns a 1 when: + * NPS_PKT_IN(i)_DONE_CNTS[CNT] > NPS_PKT_IN(i)_INT_LEVELS[CNT]. + * To clear the bit, the DONE_CNTS register must be written to clear. + * @mbox_int: Returns a 1 when: + * NPS_PKT_MBOX_PF_VF(i)_INT[INTR] is set. To clear the bit, + * write NPS_PKT_MBOX_PF_VF(i)_INT[INTR] with 1. + * @timer: Timer, incremented every 2048 coprocessor clock cycles + * when [CNT] is not zero. The hardware clears both [TIMER] and + * [INT] when [CNT] goes to 0. + * @cnt: Packet counter. Hardware adds to [CNT] as it sends packets out. + * On a write to this CSR, hardware subtracts the amount written to the + * [CNT] field from [CNT]. + */ +union nps_pkt_slc_cnts { + u64 value; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 slc_int : 1; + u64 uns_int : 1; + u64 in_int : 1; + u64 mbox_int : 1; + u64 resend : 1; + u64 raz : 5; + u64 timer : 22; + u64 cnt : 32; +#else + u64 cnt : 32; + u64 timer : 22; + u64 raz : 5; + u64 resend : 1; + u64 mbox_int : 1; + u64 in_int : 1; + u64 uns_int : 1; + u64 slc_int : 1; +#endif + } s; +}; + +/** + * struct nps_pkt_slc_int_levels - Solicited Packet Out Interrupt Levels + * Registers. + * @bmode: Determines whether NPS_PKT_SLC_CNTS[CNT] is a byte or + * packet counter. + * @timet: Output port counter time interrupt threshold. + * @cnt: Output port counter interrupt threshold. + */ +union nps_pkt_slc_int_levels { + u64 value; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 bmode : 1; + u64 raz : 9; + u64 timet : 22; + u64 cnt : 32; +#else + u64 cnt : 32; + u64 timet : 22; + u64 raz : 9; + u64 bmode : 1; +#endif + } s; +}; + +/** + * struct nps_pkt_inst - NPS Packet Interrupt Register + * @in_err: Set when any NPS_PKT_IN_RERR_HI/LO bit and + * corresponding NPS_PKT_IN_RERR_*_ENA_* bit are bot set. + * @uns_err: Set when any NSP_PKT_UNS_RERR_HI/LO bit and + * corresponding NPS_PKT_UNS_RERR_*_ENA_* bit are both set. + * @slc_er: Set when any NSP_PKT_SLC_RERR_HI/LO bit and + * corresponding NPS_PKT_SLC_RERR_*_ENA_* bit are both set. + */ +union nps_pkt_int { + u64 value; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 raz : 54; + u64 uns_wto : 1; + u64 in_err : 1; + u64 uns_err : 1; + u64 slc_err : 1; + u64 in_dbe : 1; + u64 in_sbe : 1; + u64 uns_dbe : 1; + u64 uns_sbe : 1; + u64 slc_dbe : 1; + u64 slc_sbe : 1; +#else + u64 slc_sbe : 1; + u64 slc_dbe : 1; + u64 uns_sbe : 1; + u64 uns_dbe : 1; + u64 in_sbe : 1; + u64 in_dbe : 1; + u64 slc_err : 1; + u64 uns_err : 1; + u64 in_err : 1; + u64 uns_wto : 1; + u64 raz : 54; +#endif + } s; +}; + +/** + * struct nps_pkt_in_done_cnts - Input instruction ring counts registers + * @slc_cnt: Returns a 1 when: + * NPS_PKT_SLC(i)_CNTS[CNT] > NPS_PKT_SLC(i)_INT_LEVELS[CNT], or + * NPS_PKT_SLC(i)_CNTS[TIMER] > NPS_PKT_SCL(i)_INT_LEVELS[TIMET] + * To clear the bit, the CNTS register must be + * written to clear the underlying condition + * @uns_int: Return a 1 when: + * NPS_PKT_UNS(i)_CNTS[CNT] > NPS_PKT_UNS(i)_INT_LEVELS[CNT], or + * NPS_PKT_UNS(i)_CNTS[TIMER] > NPS_PKT_UNS(i)_INT_LEVELS[TIMET] + * To clear the bit, the CNTS register must be + * written to clear the underlying condition + * @in_int: Returns a 1 when: + * NPS_PKT_IN(i)_DONE_CNTS[CNT] > NPS_PKT_IN(i)_INT_LEVELS[CNT] + * To clear the bit, the DONE_CNTS register + * must be written to clear the underlying condition + * @mbox_int: Returns a 1 when: + * NPS_PKT_MBOX_PF_VF(i)_INT[INTR] is set. + * To clear the bit, write NPS_PKT_MBOX_PF_VF(i)_INT[INTR] + * with 1. + * @resend: A write of 1 will resend an MSI-X interrupt message if any + * of the following conditions are true for this ring "i". + * NPS_PKT_SLC(i)_CNTS[CNT] > NPS_PKT_SLC(i)_INT_LEVELS[CNT] + * NPS_PKT_SLC(i)_CNTS[TIMER] > NPS_PKT_SLC(i)_INT_LEVELS[TIMET] + * NPS_PKT_UNS(i)_CNTS[CNT] > NPS_PKT_UNS(i)_INT_LEVELS[CNT] + * NPS_PKT_UNS(i)_CNTS[TIMER] > NPS_PKT_UNS(i)_INT_LEVELS[TIMET] + * NPS_PKT_IN(i)_DONE_CNTS[CNT] > NPS_PKT_IN(i)_INT_LEVELS[CNT] + * NPS_PKT_MBOX_PF_VF(i)_INT[INTR] is set + * @cnt: Packet counter. Hardware adds to [CNT] as it reads + * packets. On a write to this CSR, hardware substracts the + * amount written to the [CNT] field from [CNT], which will + * clear PKT_IN(i)_INT_STATUS[INTR] if [CNT] becomes <= + * NPS_PKT_IN(i)_INT_LEVELS[CNT]. This register should be + * cleared before enabling a ring by reading the current + * value and writing it back. + */ +union nps_pkt_in_done_cnts { + u64 value; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 slc_int : 1; + u64 uns_int : 1; + u64 in_int : 1; + u64 mbox_int : 1; + u64 resend : 1; + u64 raz : 27; + u64 cnt : 32; +#else + u64 cnt : 32; + u64 raz : 27; + u64 resend : 1; + u64 mbox_int : 1; + u64 in_int : 1; + u64 uns_int : 1; + u64 slc_int : 1; +#endif + } s; +}; + +/** + * struct nps_pkt_in_instr_ctl - Input Instruction Ring Control Registers. + * @is64b: If 1, the ring uses 64-byte instructions. If 0, the + * ring uses 32-byte instructions. + * @enb: Enable for the input ring. + */ +union nps_pkt_in_instr_ctl { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz : 62; + u64 is64b : 1; + u64 enb : 1; +#else + u64 enb : 1; + u64 is64b : 1; + u64 raz : 62; +#endif + } s; +}; + +/** + * struct nps_pkt_in_instr_rsize - Input instruction ring size registers + * @rsize: Ring size (number of instructions) + */ +union nps_pkt_in_instr_rsize { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz : 32; + u64 rsize : 32; +#else + u64 rsize : 32; + u64 raz : 32; +#endif + } s; +}; + +/** + * struct nps_pkt_in_instr_baoff_dbell - Input instruction ring + * base address offset and doorbell registers + * @aoff: Address offset. The offset from the NPS_PKT_IN_INSTR_BADDR + * where the next pointer is read. + * @dbell: Pointer list doorbell count. Write operations to this field + * increments the present value here. Read operations return the + * present value. + */ +union nps_pkt_in_instr_baoff_dbell { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 aoff : 32; + u64 dbell : 32; +#else + u64 dbell : 32; + u64 aoff : 32; +#endif + } s; +}; + +/** + * struct nps_core_int_ena_w1s - NPS core interrupt enable set register + * @host_nps_wr_err: Reads or sets enable for + * NPS_CORE_INT[HOST_NPS_WR_ERR]. + * @npco_dma_malform: Reads or sets enable for + * NPS_CORE_INT[NPCO_DMA_MALFORM]. + * @exec_wr_timeout: Reads or sets enable for + * NPS_CORE_INT[EXEC_WR_TIMEOUT]. + * @host_wr_timeout: Reads or sets enable for + * NPS_CORE_INT[HOST_WR_TIMEOUT]. + * @host_wr_err: Reads or sets enable for + * NPS_CORE_INT[HOST_WR_ERR] + */ +union nps_core_int_ena_w1s { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz4 : 55; + u64 host_nps_wr_err : 1; + u64 npco_dma_malform : 1; + u64 exec_wr_timeout : 1; + u64 host_wr_timeout : 1; + u64 host_wr_err : 1; + u64 raz3 : 1; + u64 raz2 : 1; + u64 raz1 : 1; + u64 raz0 : 1; +#else + u64 raz0 : 1; + u64 raz1 : 1; + u64 raz2 : 1; + u64 raz3 : 1; + u64 host_wr_err : 1; + u64 host_wr_timeout : 1; + u64 exec_wr_timeout : 1; + u64 npco_dma_malform : 1; + u64 host_nps_wr_err : 1; + u64 raz4 : 55; +#endif + } s; +}; + +/** + * struct nps_core_gbl_vfcfg - Global VF Configuration Register. + * @ilk_disable: When set, this bit indicates that the ILK interface has + * been disabled. + * @obaf: BMO allocation control + * 0 = allocate per queue + * 1 = allocate per VF + * @ibaf: BMI allocation control + * 0 = allocate per queue + * 1 = allocate per VF + * @zaf: ZIP allocation control + * 0 = allocate per queue + * 1 = allocate per VF + * @aeaf: AE allocation control + * 0 = allocate per queue + * 1 = allocate per VF + * @seaf: SE allocation control + * 0 = allocation per queue + * 1 = allocate per VF + * @cfg: VF/PF mode. + */ +union nps_core_gbl_vfcfg { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz :55; + u64 ilk_disable :1; + u64 obaf :1; + u64 ibaf :1; + u64 zaf :1; + u64 aeaf :1; + u64 seaf :1; + u64 cfg :3; +#else + u64 cfg :3; + u64 seaf :1; + u64 aeaf :1; + u64 zaf :1; + u64 ibaf :1; + u64 obaf :1; + u64 ilk_disable :1; + u64 raz :55; +#endif + } s; +}; + +/** + * struct nps_core_int_active - NPS Core Interrupt Active Register + * @resend: Resend MSI-X interrupt if needs to handle interrupts + * Sofware can set this bit and then exit the ISR. + * @ocla: Set when any OCLA(0)_INT and corresponding OCLA(0_INT_ENA_W1C + * bit are set + * @mbox: Set when any NPS_PKT_MBOX_INT_LO/HI and corresponding + * NPS_PKT_MBOX_INT_LO_ENA_W1C/HI_ENA_W1C bits are set + * @emu: bit i is set in [EMU] when any EMU(i)_INT bit is set + * @bmo: Set when any BMO_INT bit is set + * @bmi: Set when any BMI_INT bit is set or when any non-RO + * BMI_INT and corresponding BMI_INT_ENA_W1C bits are both set + * @aqm: Set when any AQM_INT bit is set + * @zqm: Set when any ZQM_INT bit is set + * @efl: Set when any EFL_INT RO bit is set or when any non-RO EFL_INT + * and corresponding EFL_INT_ENA_W1C bits are both set + * @ilk: Set when any ILK_INT bit is set + * @lbc: Set when any LBC_INT RO bit is set or when any non-RO LBC_INT + * and corresponding LBC_INT_ENA_W1C bits are bot set + * @pem: Set when any PEM(0)_INT RO bit is set or when any non-RO + * PEM(0)_INT and corresponding PEM(0)_INT_ENA_W1C bit are both set + * @ucd: Set when any UCD_INT bit is set + * @zctl: Set when any ZIP_INT RO bit is set or when any non-RO ZIP_INT + * and corresponding ZIP_INT_ENA_W1C bits are both set + * @lbm: Set when any LBM_INT bit is set + * @nps_pkt: Set when any NPS_PKT_INT bit is set + * @nps_core: Set when any NPS_CORE_INT RO bit is set or when non-RO + * NPS_CORE_INT and corresponding NSP_CORE_INT_ENA_W1C bits are both set + */ +union nps_core_int_active { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 resend : 1; + u64 raz : 43; + u64 ocla : 1; + u64 mbox : 1; + u64 emu : 4; + u64 bmo : 1; + u64 bmi : 1; + u64 aqm : 1; + u64 zqm : 1; + u64 efl : 1; + u64 ilk : 1; + u64 lbc : 1; + u64 pem : 1; + u64 pom : 1; + u64 ucd : 1; + u64 zctl : 1; + u64 lbm : 1; + u64 nps_pkt : 1; + u64 nps_core : 1; +#else + u64 nps_core : 1; + u64 nps_pkt : 1; + u64 lbm : 1; + u64 zctl: 1; + u64 ucd : 1; + u64 pom : 1; + u64 pem : 1; + u64 lbc : 1; + u64 ilk : 1; + u64 efl : 1; + u64 zqm : 1; + u64 aqm : 1; + u64 bmi : 1; + u64 bmo : 1; + u64 emu : 4; + u64 mbox : 1; + u64 ocla : 1; + u64 raz : 43; + u64 resend : 1; +#endif + } s; +}; + +/** + * struct efl_core_int - EFL Interrupt Registers + * @epci_decode_err: EPCI decoded a transacation that was unknown + * This error should only occurred when there is a micrcode/SE error + * and should be considered fatal + * @ae_err: An AE uncorrectable error occurred. + * See EFL_CORE(0..3)_AE_ERR_INT + * @se_err: An SE uncorrectable error occurred. + * See EFL_CORE(0..3)_SE_ERR_INT + * @dbe: Double-bit error occurred in EFL + * @sbe: Single-bit error occurred in EFL + * @d_left: Asserted when new POM-Header-BMI-data is + * being sent to an Exec, and that Exec has Not read all BMI + * data associated with the previous POM header + * @len_ovr: Asserted when an Exec-Read is issued that is more than + * 14 greater in length that the BMI data left to be read + */ +union efl_core_int { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz : 57; + u64 epci_decode_err : 1; + u64 ae_err : 1; + u64 se_err : 1; + u64 dbe : 1; + u64 sbe : 1; + u64 d_left : 1; + u64 len_ovr : 1; +#else + u64 len_ovr : 1; + u64 d_left : 1; + u64 sbe : 1; + u64 dbe : 1; + u64 se_err : 1; + u64 ae_err : 1; + u64 epci_decode_err : 1; + u64 raz : 57; +#endif + } s; +}; + +/** + * struct efl_core_int_ena_w1s - EFL core interrupt enable set register + * @epci_decode_err: Reads or sets enable for + * EFL_CORE(0..3)_INT[EPCI_DECODE_ERR]. + * @d_left: Reads or sets enable for + * EFL_CORE(0..3)_INT[D_LEFT]. + * @len_ovr: Reads or sets enable for + * EFL_CORE(0..3)_INT[LEN_OVR]. + */ +union efl_core_int_ena_w1s { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_7_63 : 57; + u64 epci_decode_err : 1; + u64 raz_2_5 : 4; + u64 d_left : 1; + u64 len_ovr : 1; +#else + u64 len_ovr : 1; + u64 d_left : 1; + u64 raz_2_5 : 4; + u64 epci_decode_err : 1; + u64 raz_7_63 : 57; +#endif + } s; +}; + +/** + * struct efl_rnm_ctl_status - RNM Control and Status Register + * @ent_sel: Select input to RNM FIFO + * @exp_ent: Exported entropy enable for random number generator + * @rng_rst: Reset to RNG. Setting this bit to 1 cancels the generation + * of the current random number. + * @rnm_rst: Reset the RNM. Setting this bit to 1 clears all sorted numbers + * in the random number memory. + * @rng_en: Enabled the output of the RNG. + * @ent_en: Entropy enable for random number generator. + */ +union efl_rnm_ctl_status { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_9_63 : 55; + u64 ent_sel : 4; + u64 exp_ent : 1; + u64 rng_rst : 1; + u64 rnm_rst : 1; + u64 rng_en : 1; + u64 ent_en : 1; +#else + u64 ent_en : 1; + u64 rng_en : 1; + u64 rnm_rst : 1; + u64 rng_rst : 1; + u64 exp_ent : 1; + u64 ent_sel : 4; + u64 raz_9_63 : 55; +#endif + } s; +}; + +/** + * struct bmi_ctl - BMI control register + * @ilk_hdrq_thrsh: Maximum number of header queue locations + * that ILK packets may consume. When the threshold is + * exceeded ILK_XOFF is sent to the BMI_X2P_ARB. + * @nps_hdrq_thrsh: Maximum number of header queue locations + * that NPS packets may consume. When the threshold is + * exceeded NPS_XOFF is sent to the BMI_X2P_ARB. + * @totl_hdrq_thrsh: Maximum number of header queue locations + * that the sum of ILK and NPS packets may consume. + * @ilk_free_thrsh: Maximum number of buffers that ILK packet + * flows may consume before ILK_XOFF is sent to the BMI_X2P_ARB. + * @nps_free_thrsh: Maximum number of buffers that NPS packet + * flows may consume before NPS XOFF is sent to the BMI_X2p_ARB. + * @totl_free_thrsh: Maximum number of buffers that bot ILK and NPS + * packet flows may consume before both NPS_XOFF and ILK_XOFF + * are asserted to the BMI_X2P_ARB. + * @max_pkt_len: Maximum packet length, integral number of 256B + * buffers. + */ +union bmi_ctl { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_56_63 : 8; + u64 ilk_hdrq_thrsh : 8; + u64 nps_hdrq_thrsh : 8; + u64 totl_hdrq_thrsh : 8; + u64 ilk_free_thrsh : 8; + u64 nps_free_thrsh : 8; + u64 totl_free_thrsh : 8; + u64 max_pkt_len : 8; +#else + u64 max_pkt_len : 8; + u64 totl_free_thrsh : 8; + u64 nps_free_thrsh : 8; + u64 ilk_free_thrsh : 8; + u64 totl_hdrq_thrsh : 8; + u64 nps_hdrq_thrsh : 8; + u64 ilk_hdrq_thrsh : 8; + u64 raz_56_63 : 8; +#endif + } s; +}; + +/** + * struct bmi_int_ena_w1s - BMI interrupt enable set register + * @ilk_req_oflw: Reads or sets enable for + * BMI_INT[ILK_REQ_OFLW]. + * @nps_req_oflw: Reads or sets enable for + * BMI_INT[NPS_REQ_OFLW]. + * @fpf_undrrn: Reads or sets enable for + * BMI_INT[FPF_UNDRRN]. + * @eop_err_ilk: Reads or sets enable for + * BMI_INT[EOP_ERR_ILK]. + * @eop_err_nps: Reads or sets enable for + * BMI_INT[EOP_ERR_NPS]. + * @sop_err_ilk: Reads or sets enable for + * BMI_INT[SOP_ERR_ILK]. + * @sop_err_nps: Reads or sets enable for + * BMI_INT[SOP_ERR_NPS]. + * @pkt_rcv_err_ilk: Reads or sets enable for + * BMI_INT[PKT_RCV_ERR_ILK]. + * @pkt_rcv_err_nps: Reads or sets enable for + * BMI_INT[PKT_RCV_ERR_NPS]. + * @max_len_err_ilk: Reads or sets enable for + * BMI_INT[MAX_LEN_ERR_ILK]. + * @max_len_err_nps: Reads or sets enable for + * BMI_INT[MAX_LEN_ERR_NPS]. + */ +union bmi_int_ena_w1s { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_13_63 : 51; + u64 ilk_req_oflw : 1; + u64 nps_req_oflw : 1; + u64 raz_10 : 1; + u64 raz_9 : 1; + u64 fpf_undrrn : 1; + u64 eop_err_ilk : 1; + u64 eop_err_nps : 1; + u64 sop_err_ilk : 1; + u64 sop_err_nps : 1; + u64 pkt_rcv_err_ilk : 1; + u64 pkt_rcv_err_nps : 1; + u64 max_len_err_ilk : 1; + u64 max_len_err_nps : 1; +#else + u64 max_len_err_nps : 1; + u64 max_len_err_ilk : 1; + u64 pkt_rcv_err_nps : 1; + u64 pkt_rcv_err_ilk : 1; + u64 sop_err_nps : 1; + u64 sop_err_ilk : 1; + u64 eop_err_nps : 1; + u64 eop_err_ilk : 1; + u64 fpf_undrrn : 1; + u64 raz_9 : 1; + u64 raz_10 : 1; + u64 nps_req_oflw : 1; + u64 ilk_req_oflw : 1; + u64 raz_13_63 : 51; +#endif + } s; +}; + +/** + * struct bmo_ctl2 - BMO Control2 Register + * @arb_sel: Determines P2X Arbitration + * @ilk_buf_thrsh: Maximum number of buffers that the + * ILK packet flows may consume before ILK XOFF is + * asserted to the POM. + * @nps_slc_buf_thrsh: Maximum number of buffers that the + * NPS_SLC packet flow may consume before NPS_SLC XOFF is + * asserted to the POM. + * @nps_uns_buf_thrsh: Maximum number of buffers that the + * NPS_UNS packet flow may consume before NPS_UNS XOFF is + * asserted to the POM. + * @totl_buf_thrsh: Maximum number of buffers that ILK, NPS_UNS and + * NPS_SLC packet flows may consume before NPS_UNS XOFF, NSP_SLC and + * ILK_XOFF are all asserted POM. + */ +union bmo_ctl2 { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 arb_sel : 1; + u64 raz_32_62 : 31; + u64 ilk_buf_thrsh : 8; + u64 nps_slc_buf_thrsh : 8; + u64 nps_uns_buf_thrsh : 8; + u64 totl_buf_thrsh : 8; +#else + u64 totl_buf_thrsh : 8; + u64 nps_uns_buf_thrsh : 8; + u64 nps_slc_buf_thrsh : 8; + u64 ilk_buf_thrsh : 8; + u64 raz_32_62 : 31; + u64 arb_sel : 1; +#endif + } s; +}; + +/** + * struct pom_int_ena_w1s - POM interrupt enable set register + * @illegal_intf: Reads or sets enable for POM_INT[ILLEGAL_INTF]. + * @illegal_dport: Reads or sets enable for POM_INT[ILLEGAL_DPORT]. + */ +union pom_int_ena_w1s { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz2 : 60; + u64 illegal_intf : 1; + u64 illegal_dport : 1; + u64 raz1 : 1; + u64 raz0 : 1; +#else + u64 raz0 : 1; + u64 raz1 : 1; + u64 illegal_dport : 1; + u64 illegal_intf : 1; + u64 raz2 : 60; +#endif + } s; +}; + +/** + * struct lbc_inval_ctl - LBC invalidation control register + * @wait_timer: Wait timer for wait state. [WAIT_TIMER] must + * always be written with its reset value. + * @cam_inval_start: Software should write [CAM_INVAL_START]=1 + * to initiate an LBC cache invalidation. After this, software + * should read LBC_INVAL_STATUS until LBC_INVAL_STATUS[DONE] is set. + * LBC hardware clears [CAVM_INVAL_START] before software can + * observed LBC_INVAL_STATUS[DONE] to be set + */ +union lbc_inval_ctl { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz2 : 48; + u64 wait_timer : 8; + u64 raz1 : 6; + u64 cam_inval_start : 1; + u64 raz0 : 1; +#else + u64 raz0 : 1; + u64 cam_inval_start : 1; + u64 raz1 : 6; + u64 wait_timer : 8; + u64 raz2 : 48; +#endif + } s; +}; + +/** + * struct lbc_int_ena_w1s - LBC interrupt enable set register + * @cam_hard_err: Reads or sets enable for LBC_INT[CAM_HARD_ERR]. + * @cam_inval_abort: Reads or sets enable for LBC_INT[CAM_INVAL_ABORT]. + * @over_fetch_err: Reads or sets enable for LBC_INT[OVER_FETCH_ERR]. + * @cache_line_to_err: Reads or sets enable for + * LBC_INT[CACHE_LINE_TO_ERR]. + * @cam_soft_err: Reads or sets enable for + * LBC_INT[CAM_SOFT_ERR]. + * @dma_rd_err: Reads or sets enable for + * LBC_INT[DMA_RD_ERR]. + */ +union lbc_int_ena_w1s { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_10_63 : 54; + u64 cam_hard_err : 1; + u64 cam_inval_abort : 1; + u64 over_fetch_err : 1; + u64 cache_line_to_err : 1; + u64 raz_2_5 : 4; + u64 cam_soft_err : 1; + u64 dma_rd_err : 1; +#else + u64 dma_rd_err : 1; + u64 cam_soft_err : 1; + u64 raz_2_5 : 4; + u64 cache_line_to_err : 1; + u64 over_fetch_err : 1; + u64 cam_inval_abort : 1; + u64 cam_hard_err : 1; + u64 raz_10_63 : 54; +#endif + } s; +}; + +/** + * struct lbc_int - LBC interrupt summary register + * @cam_hard_err: indicates a fatal hardware error. + * It requires system reset. + * When [CAM_HARD_ERR] is set, LBC stops logging any new information in + * LBC_POM_MISS_INFO_LOG, + * LBC_POM_MISS_ADDR_LOG, + * LBC_EFL_MISS_INFO_LOG, and + * LBC_EFL_MISS_ADDR_LOG. + * Software should sample them. + * @cam_inval_abort: indicates a fatal hardware error. + * System reset is required. + * @over_fetch_err: indicates a fatal hardware error + * System reset is required + * @cache_line_to_err: is a debug feature. + * This timeout interrupt bit tells the software that + * a cacheline in LBC has non-zero usage and the context + * has not been used for greater than the + * LBC_TO_CNT[TO_CNT] time interval. + * @sbe: Memory SBE error. This is recoverable via ECC. + * See LBC_ECC_INT for more details. + * @dbe: Memory DBE error. This is a fatal and requires a + * system reset. + * @pref_dat_len_mismatch_err: Summary bit for context length + * mismatch errors. + * @rd_dat_len_mismatch_err: Summary bit for SE read data length + * greater than data prefect length errors. + * @cam_soft_err: is recoverable. Software must complete a + * LBC_INVAL_CTL[CAM_INVAL_START] invalidation sequence and + * then clear [CAM_SOFT_ERR]. + * @dma_rd_err: A context prefect read of host memory returned with + * a read error. + */ +union lbc_int { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_10_63 : 54; + u64 cam_hard_err : 1; + u64 cam_inval_abort : 1; + u64 over_fetch_err : 1; + u64 cache_line_to_err : 1; + u64 sbe : 1; + u64 dbe : 1; + u64 pref_dat_len_mismatch_err : 1; + u64 rd_dat_len_mismatch_err : 1; + u64 cam_soft_err : 1; + u64 dma_rd_err : 1; +#else + u64 dma_rd_err : 1; + u64 cam_soft_err : 1; + u64 rd_dat_len_mismatch_err : 1; + u64 pref_dat_len_mismatch_err : 1; + u64 dbe : 1; + u64 sbe : 1; + u64 cache_line_to_err : 1; + u64 over_fetch_err : 1; + u64 cam_inval_abort : 1; + u64 cam_hard_err : 1; + u64 raz_10_63 : 54; +#endif + } s; +}; + +/** + * struct lbc_inval_status: LBC Invalidation status register + * @cam_clean_entry_complete_cnt: The number of entries that are + * cleaned up successfully. + * @cam_clean_entry_cnt: The number of entries that have the CAM + * inval command issued. + * @cam_inval_state: cam invalidation FSM state + * @cam_inval_abort: cam invalidation abort + * @cam_rst_rdy: lbc_cam reset ready + * @done: LBC clears [DONE] when + * LBC_INVAL_CTL[CAM_INVAL_START] is written with a one, + * and sets [DONE] when it completes the invalidation + * sequence. + */ +union lbc_inval_status { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz3 : 23; + u64 cam_clean_entry_complete_cnt : 9; + u64 raz2 : 7; + u64 cam_clean_entry_cnt : 9; + u64 raz1 : 5; + u64 cam_inval_state : 3; + u64 raz0 : 5; + u64 cam_inval_abort : 1; + u64 cam_rst_rdy : 1; + u64 done : 1; +#else + u64 done : 1; + u64 cam_rst_rdy : 1; + u64 cam_inval_abort : 1; + u64 raz0 : 5; + u64 cam_inval_state : 3; + u64 raz1 : 5; + u64 cam_clean_entry_cnt : 9; + u64 raz2 : 7; + u64 cam_clean_entry_complete_cnt : 9; + u64 raz3 : 23; +#endif + } s; +}; + +/** + * struct rst_boot: RST Boot Register + * @jtcsrdis: when set, internal CSR access via JTAG TAP controller + * is disabled + * @jt_tst_mode: JTAG test mode + * @io_supply: I/O power supply setting based on IO_VDD_SELECT pin: + * 0x1 = 1.8V + * 0x2 = 2.5V + * 0x4 = 3.3V + * All other values are reserved + * @pnr_mul: clock multiplier + * @lboot: last boot cause mask, resets only with PLL_DC_OK + * @rboot: determines whether core 0 remains in reset after + * chip cold or warm or soft reset + * @rboot_pin: read only access to REMOTE_BOOT pin + */ +union rst_boot { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_63 : 1; + u64 jtcsrdis : 1; + u64 raz_59_61 : 3; + u64 jt_tst_mode : 1; + u64 raz_40_57 : 18; + u64 io_supply : 3; + u64 raz_30_36 : 7; + u64 pnr_mul : 6; + u64 raz_12_23 : 12; + u64 lboot : 10; + u64 rboot : 1; + u64 rboot_pin : 1; +#else + u64 rboot_pin : 1; + u64 rboot : 1; + u64 lboot : 10; + u64 raz_12_23 : 12; + u64 pnr_mul : 6; + u64 raz_30_36 : 7; + u64 io_supply : 3; + u64 raz_40_57 : 18; + u64 jt_tst_mode : 1; + u64 raz_59_61 : 3; + u64 jtcsrdis : 1; + u64 raz_63 : 1; +#endif + }; +}; + +/** + * struct fus_dat1: Fuse Data 1 Register + * @pll_mul: main clock PLL multiplier hardware limit + * @pll_half_dis: main clock PLL control + * @efus_lck: efuse lockdown + * @zip_info: ZIP information + * @bar2_sz_conf: when zero, BAR2 size conforms to + * PCIe specification + * @efus_ign: efuse ignore + * @nozip: ZIP disable + * @pll_alt_matrix: select alternate PLL matrix + * @pll_bwadj_denom: select CLKF denominator for + * BWADJ value + * @chip_id: chip ID + */ +union fus_dat1 { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_57_63 : 7; + u64 pll_mul : 3; + u64 pll_half_dis : 1; + u64 raz_43_52 : 10; + u64 efus_lck : 3; + u64 raz_26_39 : 14; + u64 zip_info : 5; + u64 bar2_sz_conf : 1; + u64 efus_ign : 1; + u64 nozip : 1; + u64 raz_11_17 : 7; + u64 pll_alt_matrix : 1; + u64 pll_bwadj_denom : 2; + u64 chip_id : 8; +#else + u64 chip_id : 8; + u64 pll_bwadj_denom : 2; + u64 pll_alt_matrix : 1; + u64 raz_11_17 : 7; + u64 nozip : 1; + u64 efus_ign : 1; + u64 bar2_sz_conf : 1; + u64 zip_info : 5; + u64 raz_26_39 : 14; + u64 efus_lck : 3; + u64 raz_43_52 : 10; + u64 pll_half_dis : 1; + u64 pll_mul : 3; + u64 raz_57_63 : 7; +#endif + }; +}; + +#endif /* __NITROX_CSR_H */ diff --git a/drivers/crypto/cavium/nitrox/nitrox_debugfs.c b/drivers/crypto/cavium/nitrox/nitrox_debugfs.c new file mode 100644 index 000000000..741572a01 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_debugfs.c @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/seq_file.h> +#include <linux/debugfs.h> + +#include "nitrox_csr.h" +#include "nitrox_debugfs.h" +#include "nitrox_dev.h" + +static int firmware_show(struct seq_file *s, void *v) +{ + struct nitrox_device *ndev = s->private; + + seq_printf(s, "Version: %s\n", ndev->hw.fw_name[0]); + seq_printf(s, "Version: %s\n", ndev->hw.fw_name[1]); + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(firmware); + +static int device_show(struct seq_file *s, void *v) +{ + struct nitrox_device *ndev = s->private; + + seq_printf(s, "NITROX [%d]\n", ndev->idx); + seq_printf(s, " Part Name: %s\n", ndev->hw.partname); + seq_printf(s, " Frequency: %d MHz\n", ndev->hw.freq); + seq_printf(s, " Device ID: 0x%0x\n", ndev->hw.device_id); + seq_printf(s, " Revision ID: 0x%0x\n", ndev->hw.revision_id); + seq_printf(s, " Cores: [AE=%u SE=%u ZIP=%u]\n", + ndev->hw.ae_cores, ndev->hw.se_cores, ndev->hw.zip_cores); + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(device); + +static int stats_show(struct seq_file *s, void *v) +{ + struct nitrox_device *ndev = s->private; + + seq_printf(s, "NITROX [%d] Request Statistics\n", ndev->idx); + seq_printf(s, " Posted: %llu\n", + (u64)atomic64_read(&ndev->stats.posted)); + seq_printf(s, " Completed: %llu\n", + (u64)atomic64_read(&ndev->stats.completed)); + seq_printf(s, " Dropped: %llu\n", + (u64)atomic64_read(&ndev->stats.dropped)); + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(stats); + +void nitrox_debugfs_exit(struct nitrox_device *ndev) +{ + debugfs_remove_recursive(ndev->debugfs_dir); + ndev->debugfs_dir = NULL; +} + +void nitrox_debugfs_init(struct nitrox_device *ndev) +{ + struct dentry *dir; + + dir = debugfs_create_dir(KBUILD_MODNAME, NULL); + + ndev->debugfs_dir = dir; + debugfs_create_file("firmware", 0400, dir, ndev, &firmware_fops); + debugfs_create_file("device", 0400, dir, ndev, &device_fops); + debugfs_create_file("stats", 0400, dir, ndev, &stats_fops); +} diff --git a/drivers/crypto/cavium/nitrox/nitrox_debugfs.h b/drivers/crypto/cavium/nitrox/nitrox_debugfs.h new file mode 100644 index 000000000..09c4cf251 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_debugfs.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __NITROX_DEBUGFS_H +#define __NITROX_DEBUGFS_H + +#include "nitrox_dev.h" + +#ifdef CONFIG_DEBUG_FS +void nitrox_debugfs_init(struct nitrox_device *ndev); +void nitrox_debugfs_exit(struct nitrox_device *ndev); +#else +static inline void nitrox_debugfs_init(struct nitrox_device *ndev) +{ +} + +static inline void nitrox_debugfs_exit(struct nitrox_device *ndev) +{ +} +#endif /* !CONFIG_DEBUG_FS */ + +#endif /* __NITROX_DEBUGFS_H */ diff --git a/drivers/crypto/cavium/nitrox/nitrox_dev.h b/drivers/crypto/cavium/nitrox/nitrox_dev.h new file mode 100644 index 000000000..c2d0c23fb --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_dev.h @@ -0,0 +1,302 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __NITROX_DEV_H +#define __NITROX_DEV_H + +#include <linux/dma-mapping.h> +#include <linux/interrupt.h> +#include <linux/pci.h> +#include <linux/if.h> + +#define VERSION_LEN 32 +/* Maximum queues in PF mode */ +#define MAX_PF_QUEUES 64 +/* Maximum device queues */ +#define MAX_DEV_QUEUES (MAX_PF_QUEUES) +/* Maximum UCD Blocks */ +#define CNN55XX_MAX_UCD_BLOCKS 8 + +/** + * struct nitrox_cmdq - NITROX command queue + * @cmd_qlock: command queue lock + * @resp_qlock: response queue lock + * @backlog_qlock: backlog queue lock + * @ndev: NITROX device + * @response_head: submitted request list + * @backlog_head: backlog queue + * @dbell_csr_addr: doorbell register address for this queue + * @compl_cnt_csr_addr: completion count register address of the slc port + * @base: command queue base address + * @dma: dma address of the base + * @pending_count: request pending at device + * @backlog_count: backlog request count + * @write_idx: next write index for the command + * @instr_size: command size + * @qno: command queue number + * @qsize: command queue size + * @unalign_base: unaligned base address + * @unalign_dma: unaligned dma address + */ +struct nitrox_cmdq { + spinlock_t cmd_qlock; + spinlock_t resp_qlock; + spinlock_t backlog_qlock; + + struct nitrox_device *ndev; + struct list_head response_head; + struct list_head backlog_head; + + u8 __iomem *dbell_csr_addr; + u8 __iomem *compl_cnt_csr_addr; + u8 *base; + dma_addr_t dma; + + struct work_struct backlog_qflush; + + atomic_t pending_count; + atomic_t backlog_count; + + int write_idx; + u8 instr_size; + u8 qno; + u32 qsize; + + u8 *unalign_base; + dma_addr_t unalign_dma; +}; + +/** + * struct nitrox_hw - NITROX hardware information + * @partname: partname ex: CNN55xxx-xxx + * @fw_name: firmware version + * @freq: NITROX frequency + * @vendor_id: vendor ID + * @device_id: device ID + * @revision_id: revision ID + * @se_cores: number of symmetric cores + * @ae_cores: number of asymmetric cores + * @zip_cores: number of zip cores + */ +struct nitrox_hw { + char partname[IFNAMSIZ * 2]; + char fw_name[CNN55XX_MAX_UCD_BLOCKS][VERSION_LEN]; + + int freq; + u16 vendor_id; + u16 device_id; + u8 revision_id; + + u8 se_cores; + u8 ae_cores; + u8 zip_cores; +}; + +struct nitrox_stats { + atomic64_t posted; + atomic64_t completed; + atomic64_t dropped; +}; + +#define IRQ_NAMESZ 32 + +struct nitrox_q_vector { + char name[IRQ_NAMESZ]; + bool valid; + int ring; + struct tasklet_struct resp_tasklet; + union { + struct nitrox_cmdq *cmdq; + struct nitrox_device *ndev; + }; +}; + +enum mcode_type { + MCODE_TYPE_INVALID, + MCODE_TYPE_AE, + MCODE_TYPE_SE_SSL, + MCODE_TYPE_SE_IPSEC, +}; + +/** + * mbox_msg - Mailbox message data + * @type: message type + * @opcode: message opcode + * @data: message data + */ +union mbox_msg { + u64 value; + struct { + u64 type: 2; + u64 opcode: 6; + u64 data: 58; + }; + struct { + u64 type: 2; + u64 opcode: 6; + u64 chipid: 8; + u64 vfid: 8; + } id; + struct { + u64 type: 2; + u64 opcode: 6; + u64 count: 4; + u64 info: 40; + u64 next_se_grp: 3; + u64 next_ae_grp: 3; + } mcode_info; +}; + +/** + * nitrox_vfdev - NITROX VF device instance in PF + * @state: VF device state + * @vfno: VF number + * @nr_queues: number of queues enabled in VF + * @ring: ring to communicate with VF + * @msg: Mailbox message data from VF + * @mbx_resp: Mailbox counters + */ +struct nitrox_vfdev { + atomic_t state; + int vfno; + int nr_queues; + int ring; + union mbox_msg msg; + atomic64_t mbx_resp; +}; + +/** + * struct nitrox_iov - SR-IOV information + * @num_vfs: number of VF(s) enabled + * @max_vf_queues: Maximum number of queues allowed for VF + * @vfdev: VF(s) devices + * @pf2vf_wq: workqueue for PF2VF communication + * @msix: MSI-X entry for PF in SR-IOV case + */ +struct nitrox_iov { + int num_vfs; + int max_vf_queues; + struct nitrox_vfdev *vfdev; + struct workqueue_struct *pf2vf_wq; + struct msix_entry msix; +}; + +/* + * NITROX Device states + */ +enum ndev_state { + __NDEV_NOT_READY, + __NDEV_READY, + __NDEV_IN_RESET, +}; + +/* NITROX support modes for VF(s) */ +enum vf_mode { + __NDEV_MODE_PF, + __NDEV_MODE_VF16, + __NDEV_MODE_VF32, + __NDEV_MODE_VF64, + __NDEV_MODE_VF128, +}; + +#define __NDEV_SRIOV_BIT 0 + +/* command queue size */ +#define DEFAULT_CMD_QLEN 2048 +/* command timeout in milliseconds */ +#define CMD_TIMEOUT 2000 + +#define DEV(ndev) ((struct device *)(&(ndev)->pdev->dev)) + +#define NITROX_CSR_ADDR(ndev, offset) \ + ((ndev)->bar_addr + (offset)) + +/** + * struct nitrox_device - NITROX Device Information. + * @list: pointer to linked list of devices + * @bar_addr: iomap address + * @pdev: PCI device information + * @state: NITROX device state + * @flags: flags to indicate device the features + * @timeout: Request timeout in jiffies + * @refcnt: Device usage count + * @idx: device index (0..N) + * @node: NUMA node id attached + * @qlen: Command queue length + * @nr_queues: Number of command queues + * @mode: Device mode PF/VF + * @ctx_pool: DMA pool for crypto context + * @pkt_inq: Packet input rings + * @aqmq: AQM command queues + * @qvec: MSI-X queue vectors information + * @iov: SR-IOV informatin + * @num_vecs: number of MSI-X vectors + * @stats: request statistics + * @hw: hardware information + * @debugfs_dir: debugfs directory + */ +struct nitrox_device { + struct list_head list; + + u8 __iomem *bar_addr; + struct pci_dev *pdev; + + atomic_t state; + unsigned long flags; + unsigned long timeout; + refcount_t refcnt; + + u8 idx; + int node; + u16 qlen; + u16 nr_queues; + enum vf_mode mode; + + struct dma_pool *ctx_pool; + struct nitrox_cmdq *pkt_inq; + struct nitrox_cmdq *aqmq[MAX_DEV_QUEUES] ____cacheline_aligned_in_smp; + + struct nitrox_q_vector *qvec; + struct nitrox_iov iov; + int num_vecs; + + struct nitrox_stats stats; + struct nitrox_hw hw; +#if IS_ENABLED(CONFIG_DEBUG_FS) + struct dentry *debugfs_dir; +#endif +}; + +/** + * nitrox_read_csr - Read from device register + * @ndev: NITROX device + * @offset: offset of the register to read + * + * Returns: value read + */ +static inline u64 nitrox_read_csr(struct nitrox_device *ndev, u64 offset) +{ + return readq(ndev->bar_addr + offset); +} + +/** + * nitrox_write_csr - Write to device register + * @ndev: NITROX device + * @offset: offset of the register to write + * @value: value to write + */ +static inline void nitrox_write_csr(struct nitrox_device *ndev, u64 offset, + u64 value) +{ + writeq(value, (ndev->bar_addr + offset)); +} + +static inline bool nitrox_ready(struct nitrox_device *ndev) +{ + return atomic_read(&ndev->state) == __NDEV_READY; +} + +static inline bool nitrox_vfdev_ready(struct nitrox_vfdev *vfdev) +{ + return atomic_read(&vfdev->state) == __NDEV_READY; +} + +#endif /* __NITROX_DEV_H */ diff --git a/drivers/crypto/cavium/nitrox/nitrox_hal.c b/drivers/crypto/cavium/nitrox/nitrox_hal.c new file mode 100644 index 000000000..13b137410 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_hal.c @@ -0,0 +1,679 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/delay.h> + +#include "nitrox_dev.h" +#include "nitrox_csr.h" +#include "nitrox_hal.h" + +#define PLL_REF_CLK 50 +#define MAX_CSR_RETRIES 10 + +/** + * emu_enable_cores - Enable EMU cluster cores. + * @ndev: NITROX device + */ +static void emu_enable_cores(struct nitrox_device *ndev) +{ + union emu_se_enable emu_se; + union emu_ae_enable emu_ae; + int i; + + /* AE cores 20 per cluster */ + emu_ae.value = 0; + emu_ae.s.enable = 0xfffff; + + /* SE cores 16 per cluster */ + emu_se.value = 0; + emu_se.s.enable = 0xffff; + + /* enable per cluster cores */ + for (i = 0; i < NR_CLUSTERS; i++) { + nitrox_write_csr(ndev, EMU_AE_ENABLEX(i), emu_ae.value); + nitrox_write_csr(ndev, EMU_SE_ENABLEX(i), emu_se.value); + } +} + +/** + * nitrox_config_emu_unit - configure EMU unit. + * @ndev: NITROX device + */ +void nitrox_config_emu_unit(struct nitrox_device *ndev) +{ + union emu_wd_int_ena_w1s emu_wd_int; + union emu_ge_int_ena_w1s emu_ge_int; + u64 offset; + int i; + + /* enable cores */ + emu_enable_cores(ndev); + + /* enable general error and watch dog interrupts */ + emu_ge_int.value = 0; + emu_ge_int.s.se_ge = 0xffff; + emu_ge_int.s.ae_ge = 0xfffff; + emu_wd_int.value = 0; + emu_wd_int.s.se_wd = 1; + + for (i = 0; i < NR_CLUSTERS; i++) { + offset = EMU_WD_INT_ENA_W1SX(i); + nitrox_write_csr(ndev, offset, emu_wd_int.value); + offset = EMU_GE_INT_ENA_W1SX(i); + nitrox_write_csr(ndev, offset, emu_ge_int.value); + } +} + +static void reset_pkt_input_ring(struct nitrox_device *ndev, int ring) +{ + union nps_pkt_in_instr_ctl pkt_in_ctl; + union nps_pkt_in_done_cnts pkt_in_cnts; + int max_retries = MAX_CSR_RETRIES; + u64 offset; + + /* step 1: disable the ring, clear enable bit */ + offset = NPS_PKT_IN_INSTR_CTLX(ring); + pkt_in_ctl.value = nitrox_read_csr(ndev, offset); + pkt_in_ctl.s.enb = 0; + nitrox_write_csr(ndev, offset, pkt_in_ctl.value); + + /* step 2: wait to clear [ENB] */ + usleep_range(100, 150); + do { + pkt_in_ctl.value = nitrox_read_csr(ndev, offset); + if (!pkt_in_ctl.s.enb) + break; + udelay(50); + } while (max_retries--); + + /* step 3: clear done counts */ + offset = NPS_PKT_IN_DONE_CNTSX(ring); + pkt_in_cnts.value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, pkt_in_cnts.value); + usleep_range(50, 100); +} + +void enable_pkt_input_ring(struct nitrox_device *ndev, int ring) +{ + union nps_pkt_in_instr_ctl pkt_in_ctl; + int max_retries = MAX_CSR_RETRIES; + u64 offset; + + /* 64-byte instruction size */ + offset = NPS_PKT_IN_INSTR_CTLX(ring); + pkt_in_ctl.value = nitrox_read_csr(ndev, offset); + pkt_in_ctl.s.is64b = 1; + pkt_in_ctl.s.enb = 1; + nitrox_write_csr(ndev, offset, pkt_in_ctl.value); + + /* wait for set [ENB] */ + do { + pkt_in_ctl.value = nitrox_read_csr(ndev, offset); + if (pkt_in_ctl.s.enb) + break; + udelay(50); + } while (max_retries--); +} + +/** + * nitrox_config_pkt_input_rings - configure Packet Input Rings + * @ndev: NITROX device + */ +void nitrox_config_pkt_input_rings(struct nitrox_device *ndev) +{ + int i; + + for (i = 0; i < ndev->nr_queues; i++) { + struct nitrox_cmdq *cmdq = &ndev->pkt_inq[i]; + union nps_pkt_in_instr_rsize pkt_in_rsize; + union nps_pkt_in_instr_baoff_dbell pkt_in_dbell; + u64 offset; + + reset_pkt_input_ring(ndev, i); + + /** + * step 4: + * configure ring base address 16-byte aligned, + * size and interrupt threshold. + */ + offset = NPS_PKT_IN_INSTR_BADDRX(i); + nitrox_write_csr(ndev, offset, cmdq->dma); + + /* configure ring size */ + offset = NPS_PKT_IN_INSTR_RSIZEX(i); + pkt_in_rsize.value = 0; + pkt_in_rsize.s.rsize = ndev->qlen; + nitrox_write_csr(ndev, offset, pkt_in_rsize.value); + + /* set high threshold for pkt input ring interrupts */ + offset = NPS_PKT_IN_INT_LEVELSX(i); + nitrox_write_csr(ndev, offset, 0xffffffff); + + /* step 5: clear off door bell counts */ + offset = NPS_PKT_IN_INSTR_BAOFF_DBELLX(i); + pkt_in_dbell.value = 0; + pkt_in_dbell.s.dbell = 0xffffffff; + nitrox_write_csr(ndev, offset, pkt_in_dbell.value); + + /* enable the ring */ + enable_pkt_input_ring(ndev, i); + } +} + +static void reset_pkt_solicit_port(struct nitrox_device *ndev, int port) +{ + union nps_pkt_slc_ctl pkt_slc_ctl; + union nps_pkt_slc_cnts pkt_slc_cnts; + int max_retries = MAX_CSR_RETRIES; + u64 offset; + + /* step 1: disable slc port */ + offset = NPS_PKT_SLC_CTLX(port); + pkt_slc_ctl.value = nitrox_read_csr(ndev, offset); + pkt_slc_ctl.s.enb = 0; + nitrox_write_csr(ndev, offset, pkt_slc_ctl.value); + + /* step 2 */ + usleep_range(100, 150); + /* wait to clear [ENB] */ + do { + pkt_slc_ctl.value = nitrox_read_csr(ndev, offset); + if (!pkt_slc_ctl.s.enb) + break; + udelay(50); + } while (max_retries--); + + /* step 3: clear slc counters */ + offset = NPS_PKT_SLC_CNTSX(port); + pkt_slc_cnts.value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, pkt_slc_cnts.value); + usleep_range(50, 100); +} + +void enable_pkt_solicit_port(struct nitrox_device *ndev, int port) +{ + union nps_pkt_slc_ctl pkt_slc_ctl; + int max_retries = MAX_CSR_RETRIES; + u64 offset; + + offset = NPS_PKT_SLC_CTLX(port); + pkt_slc_ctl.value = 0; + pkt_slc_ctl.s.enb = 1; + /* + * 8 trailing 0x00 bytes will be added + * to the end of the outgoing packet. + */ + pkt_slc_ctl.s.z = 1; + /* enable response header */ + pkt_slc_ctl.s.rh = 1; + nitrox_write_csr(ndev, offset, pkt_slc_ctl.value); + + /* wait to set [ENB] */ + do { + pkt_slc_ctl.value = nitrox_read_csr(ndev, offset); + if (pkt_slc_ctl.s.enb) + break; + udelay(50); + } while (max_retries--); +} + +static void config_pkt_solicit_port(struct nitrox_device *ndev, int port) +{ + union nps_pkt_slc_int_levels pkt_slc_int; + u64 offset; + + reset_pkt_solicit_port(ndev, port); + + /* step 4: configure interrupt levels */ + offset = NPS_PKT_SLC_INT_LEVELSX(port); + pkt_slc_int.value = 0; + /* time interrupt threshold */ + pkt_slc_int.s.timet = 0x3fffff; + nitrox_write_csr(ndev, offset, pkt_slc_int.value); + + /* enable the solicit port */ + enable_pkt_solicit_port(ndev, port); +} + +void nitrox_config_pkt_solicit_ports(struct nitrox_device *ndev) +{ + int i; + + for (i = 0; i < ndev->nr_queues; i++) + config_pkt_solicit_port(ndev, i); +} + +/** + * enable_nps_core_interrupts - enable NPS core interrutps + * @ndev: NITROX device. + * + * This includes NPS core interrupts. + */ +static void enable_nps_core_interrupts(struct nitrox_device *ndev) +{ + union nps_core_int_ena_w1s core_int; + + /* NPS core interrutps */ + core_int.value = 0; + core_int.s.host_wr_err = 1; + core_int.s.host_wr_timeout = 1; + core_int.s.exec_wr_timeout = 1; + core_int.s.npco_dma_malform = 1; + core_int.s.host_nps_wr_err = 1; + nitrox_write_csr(ndev, NPS_CORE_INT_ENA_W1S, core_int.value); +} + +void nitrox_config_nps_core_unit(struct nitrox_device *ndev) +{ + union nps_core_gbl_vfcfg core_gbl_vfcfg; + + /* endian control information */ + nitrox_write_csr(ndev, NPS_CORE_CONTROL, 1ULL); + + /* disable ILK interface */ + core_gbl_vfcfg.value = 0; + core_gbl_vfcfg.s.ilk_disable = 1; + core_gbl_vfcfg.s.cfg = __NDEV_MODE_PF; + nitrox_write_csr(ndev, NPS_CORE_GBL_VFCFG, core_gbl_vfcfg.value); + + /* enable nps core interrupts */ + enable_nps_core_interrupts(ndev); +} + +/** + * enable_nps_pkt_interrupts - enable NPS packet interrutps + * @ndev: NITROX device. + * + * This includes NPS packet in and slc interrupts. + */ +static void enable_nps_pkt_interrupts(struct nitrox_device *ndev) +{ + /* NPS packet in ring interrupts */ + nitrox_write_csr(ndev, NPS_PKT_IN_RERR_LO_ENA_W1S, (~0ULL)); + nitrox_write_csr(ndev, NPS_PKT_IN_RERR_HI_ENA_W1S, (~0ULL)); + nitrox_write_csr(ndev, NPS_PKT_IN_ERR_TYPE_ENA_W1S, (~0ULL)); + /* NPS packet slc port interrupts */ + nitrox_write_csr(ndev, NPS_PKT_SLC_RERR_HI_ENA_W1S, (~0ULL)); + nitrox_write_csr(ndev, NPS_PKT_SLC_RERR_LO_ENA_W1S, (~0ULL)); + nitrox_write_csr(ndev, NPS_PKT_SLC_ERR_TYPE_ENA_W1S, (~0uLL)); +} + +void nitrox_config_nps_pkt_unit(struct nitrox_device *ndev) +{ + /* config input and solicit ports */ + nitrox_config_pkt_input_rings(ndev); + nitrox_config_pkt_solicit_ports(ndev); + + /* enable nps packet interrupts */ + enable_nps_pkt_interrupts(ndev); +} + +static void reset_aqm_ring(struct nitrox_device *ndev, int ring) +{ + union aqmq_en aqmq_en_reg; + union aqmq_activity_stat activity_stat; + union aqmq_cmp_cnt cmp_cnt; + int max_retries = MAX_CSR_RETRIES; + u64 offset; + + /* step 1: disable the queue */ + offset = AQMQ_ENX(ring); + aqmq_en_reg.value = 0; + aqmq_en_reg.queue_enable = 0; + nitrox_write_csr(ndev, offset, aqmq_en_reg.value); + + /* step 2: wait for AQMQ_ACTIVITY_STATX[QUEUE_ACTIVE] to clear */ + usleep_range(100, 150); + offset = AQMQ_ACTIVITY_STATX(ring); + do { + activity_stat.value = nitrox_read_csr(ndev, offset); + if (!activity_stat.queue_active) + break; + udelay(50); + } while (max_retries--); + + /* step 3: clear commands completed count */ + offset = AQMQ_CMP_CNTX(ring); + cmp_cnt.value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, cmp_cnt.value); + usleep_range(50, 100); +} + +void enable_aqm_ring(struct nitrox_device *ndev, int ring) +{ + union aqmq_en aqmq_en_reg; + u64 offset; + + offset = AQMQ_ENX(ring); + aqmq_en_reg.value = 0; + aqmq_en_reg.queue_enable = 1; + nitrox_write_csr(ndev, offset, aqmq_en_reg.value); + usleep_range(50, 100); +} + +void nitrox_config_aqm_rings(struct nitrox_device *ndev) +{ + int ring; + + for (ring = 0; ring < ndev->nr_queues; ring++) { + struct nitrox_cmdq *cmdq = ndev->aqmq[ring]; + union aqmq_drbl drbl; + union aqmq_qsz qsize; + union aqmq_cmp_thr cmp_thr; + u64 offset; + + /* steps 1 - 3 */ + reset_aqm_ring(ndev, ring); + + /* step 4: clear doorbell count of ring */ + offset = AQMQ_DRBLX(ring); + drbl.value = 0; + drbl.dbell_count = 0xFFFFFFFF; + nitrox_write_csr(ndev, offset, drbl.value); + + /* step 5: configure host ring details */ + + /* set host address for next command of ring */ + offset = AQMQ_NXT_CMDX(ring); + nitrox_write_csr(ndev, offset, 0ULL); + + /* set host address of ring base */ + offset = AQMQ_BADRX(ring); + nitrox_write_csr(ndev, offset, cmdq->dma); + + /* set ring size */ + offset = AQMQ_QSZX(ring); + qsize.value = 0; + qsize.host_queue_size = ndev->qlen; + nitrox_write_csr(ndev, offset, qsize.value); + + /* set command completion threshold */ + offset = AQMQ_CMP_THRX(ring); + cmp_thr.value = 0; + cmp_thr.commands_completed_threshold = 1; + nitrox_write_csr(ndev, offset, cmp_thr.value); + + /* step 6: enable the queue */ + enable_aqm_ring(ndev, ring); + } +} + +static void enable_aqm_interrupts(struct nitrox_device *ndev) +{ + /* clear interrupt enable bits */ + nitrox_write_csr(ndev, AQM_DBELL_OVF_LO_ENA_W1S, (~0ULL)); + nitrox_write_csr(ndev, AQM_DBELL_OVF_HI_ENA_W1S, (~0ULL)); + nitrox_write_csr(ndev, AQM_DMA_RD_ERR_LO_ENA_W1S, (~0ULL)); + nitrox_write_csr(ndev, AQM_DMA_RD_ERR_HI_ENA_W1S, (~0ULL)); + nitrox_write_csr(ndev, AQM_EXEC_NA_LO_ENA_W1S, (~0ULL)); + nitrox_write_csr(ndev, AQM_EXEC_NA_HI_ENA_W1S, (~0ULL)); + nitrox_write_csr(ndev, AQM_EXEC_ERR_LO_ENA_W1S, (~0ULL)); + nitrox_write_csr(ndev, AQM_EXEC_ERR_HI_ENA_W1S, (~0ULL)); +} + +void nitrox_config_aqm_unit(struct nitrox_device *ndev) +{ + /* config aqm command queues */ + nitrox_config_aqm_rings(ndev); + + /* enable aqm interrupts */ + enable_aqm_interrupts(ndev); +} + +void nitrox_config_pom_unit(struct nitrox_device *ndev) +{ + union pom_int_ena_w1s pom_int; + int i; + + /* enable pom interrupts */ + pom_int.value = 0; + pom_int.s.illegal_dport = 1; + nitrox_write_csr(ndev, POM_INT_ENA_W1S, pom_int.value); + + /* enable perf counters */ + for (i = 0; i < ndev->hw.se_cores; i++) + nitrox_write_csr(ndev, POM_PERF_CTL, BIT_ULL(i)); +} + +/** + * nitrox_config_rand_unit - enable NITROX random number unit + * @ndev: NITROX device + */ +void nitrox_config_rand_unit(struct nitrox_device *ndev) +{ + union efl_rnm_ctl_status efl_rnm_ctl; + u64 offset; + + offset = EFL_RNM_CTL_STATUS; + efl_rnm_ctl.value = nitrox_read_csr(ndev, offset); + efl_rnm_ctl.s.ent_en = 1; + efl_rnm_ctl.s.rng_en = 1; + nitrox_write_csr(ndev, offset, efl_rnm_ctl.value); +} + +void nitrox_config_efl_unit(struct nitrox_device *ndev) +{ + int i; + + for (i = 0; i < NR_CLUSTERS; i++) { + union efl_core_int_ena_w1s efl_core_int; + u64 offset; + + /* EFL core interrupts */ + offset = EFL_CORE_INT_ENA_W1SX(i); + efl_core_int.value = 0; + efl_core_int.s.len_ovr = 1; + efl_core_int.s.d_left = 1; + efl_core_int.s.epci_decode_err = 1; + nitrox_write_csr(ndev, offset, efl_core_int.value); + + offset = EFL_CORE_VF_ERR_INT0_ENA_W1SX(i); + nitrox_write_csr(ndev, offset, (~0ULL)); + offset = EFL_CORE_VF_ERR_INT1_ENA_W1SX(i); + nitrox_write_csr(ndev, offset, (~0ULL)); + } +} + +void nitrox_config_bmi_unit(struct nitrox_device *ndev) +{ + union bmi_ctl bmi_ctl; + union bmi_int_ena_w1s bmi_int_ena; + u64 offset; + + /* no threshold limits for PCIe */ + offset = BMI_CTL; + bmi_ctl.value = nitrox_read_csr(ndev, offset); + bmi_ctl.s.max_pkt_len = 0xff; + bmi_ctl.s.nps_free_thrsh = 0xff; + bmi_ctl.s.nps_hdrq_thrsh = 0x7a; + nitrox_write_csr(ndev, offset, bmi_ctl.value); + + /* enable interrupts */ + offset = BMI_INT_ENA_W1S; + bmi_int_ena.value = 0; + bmi_int_ena.s.max_len_err_nps = 1; + bmi_int_ena.s.pkt_rcv_err_nps = 1; + bmi_int_ena.s.fpf_undrrn = 1; + nitrox_write_csr(ndev, offset, bmi_int_ena.value); +} + +void nitrox_config_bmo_unit(struct nitrox_device *ndev) +{ + union bmo_ctl2 bmo_ctl2; + u64 offset; + + /* no threshold limits for PCIe */ + offset = BMO_CTL2; + bmo_ctl2.value = nitrox_read_csr(ndev, offset); + bmo_ctl2.s.nps_slc_buf_thrsh = 0xff; + nitrox_write_csr(ndev, offset, bmo_ctl2.value); +} + +void invalidate_lbc(struct nitrox_device *ndev) +{ + union lbc_inval_ctl lbc_ctl; + union lbc_inval_status lbc_stat; + int max_retries = MAX_CSR_RETRIES; + u64 offset; + + /* invalidate LBC */ + offset = LBC_INVAL_CTL; + lbc_ctl.value = nitrox_read_csr(ndev, offset); + lbc_ctl.s.cam_inval_start = 1; + nitrox_write_csr(ndev, offset, lbc_ctl.value); + + offset = LBC_INVAL_STATUS; + do { + lbc_stat.value = nitrox_read_csr(ndev, offset); + if (lbc_stat.s.done) + break; + udelay(50); + } while (max_retries--); +} + +void nitrox_config_lbc_unit(struct nitrox_device *ndev) +{ + union lbc_int_ena_w1s lbc_int_ena; + u64 offset; + + invalidate_lbc(ndev); + + /* enable interrupts */ + offset = LBC_INT_ENA_W1S; + lbc_int_ena.value = 0; + lbc_int_ena.s.dma_rd_err = 1; + lbc_int_ena.s.over_fetch_err = 1; + lbc_int_ena.s.cam_inval_abort = 1; + lbc_int_ena.s.cam_hard_err = 1; + nitrox_write_csr(ndev, offset, lbc_int_ena.value); + + offset = LBC_PLM_VF1_64_INT_ENA_W1S; + nitrox_write_csr(ndev, offset, (~0ULL)); + offset = LBC_PLM_VF65_128_INT_ENA_W1S; + nitrox_write_csr(ndev, offset, (~0ULL)); + + offset = LBC_ELM_VF1_64_INT_ENA_W1S; + nitrox_write_csr(ndev, offset, (~0ULL)); + offset = LBC_ELM_VF65_128_INT_ENA_W1S; + nitrox_write_csr(ndev, offset, (~0ULL)); +} + +void config_nps_core_vfcfg_mode(struct nitrox_device *ndev, enum vf_mode mode) +{ + union nps_core_gbl_vfcfg vfcfg; + + vfcfg.value = nitrox_read_csr(ndev, NPS_CORE_GBL_VFCFG); + vfcfg.s.cfg = mode & 0x7; + + nitrox_write_csr(ndev, NPS_CORE_GBL_VFCFG, vfcfg.value); +} + +static const char *get_core_option(u8 se_cores, u8 ae_cores) +{ + const char *option = ""; + + if (ae_cores == AE_MAX_CORES) { + switch (se_cores) { + case SE_MAX_CORES: + option = "60"; + break; + case 40: + option = "60s"; + break; + } + } else if (ae_cores == (AE_MAX_CORES / 2)) { + option = "30"; + } else { + option = "60i"; + } + + return option; +} + +static const char *get_feature_option(u8 zip_cores, int core_freq) +{ + if (zip_cores == 0) + return ""; + else if (zip_cores < ZIP_MAX_CORES) + return "-C15"; + + if (core_freq >= 850) + return "-C45"; + else if (core_freq >= 750) + return "-C35"; + else if (core_freq >= 550) + return "-C25"; + + return ""; +} + +void nitrox_get_hwinfo(struct nitrox_device *ndev) +{ + union emu_fuse_map emu_fuse; + union rst_boot rst_boot; + union fus_dat1 fus_dat1; + unsigned char name[IFNAMSIZ * 2] = {}; + int i, dead_cores; + u64 offset; + + /* get core frequency */ + offset = RST_BOOT; + rst_boot.value = nitrox_read_csr(ndev, offset); + ndev->hw.freq = (rst_boot.pnr_mul + 3) * PLL_REF_CLK; + + for (i = 0; i < NR_CLUSTERS; i++) { + offset = EMU_FUSE_MAPX(i); + emu_fuse.value = nitrox_read_csr(ndev, offset); + if (emu_fuse.s.valid) { + dead_cores = hweight32(emu_fuse.s.ae_fuse); + ndev->hw.ae_cores += AE_CORES_PER_CLUSTER - dead_cores; + dead_cores = hweight16(emu_fuse.s.se_fuse); + ndev->hw.se_cores += SE_CORES_PER_CLUSTER - dead_cores; + } + } + /* find zip hardware availability */ + offset = FUS_DAT1; + fus_dat1.value = nitrox_read_csr(ndev, offset); + if (!fus_dat1.nozip) { + dead_cores = hweight8(fus_dat1.zip_info); + ndev->hw.zip_cores = ZIP_MAX_CORES - dead_cores; + } + + /* determine the partname + * CNN55<core option>-<freq><pincount>-<feature option>-<rev> + */ + snprintf(name, sizeof(name), "CNN55%s-%3dBG676%s-1.%u", + get_core_option(ndev->hw.se_cores, ndev->hw.ae_cores), + ndev->hw.freq, + get_feature_option(ndev->hw.zip_cores, ndev->hw.freq), + ndev->hw.revision_id); + + /* copy partname */ + strncpy(ndev->hw.partname, name, sizeof(ndev->hw.partname)); +} + +void enable_pf2vf_mbox_interrupts(struct nitrox_device *ndev) +{ + u64 value = ~0ULL; + u64 reg_addr; + + /* Mailbox interrupt low enable set register */ + reg_addr = NPS_PKT_MBOX_INT_LO_ENA_W1S; + nitrox_write_csr(ndev, reg_addr, value); + + /* Mailbox interrupt high enable set register */ + reg_addr = NPS_PKT_MBOX_INT_HI_ENA_W1S; + nitrox_write_csr(ndev, reg_addr, value); +} + +void disable_pf2vf_mbox_interrupts(struct nitrox_device *ndev) +{ + u64 value = ~0ULL; + u64 reg_addr; + + /* Mailbox interrupt low enable clear register */ + reg_addr = NPS_PKT_MBOX_INT_LO_ENA_W1C; + nitrox_write_csr(ndev, reg_addr, value); + + /* Mailbox interrupt high enable clear register */ + reg_addr = NPS_PKT_MBOX_INT_HI_ENA_W1C; + nitrox_write_csr(ndev, reg_addr, value); +} diff --git a/drivers/crypto/cavium/nitrox/nitrox_hal.h b/drivers/crypto/cavium/nitrox/nitrox_hal.h new file mode 100644 index 000000000..48b0af039 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_hal.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __NITROX_HAL_H +#define __NITROX_HAL_H + +#include "nitrox_dev.h" + +void nitrox_config_aqm_rings(struct nitrox_device *ndev); +void nitrox_config_aqm_unit(struct nitrox_device *ndev); +void nitrox_config_emu_unit(struct nitrox_device *ndev); +void nitrox_config_pkt_input_rings(struct nitrox_device *ndev); +void nitrox_config_pkt_solicit_ports(struct nitrox_device *ndev); +void nitrox_config_nps_core_unit(struct nitrox_device *ndev); +void nitrox_config_nps_pkt_unit(struct nitrox_device *ndev); +void nitrox_config_pom_unit(struct nitrox_device *ndev); +void nitrox_config_rand_unit(struct nitrox_device *ndev); +void nitrox_config_efl_unit(struct nitrox_device *ndev); +void nitrox_config_bmi_unit(struct nitrox_device *ndev); +void nitrox_config_bmo_unit(struct nitrox_device *ndev); +void nitrox_config_lbc_unit(struct nitrox_device *ndev); +void invalidate_lbc(struct nitrox_device *ndev); +void enable_aqm_ring(struct nitrox_device *ndev, int qno); +void enable_pkt_input_ring(struct nitrox_device *ndev, int ring); +void enable_pkt_solicit_port(struct nitrox_device *ndev, int port); +void config_nps_core_vfcfg_mode(struct nitrox_device *ndev, enum vf_mode mode); +void nitrox_get_hwinfo(struct nitrox_device *ndev); +void enable_pf2vf_mbox_interrupts(struct nitrox_device *ndev); +void disable_pf2vf_mbox_interrupts(struct nitrox_device *ndev); + +#endif /* __NITROX_HAL_H */ diff --git a/drivers/crypto/cavium/nitrox/nitrox_isr.c b/drivers/crypto/cavium/nitrox/nitrox_isr.c new file mode 100644 index 000000000..f19e520da --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_isr.c @@ -0,0 +1,458 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/pci.h> +#include <linux/printk.h> +#include <linux/slab.h> + +#include "nitrox_dev.h" +#include "nitrox_csr.h" +#include "nitrox_common.h" +#include "nitrox_hal.h" +#include "nitrox_isr.h" +#include "nitrox_mbx.h" + +/* + * One vector for each type of ring + * - NPS packet ring, AQMQ ring and ZQMQ ring + */ +#define NR_RING_VECTORS 3 +#define NR_NON_RING_VECTORS 1 +/* base entry for packet ring/port */ +#define PKT_RING_MSIX_BASE 0 +#define NON_RING_MSIX_BASE 192 + +/** + * nps_pkt_slc_isr - IRQ handler for NPS solicit port + * @irq: irq number + * @data: argument + */ +static irqreturn_t nps_pkt_slc_isr(int irq, void *data) +{ + struct nitrox_q_vector *qvec = data; + union nps_pkt_slc_cnts slc_cnts; + struct nitrox_cmdq *cmdq = qvec->cmdq; + + slc_cnts.value = readq(cmdq->compl_cnt_csr_addr); + /* New packet on SLC output port */ + if (slc_cnts.s.slc_int) + tasklet_hi_schedule(&qvec->resp_tasklet); + + return IRQ_HANDLED; +} + +static void clear_nps_core_err_intr(struct nitrox_device *ndev) +{ + u64 value; + + /* Write 1 to clear */ + value = nitrox_read_csr(ndev, NPS_CORE_INT); + nitrox_write_csr(ndev, NPS_CORE_INT, value); + + dev_err_ratelimited(DEV(ndev), "NSP_CORE_INT 0x%016llx\n", value); +} + +static void clear_nps_pkt_err_intr(struct nitrox_device *ndev) +{ + union nps_pkt_int pkt_int; + unsigned long value, offset; + int i; + + pkt_int.value = nitrox_read_csr(ndev, NPS_PKT_INT); + dev_err_ratelimited(DEV(ndev), "NPS_PKT_INT 0x%016llx\n", + pkt_int.value); + + if (pkt_int.s.slc_err) { + offset = NPS_PKT_SLC_ERR_TYPE; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + dev_err_ratelimited(DEV(ndev), + "NPS_PKT_SLC_ERR_TYPE 0x%016lx\n", value); + + offset = NPS_PKT_SLC_RERR_LO; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + /* enable the solicit ports */ + for_each_set_bit(i, &value, BITS_PER_LONG) + enable_pkt_solicit_port(ndev, i); + + dev_err_ratelimited(DEV(ndev), + "NPS_PKT_SLC_RERR_LO 0x%016lx\n", value); + + offset = NPS_PKT_SLC_RERR_HI; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + dev_err_ratelimited(DEV(ndev), + "NPS_PKT_SLC_RERR_HI 0x%016lx\n", value); + } + + if (pkt_int.s.in_err) { + offset = NPS_PKT_IN_ERR_TYPE; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + dev_err_ratelimited(DEV(ndev), + "NPS_PKT_IN_ERR_TYPE 0x%016lx\n", value); + offset = NPS_PKT_IN_RERR_LO; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + /* enable the input ring */ + for_each_set_bit(i, &value, BITS_PER_LONG) + enable_pkt_input_ring(ndev, i); + + dev_err_ratelimited(DEV(ndev), + "NPS_PKT_IN_RERR_LO 0x%016lx\n", value); + + offset = NPS_PKT_IN_RERR_HI; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + dev_err_ratelimited(DEV(ndev), + "NPS_PKT_IN_RERR_HI 0x%016lx\n", value); + } +} + +static void clear_pom_err_intr(struct nitrox_device *ndev) +{ + u64 value; + + value = nitrox_read_csr(ndev, POM_INT); + nitrox_write_csr(ndev, POM_INT, value); + dev_err_ratelimited(DEV(ndev), "POM_INT 0x%016llx\n", value); +} + +static void clear_pem_err_intr(struct nitrox_device *ndev) +{ + u64 value; + + value = nitrox_read_csr(ndev, PEM0_INT); + nitrox_write_csr(ndev, PEM0_INT, value); + dev_err_ratelimited(DEV(ndev), "PEM(0)_INT 0x%016llx\n", value); +} + +static void clear_lbc_err_intr(struct nitrox_device *ndev) +{ + union lbc_int lbc_int; + u64 value, offset; + int i; + + lbc_int.value = nitrox_read_csr(ndev, LBC_INT); + dev_err_ratelimited(DEV(ndev), "LBC_INT 0x%016llx\n", lbc_int.value); + + if (lbc_int.s.dma_rd_err) { + for (i = 0; i < NR_CLUSTERS; i++) { + offset = EFL_CORE_VF_ERR_INT0X(i); + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + offset = EFL_CORE_VF_ERR_INT1X(i); + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + } + } + + if (lbc_int.s.cam_soft_err) { + dev_err_ratelimited(DEV(ndev), "CAM_SOFT_ERR, invalidating LBC\n"); + invalidate_lbc(ndev); + } + + if (lbc_int.s.pref_dat_len_mismatch_err) { + offset = LBC_PLM_VF1_64_INT; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + offset = LBC_PLM_VF65_128_INT; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + } + + if (lbc_int.s.rd_dat_len_mismatch_err) { + offset = LBC_ELM_VF1_64_INT; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + offset = LBC_ELM_VF65_128_INT; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + } + nitrox_write_csr(ndev, LBC_INT, lbc_int.value); +} + +static void clear_efl_err_intr(struct nitrox_device *ndev) +{ + int i; + + for (i = 0; i < NR_CLUSTERS; i++) { + union efl_core_int core_int; + u64 value, offset; + + offset = EFL_CORE_INTX(i); + core_int.value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, core_int.value); + dev_err_ratelimited(DEV(ndev), "ELF_CORE(%d)_INT 0x%016llx\n", + i, core_int.value); + if (core_int.s.se_err) { + offset = EFL_CORE_SE_ERR_INTX(i); + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + } + } +} + +static void clear_bmi_err_intr(struct nitrox_device *ndev) +{ + u64 value; + + value = nitrox_read_csr(ndev, BMI_INT); + nitrox_write_csr(ndev, BMI_INT, value); + dev_err_ratelimited(DEV(ndev), "BMI_INT 0x%016llx\n", value); +} + +static void nps_core_int_tasklet(unsigned long data) +{ + struct nitrox_q_vector *qvec = (void *)(uintptr_t)(data); + struct nitrox_device *ndev = qvec->ndev; + + /* if pf mode do queue recovery */ + if (ndev->mode == __NDEV_MODE_PF) { + } else { + /** + * if VF(s) enabled communicate the error information + * to VF(s) + */ + } +} + +/* + * nps_core_int_isr - interrupt handler for NITROX errors and + * mailbox communication + */ +static irqreturn_t nps_core_int_isr(int irq, void *data) +{ + struct nitrox_q_vector *qvec = data; + struct nitrox_device *ndev = qvec->ndev; + union nps_core_int_active core_int; + + core_int.value = nitrox_read_csr(ndev, NPS_CORE_INT_ACTIVE); + + if (core_int.s.nps_core) + clear_nps_core_err_intr(ndev); + + if (core_int.s.nps_pkt) + clear_nps_pkt_err_intr(ndev); + + if (core_int.s.pom) + clear_pom_err_intr(ndev); + + if (core_int.s.pem) + clear_pem_err_intr(ndev); + + if (core_int.s.lbc) + clear_lbc_err_intr(ndev); + + if (core_int.s.efl) + clear_efl_err_intr(ndev); + + if (core_int.s.bmi) + clear_bmi_err_intr(ndev); + + /* Mailbox interrupt */ + if (core_int.s.mbox) + nitrox_pf2vf_mbox_handler(ndev); + + /* If more work callback the ISR, set resend */ + core_int.s.resend = 1; + nitrox_write_csr(ndev, NPS_CORE_INT_ACTIVE, core_int.value); + + return IRQ_HANDLED; +} + +void nitrox_unregister_interrupts(struct nitrox_device *ndev) +{ + struct pci_dev *pdev = ndev->pdev; + int i; + + for (i = 0; i < ndev->num_vecs; i++) { + struct nitrox_q_vector *qvec; + int vec; + + qvec = ndev->qvec + i; + if (!qvec->valid) + continue; + + /* get the vector number */ + vec = pci_irq_vector(pdev, i); + irq_set_affinity_hint(vec, NULL); + free_irq(vec, qvec); + + tasklet_disable(&qvec->resp_tasklet); + tasklet_kill(&qvec->resp_tasklet); + qvec->valid = false; + } + kfree(ndev->qvec); + ndev->qvec = NULL; + pci_free_irq_vectors(pdev); +} + +int nitrox_register_interrupts(struct nitrox_device *ndev) +{ + struct pci_dev *pdev = ndev->pdev; + struct nitrox_q_vector *qvec; + int nr_vecs, vec, cpu; + int ret, i; + + /* + * PF MSI-X vectors + * + * Entry 0: NPS PKT ring 0 + * Entry 1: AQMQ ring 0 + * Entry 2: ZQM ring 0 + * Entry 3: NPS PKT ring 1 + * Entry 4: AQMQ ring 1 + * Entry 5: ZQM ring 1 + * .... + * Entry 192: NPS_CORE_INT_ACTIVE + */ + nr_vecs = pci_msix_vec_count(pdev); + if (nr_vecs < 0) { + dev_err(DEV(ndev), "Error in getting vec count %d\n", nr_vecs); + return nr_vecs; + } + + /* Enable MSI-X */ + ret = pci_alloc_irq_vectors(pdev, nr_vecs, nr_vecs, PCI_IRQ_MSIX); + if (ret < 0) { + dev_err(DEV(ndev), "msix vectors %d alloc failed\n", nr_vecs); + return ret; + } + ndev->num_vecs = nr_vecs; + + ndev->qvec = kcalloc(nr_vecs, sizeof(*qvec), GFP_KERNEL); + if (!ndev->qvec) { + pci_free_irq_vectors(pdev); + return -ENOMEM; + } + + /* request irqs for packet rings/ports */ + for (i = PKT_RING_MSIX_BASE; i < (nr_vecs - 1); i += NR_RING_VECTORS) { + qvec = &ndev->qvec[i]; + + qvec->ring = i / NR_RING_VECTORS; + if (qvec->ring >= ndev->nr_queues) + break; + + qvec->cmdq = &ndev->pkt_inq[qvec->ring]; + snprintf(qvec->name, IRQ_NAMESZ, "nitrox-pkt%d", qvec->ring); + /* get the vector number */ + vec = pci_irq_vector(pdev, i); + ret = request_irq(vec, nps_pkt_slc_isr, 0, qvec->name, qvec); + if (ret) { + dev_err(DEV(ndev), "irq failed for pkt ring/port%d\n", + qvec->ring); + goto irq_fail; + } + cpu = qvec->ring % num_online_cpus(); + irq_set_affinity_hint(vec, get_cpu_mask(cpu)); + + tasklet_init(&qvec->resp_tasklet, pkt_slc_resp_tasklet, + (unsigned long)qvec); + qvec->valid = true; + } + + /* request irqs for non ring vectors */ + i = NON_RING_MSIX_BASE; + qvec = &ndev->qvec[i]; + qvec->ndev = ndev; + + snprintf(qvec->name, IRQ_NAMESZ, "nitrox-core-int%d", i); + /* get the vector number */ + vec = pci_irq_vector(pdev, i); + ret = request_irq(vec, nps_core_int_isr, 0, qvec->name, qvec); + if (ret) { + dev_err(DEV(ndev), "irq failed for nitrox-core-int%d\n", i); + goto irq_fail; + } + cpu = num_online_cpus(); + irq_set_affinity_hint(vec, get_cpu_mask(cpu)); + + tasklet_init(&qvec->resp_tasklet, nps_core_int_tasklet, + (unsigned long)qvec); + qvec->valid = true; + + return 0; + +irq_fail: + nitrox_unregister_interrupts(ndev); + return ret; +} + +void nitrox_sriov_unregister_interrupts(struct nitrox_device *ndev) +{ + struct pci_dev *pdev = ndev->pdev; + int i; + + for (i = 0; i < ndev->num_vecs; i++) { + struct nitrox_q_vector *qvec; + int vec; + + qvec = ndev->qvec + i; + if (!qvec->valid) + continue; + + vec = ndev->iov.msix.vector; + irq_set_affinity_hint(vec, NULL); + free_irq(vec, qvec); + + tasklet_disable(&qvec->resp_tasklet); + tasklet_kill(&qvec->resp_tasklet); + qvec->valid = false; + } + kfree(ndev->qvec); + ndev->qvec = NULL; + pci_disable_msix(pdev); +} + +int nitrox_sriov_register_interupts(struct nitrox_device *ndev) +{ + struct pci_dev *pdev = ndev->pdev; + struct nitrox_q_vector *qvec; + int vec, cpu; + int ret; + + /** + * only non ring vectors i.e Entry 192 is available + * for PF in SR-IOV mode. + */ + ndev->iov.msix.entry = NON_RING_MSIX_BASE; + ret = pci_enable_msix_exact(pdev, &ndev->iov.msix, NR_NON_RING_VECTORS); + if (ret) { + dev_err(DEV(ndev), "failed to allocate nps-core-int%d\n", + NON_RING_MSIX_BASE); + return ret; + } + + qvec = kcalloc(NR_NON_RING_VECTORS, sizeof(*qvec), GFP_KERNEL); + if (!qvec) { + pci_disable_msix(pdev); + return -ENOMEM; + } + qvec->ndev = ndev; + + ndev->qvec = qvec; + ndev->num_vecs = NR_NON_RING_VECTORS; + snprintf(qvec->name, IRQ_NAMESZ, "nitrox-core-int%d", + NON_RING_MSIX_BASE); + + vec = ndev->iov.msix.vector; + ret = request_irq(vec, nps_core_int_isr, 0, qvec->name, qvec); + if (ret) { + dev_err(DEV(ndev), "irq failed for nitrox-core-int%d\n", + NON_RING_MSIX_BASE); + goto iov_irq_fail; + } + cpu = num_online_cpus(); + irq_set_affinity_hint(vec, get_cpu_mask(cpu)); + + tasklet_init(&qvec->resp_tasklet, nps_core_int_tasklet, + (unsigned long)qvec); + qvec->valid = true; + + return 0; + +iov_irq_fail: + nitrox_sriov_unregister_interrupts(ndev); + return ret; +} diff --git a/drivers/crypto/cavium/nitrox/nitrox_isr.h b/drivers/crypto/cavium/nitrox/nitrox_isr.h new file mode 100644 index 000000000..2bb123cd2 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_isr.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __NITROX_ISR_H +#define __NITROX_ISR_H + +#include "nitrox_dev.h" + +int nitrox_register_interrupts(struct nitrox_device *ndev); +void nitrox_unregister_interrupts(struct nitrox_device *ndev); +int nitrox_sriov_register_interupts(struct nitrox_device *ndev); +void nitrox_sriov_unregister_interrupts(struct nitrox_device *ndev); + +#ifdef CONFIG_PCI_IOV +int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs); +#else +static inline int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs) +{ + return 0; +} +#endif + +#endif /* __NITROX_ISR_H */ diff --git a/drivers/crypto/cavium/nitrox/nitrox_lib.c b/drivers/crypto/cavium/nitrox/nitrox_lib.c new file mode 100644 index 000000000..a5cdc2b48 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_lib.c @@ -0,0 +1,300 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/cpumask.h> +#include <linux/dma-mapping.h> +#include <linux/dmapool.h> +#include <linux/delay.h> +#include <linux/gfp.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/pci_regs.h> +#include <linux/vmalloc.h> +#include <linux/pci.h> + +#include "nitrox_dev.h" +#include "nitrox_common.h" +#include "nitrox_req.h" +#include "nitrox_csr.h" + +#define CRYPTO_CTX_SIZE 256 + +/* packet inuput ring alignments */ +#define PKTIN_Q_ALIGN_BYTES 16 +/* AQM Queue input alignments */ +#define AQM_Q_ALIGN_BYTES 32 + +static int nitrox_cmdq_init(struct nitrox_cmdq *cmdq, int align_bytes) +{ + struct nitrox_device *ndev = cmdq->ndev; + + cmdq->qsize = (ndev->qlen * cmdq->instr_size) + align_bytes; + cmdq->unalign_base = dma_alloc_coherent(DEV(ndev), cmdq->qsize, + &cmdq->unalign_dma, + GFP_KERNEL); + if (!cmdq->unalign_base) + return -ENOMEM; + + cmdq->dma = PTR_ALIGN(cmdq->unalign_dma, align_bytes); + cmdq->base = cmdq->unalign_base + (cmdq->dma - cmdq->unalign_dma); + cmdq->write_idx = 0; + + spin_lock_init(&cmdq->cmd_qlock); + spin_lock_init(&cmdq->resp_qlock); + spin_lock_init(&cmdq->backlog_qlock); + + INIT_LIST_HEAD(&cmdq->response_head); + INIT_LIST_HEAD(&cmdq->backlog_head); + INIT_WORK(&cmdq->backlog_qflush, backlog_qflush_work); + + atomic_set(&cmdq->pending_count, 0); + atomic_set(&cmdq->backlog_count, 0); + return 0; +} + +static void nitrox_cmdq_reset(struct nitrox_cmdq *cmdq) +{ + cmdq->write_idx = 0; + atomic_set(&cmdq->pending_count, 0); + atomic_set(&cmdq->backlog_count, 0); +} + +static void nitrox_cmdq_cleanup(struct nitrox_cmdq *cmdq) +{ + struct nitrox_device *ndev; + + if (!cmdq) + return; + + if (!cmdq->unalign_base) + return; + + ndev = cmdq->ndev; + cancel_work_sync(&cmdq->backlog_qflush); + + dma_free_coherent(DEV(ndev), cmdq->qsize, + cmdq->unalign_base, cmdq->unalign_dma); + nitrox_cmdq_reset(cmdq); + + cmdq->dbell_csr_addr = NULL; + cmdq->compl_cnt_csr_addr = NULL; + cmdq->unalign_base = NULL; + cmdq->base = NULL; + cmdq->unalign_dma = 0; + cmdq->dma = 0; + cmdq->qsize = 0; + cmdq->instr_size = 0; +} + +static void nitrox_free_aqm_queues(struct nitrox_device *ndev) +{ + int i; + + for (i = 0; i < ndev->nr_queues; i++) { + nitrox_cmdq_cleanup(ndev->aqmq[i]); + kfree_sensitive(ndev->aqmq[i]); + ndev->aqmq[i] = NULL; + } +} + +static int nitrox_alloc_aqm_queues(struct nitrox_device *ndev) +{ + int i, err; + + for (i = 0; i < ndev->nr_queues; i++) { + struct nitrox_cmdq *cmdq; + u64 offset; + + cmdq = kzalloc_node(sizeof(*cmdq), GFP_KERNEL, ndev->node); + if (!cmdq) { + err = -ENOMEM; + goto aqmq_fail; + } + + cmdq->ndev = ndev; + cmdq->qno = i; + cmdq->instr_size = sizeof(struct aqmq_command_s); + + /* AQM Queue Doorbell Counter Register Address */ + offset = AQMQ_DRBLX(i); + cmdq->dbell_csr_addr = NITROX_CSR_ADDR(ndev, offset); + /* AQM Queue Commands Completed Count Register Address */ + offset = AQMQ_CMD_CNTX(i); + cmdq->compl_cnt_csr_addr = NITROX_CSR_ADDR(ndev, offset); + + err = nitrox_cmdq_init(cmdq, AQM_Q_ALIGN_BYTES); + if (err) { + kfree_sensitive(cmdq); + goto aqmq_fail; + } + ndev->aqmq[i] = cmdq; + } + + return 0; + +aqmq_fail: + nitrox_free_aqm_queues(ndev); + return err; +} + +static void nitrox_free_pktin_queues(struct nitrox_device *ndev) +{ + int i; + + for (i = 0; i < ndev->nr_queues; i++) { + struct nitrox_cmdq *cmdq = &ndev->pkt_inq[i]; + + nitrox_cmdq_cleanup(cmdq); + } + kfree(ndev->pkt_inq); + ndev->pkt_inq = NULL; +} + +static int nitrox_alloc_pktin_queues(struct nitrox_device *ndev) +{ + int i, err; + + ndev->pkt_inq = kcalloc_node(ndev->nr_queues, + sizeof(struct nitrox_cmdq), + GFP_KERNEL, ndev->node); + if (!ndev->pkt_inq) + return -ENOMEM; + + for (i = 0; i < ndev->nr_queues; i++) { + struct nitrox_cmdq *cmdq; + u64 offset; + + cmdq = &ndev->pkt_inq[i]; + cmdq->ndev = ndev; + cmdq->qno = i; + cmdq->instr_size = sizeof(struct nps_pkt_instr); + + /* packet input ring doorbell address */ + offset = NPS_PKT_IN_INSTR_BAOFF_DBELLX(i); + cmdq->dbell_csr_addr = NITROX_CSR_ADDR(ndev, offset); + /* packet solicit port completion count address */ + offset = NPS_PKT_SLC_CNTSX(i); + cmdq->compl_cnt_csr_addr = NITROX_CSR_ADDR(ndev, offset); + + err = nitrox_cmdq_init(cmdq, PKTIN_Q_ALIGN_BYTES); + if (err) + goto pktq_fail; + } + return 0; + +pktq_fail: + nitrox_free_pktin_queues(ndev); + return err; +} + +static int create_crypto_dma_pool(struct nitrox_device *ndev) +{ + size_t size; + + /* Crypto context pool, 16 byte aligned */ + size = CRYPTO_CTX_SIZE + sizeof(struct ctx_hdr); + ndev->ctx_pool = dma_pool_create("nitrox-context", + DEV(ndev), size, 16, 0); + if (!ndev->ctx_pool) + return -ENOMEM; + + return 0; +} + +static void destroy_crypto_dma_pool(struct nitrox_device *ndev) +{ + if (!ndev->ctx_pool) + return; + + dma_pool_destroy(ndev->ctx_pool); + ndev->ctx_pool = NULL; +} + +/* + * crypto_alloc_context - Allocate crypto context from pool + * @ndev: NITROX Device + */ +void *crypto_alloc_context(struct nitrox_device *ndev) +{ + struct ctx_hdr *ctx; + struct crypto_ctx_hdr *chdr; + void *vaddr; + dma_addr_t dma; + + chdr = kmalloc(sizeof(*chdr), GFP_KERNEL); + if (!chdr) + return NULL; + + vaddr = dma_pool_zalloc(ndev->ctx_pool, GFP_KERNEL, &dma); + if (!vaddr) { + kfree(chdr); + return NULL; + } + + /* fill meta data */ + ctx = vaddr; + ctx->pool = ndev->ctx_pool; + ctx->dma = dma; + ctx->ctx_dma = dma + sizeof(struct ctx_hdr); + + chdr->pool = ndev->ctx_pool; + chdr->dma = dma; + chdr->vaddr = vaddr; + + return chdr; +} + +/** + * crypto_free_context - Free crypto context to pool + * @ctx: context to free + */ +void crypto_free_context(void *ctx) +{ + struct crypto_ctx_hdr *ctxp; + + if (!ctx) + return; + + ctxp = ctx; + dma_pool_free(ctxp->pool, ctxp->vaddr, ctxp->dma); + kfree(ctxp); +} + +/** + * nitrox_common_sw_init - allocate software resources. + * @ndev: NITROX device + * + * Allocates crypto context pools and command queues etc. + * + * Return: 0 on success, or a negative error code on error. + */ +int nitrox_common_sw_init(struct nitrox_device *ndev) +{ + int err = 0; + + /* per device crypto context pool */ + err = create_crypto_dma_pool(ndev); + if (err) + return err; + + err = nitrox_alloc_pktin_queues(ndev); + if (err) + destroy_crypto_dma_pool(ndev); + + err = nitrox_alloc_aqm_queues(ndev); + if (err) { + nitrox_free_pktin_queues(ndev); + destroy_crypto_dma_pool(ndev); + } + + return err; +} + +/** + * nitrox_common_sw_cleanup - free software resources. + * @ndev: NITROX device + */ +void nitrox_common_sw_cleanup(struct nitrox_device *ndev) +{ + nitrox_free_aqm_queues(ndev); + nitrox_free_pktin_queues(ndev); + destroy_crypto_dma_pool(ndev); +} diff --git a/drivers/crypto/cavium/nitrox/nitrox_main.c b/drivers/crypto/cavium/nitrox/nitrox_main.c new file mode 100644 index 000000000..432a61aca --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_main.c @@ -0,0 +1,583 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include <linux/aer.h> +#include <linux/delay.h> +#include <linux/firmware.h> +#include <linux/list.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/pci.h> +#include <linux/pci_ids.h> + +#include "nitrox_dev.h" +#include "nitrox_common.h" +#include "nitrox_csr.h" +#include "nitrox_hal.h" +#include "nitrox_isr.h" +#include "nitrox_debugfs.h" + +#define CNN55XX_DEV_ID 0x12 +#define UCODE_HLEN 48 +#define DEFAULT_SE_GROUP 0 +#define DEFAULT_AE_GROUP 0 + +#define DRIVER_VERSION "1.2" +#define CNN55XX_UCD_BLOCK_SIZE 32768 +#define CNN55XX_MAX_UCODE_SIZE (CNN55XX_UCD_BLOCK_SIZE * 2) +#define FW_DIR "cavium/" +/* SE microcode */ +#define SE_FW FW_DIR "cnn55xx_se.fw" +/* AE microcode */ +#define AE_FW FW_DIR "cnn55xx_ae.fw" + +static const char nitrox_driver_name[] = "CNN55XX"; + +static LIST_HEAD(ndevlist); +static DEFINE_MUTEX(devlist_lock); +static unsigned int num_devices; + +/* + * nitrox_pci_tbl - PCI Device ID Table + */ +static const struct pci_device_id nitrox_pci_tbl[] = { + {PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0}, + /* required last entry */ + {0, } +}; +MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl); + +static unsigned int qlen = DEFAULT_CMD_QLEN; +module_param(qlen, uint, 0644); +MODULE_PARM_DESC(qlen, "Command queue length - default 2048"); + +/** + * struct ucode - Firmware Header + * @id: microcode ID + * @version: firmware version + * @code_size: code section size + * @raz: alignment + * @code: code section + */ +struct ucode { + u8 id; + char version[VERSION_LEN - 1]; + __be32 code_size; + u8 raz[12]; + u64 code[]; +}; + +/* + * write_to_ucd_unit - Write Firmware to NITROX UCD unit + */ +static void write_to_ucd_unit(struct nitrox_device *ndev, u32 ucode_size, + u64 *ucode_data, int block_num) +{ + u32 code_size; + u64 offset, data; + int i = 0; + + /* + * UCD structure + * + * ------------- + * | BLK 7 | + * ------------- + * | BLK 6 | + * ------------- + * | ... | + * ------------- + * | BLK 0 | + * ------------- + * Total of 8 blocks, each size 32KB + */ + + /* set the block number */ + offset = UCD_UCODE_LOAD_BLOCK_NUM; + nitrox_write_csr(ndev, offset, block_num); + + code_size = roundup(ucode_size, 16); + while (code_size) { + data = ucode_data[i]; + /* write 8 bytes at a time */ + offset = UCD_UCODE_LOAD_IDX_DATAX(i); + nitrox_write_csr(ndev, offset, data); + code_size -= 8; + i++; + } + + usleep_range(300, 400); +} + +static int nitrox_load_fw(struct nitrox_device *ndev) +{ + const struct firmware *fw; + const char *fw_name; + struct ucode *ucode; + u64 *ucode_data; + u64 offset; + union ucd_core_eid_ucode_block_num core_2_eid_val; + union aqm_grp_execmsk_lo aqm_grp_execmask_lo; + union aqm_grp_execmsk_hi aqm_grp_execmask_hi; + u32 ucode_size; + int ret, i = 0; + + fw_name = SE_FW; + dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name); + + ret = request_firmware(&fw, fw_name, DEV(ndev)); + if (ret < 0) { + dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name); + return ret; + } + + ucode = (struct ucode *)fw->data; + + ucode_size = be32_to_cpu(ucode->code_size) * 2; + if (!ucode_size || ucode_size > CNN55XX_MAX_UCODE_SIZE) { + dev_err(DEV(ndev), "Invalid ucode size: %u for firmware %s\n", + ucode_size, fw_name); + release_firmware(fw); + return -EINVAL; + } + ucode_data = ucode->code; + + /* copy the firmware version */ + memcpy(&ndev->hw.fw_name[0][0], ucode->version, (VERSION_LEN - 2)); + ndev->hw.fw_name[0][VERSION_LEN - 1] = '\0'; + + /* Load SE Firmware on UCD Block 0 */ + write_to_ucd_unit(ndev, ucode_size, ucode_data, 0); + + release_firmware(fw); + + /* put all SE cores in DEFAULT_SE_GROUP */ + offset = POM_GRP_EXECMASKX(DEFAULT_SE_GROUP); + nitrox_write_csr(ndev, offset, (~0ULL)); + + /* write block number and firmware length + * bit:<2:0> block number + * bit:3 is set SE uses 32KB microcode + * bit:3 is clear SE uses 64KB microcode + */ + core_2_eid_val.value = 0ULL; + core_2_eid_val.ucode_blk = 0; + if (ucode_size <= CNN55XX_UCD_BLOCK_SIZE) + core_2_eid_val.ucode_len = 1; + else + core_2_eid_val.ucode_len = 0; + + for (i = 0; i < ndev->hw.se_cores; i++) { + offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i); + nitrox_write_csr(ndev, offset, core_2_eid_val.value); + } + + + fw_name = AE_FW; + dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name); + + ret = request_firmware(&fw, fw_name, DEV(ndev)); + if (ret < 0) { + dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name); + return ret; + } + + ucode = (struct ucode *)fw->data; + + ucode_size = be32_to_cpu(ucode->code_size) * 2; + if (!ucode_size || ucode_size > CNN55XX_MAX_UCODE_SIZE) { + dev_err(DEV(ndev), "Invalid ucode size: %u for firmware %s\n", + ucode_size, fw_name); + release_firmware(fw); + return -EINVAL; + } + ucode_data = ucode->code; + + /* copy the firmware version */ + memcpy(&ndev->hw.fw_name[1][0], ucode->version, (VERSION_LEN - 2)); + ndev->hw.fw_name[1][VERSION_LEN - 1] = '\0'; + + /* Load AE Firmware on UCD Block 2 */ + write_to_ucd_unit(ndev, ucode_size, ucode_data, 2); + + release_firmware(fw); + + /* put all AE cores in DEFAULT_AE_GROUP */ + offset = AQM_GRP_EXECMSK_LOX(DEFAULT_AE_GROUP); + aqm_grp_execmask_lo.exec_0_to_39 = 0xFFFFFFFFFFULL; + nitrox_write_csr(ndev, offset, aqm_grp_execmask_lo.value); + offset = AQM_GRP_EXECMSK_HIX(DEFAULT_AE_GROUP); + aqm_grp_execmask_hi.exec_40_to_79 = 0xFFFFFFFFFFULL; + nitrox_write_csr(ndev, offset, aqm_grp_execmask_hi.value); + + /* write block number and firmware length + * bit:<2:0> block number + * bit:3 is set AE uses 32KB microcode + * bit:3 is clear AE uses 64KB microcode + */ + core_2_eid_val.value = 0ULL; + core_2_eid_val.ucode_blk = 2; + if (ucode_size <= CNN55XX_UCD_BLOCK_SIZE) + core_2_eid_val.ucode_len = 1; + else + core_2_eid_val.ucode_len = 0; + + for (i = 0; i < ndev->hw.ae_cores; i++) { + offset = UCD_AE_EID_UCODE_BLOCK_NUMX(i); + nitrox_write_csr(ndev, offset, core_2_eid_val.value); + } + + return 0; +} + +/** + * nitrox_add_to_devlist - add NITROX device to global device list + * @ndev: NITROX device + */ +static int nitrox_add_to_devlist(struct nitrox_device *ndev) +{ + struct nitrox_device *dev; + int ret = 0; + + INIT_LIST_HEAD(&ndev->list); + refcount_set(&ndev->refcnt, 1); + + mutex_lock(&devlist_lock); + list_for_each_entry(dev, &ndevlist, list) { + if (dev == ndev) { + ret = -EEXIST; + goto unlock; + } + } + ndev->idx = num_devices++; + list_add_tail(&ndev->list, &ndevlist); +unlock: + mutex_unlock(&devlist_lock); + return ret; +} + +/** + * nitrox_remove_from_devlist - remove NITROX device from + * global device list + * @ndev: NITROX device + */ +static void nitrox_remove_from_devlist(struct nitrox_device *ndev) +{ + mutex_lock(&devlist_lock); + list_del(&ndev->list); + num_devices--; + mutex_unlock(&devlist_lock); +} + +struct nitrox_device *nitrox_get_first_device(void) +{ + struct nitrox_device *ndev = NULL, *iter; + + mutex_lock(&devlist_lock); + list_for_each_entry(iter, &ndevlist, list) { + if (nitrox_ready(iter)) { + ndev = iter; + break; + } + } + mutex_unlock(&devlist_lock); + if (!ndev) + return NULL; + + refcount_inc(&ndev->refcnt); + /* barrier to sync with other cpus */ + smp_mb__after_atomic(); + return ndev; +} + +void nitrox_put_device(struct nitrox_device *ndev) +{ + if (!ndev) + return; + + refcount_dec(&ndev->refcnt); + /* barrier to sync with other cpus */ + smp_mb__after_atomic(); +} + +static int nitrox_device_flr(struct pci_dev *pdev) +{ + int pos = 0; + + pos = pci_save_state(pdev); + if (pos) { + dev_err(&pdev->dev, "Failed to save pci state\n"); + return -ENOMEM; + } + + pcie_reset_flr(pdev, PCI_RESET_DO_RESET); + + pci_restore_state(pdev); + + return 0; +} + +static int nitrox_pf_sw_init(struct nitrox_device *ndev) +{ + int err; + + err = nitrox_common_sw_init(ndev); + if (err) + return err; + + err = nitrox_register_interrupts(ndev); + if (err) + nitrox_common_sw_cleanup(ndev); + + return err; +} + +static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev) +{ + nitrox_unregister_interrupts(ndev); + nitrox_common_sw_cleanup(ndev); +} + +/** + * nitrox_bist_check - Check NITROX BIST registers status + * @ndev: NITROX device + */ +static int nitrox_bist_check(struct nitrox_device *ndev) +{ + u64 value = 0; + int i; + + for (i = 0; i < NR_CLUSTERS; i++) { + value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i)); + value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i)); + } + value += nitrox_read_csr(ndev, UCD_BIST_STATUS); + value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG); + value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG); + value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG); + value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG); + value += nitrox_read_csr(ndev, POM_BIST_REG); + value += nitrox_read_csr(ndev, BMI_BIST_REG); + value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT); + value += nitrox_read_csr(ndev, BMO_BIST_REG); + value += nitrox_read_csr(ndev, LBC_BIST_STATUS); + value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0)); + if (value) + return -EIO; + return 0; +} + +static int nitrox_pf_hw_init(struct nitrox_device *ndev) +{ + int err; + + err = nitrox_bist_check(ndev); + if (err) { + dev_err(&ndev->pdev->dev, "BIST check failed\n"); + return err; + } + /* get cores information */ + nitrox_get_hwinfo(ndev); + + nitrox_config_nps_core_unit(ndev); + nitrox_config_aqm_unit(ndev); + nitrox_config_nps_pkt_unit(ndev); + nitrox_config_pom_unit(ndev); + nitrox_config_efl_unit(ndev); + /* configure IO units */ + nitrox_config_bmi_unit(ndev); + nitrox_config_bmo_unit(ndev); + /* configure Local Buffer Cache */ + nitrox_config_lbc_unit(ndev); + nitrox_config_rand_unit(ndev); + + /* load firmware on cores */ + err = nitrox_load_fw(ndev); + if (err) + return err; + + nitrox_config_emu_unit(ndev); + + return 0; +} + +/** + * nitrox_probe - NITROX Initialization function. + * @pdev: PCI device information struct + * @id: entry in nitrox_pci_tbl + * + * Return: 0, if the driver is bound to the device, or + * a negative error if there is failure. + */ +static int nitrox_probe(struct pci_dev *pdev, + const struct pci_device_id *id) +{ + struct nitrox_device *ndev; + int err; + + dev_info_once(&pdev->dev, "%s driver version %s\n", + nitrox_driver_name, DRIVER_VERSION); + + err = pci_enable_device_mem(pdev); + if (err) + return err; + + /* do FLR */ + err = nitrox_device_flr(pdev); + if (err) { + dev_err(&pdev->dev, "FLR failed\n"); + goto flr_fail; + } + + if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) { + dev_dbg(&pdev->dev, "DMA to 64-BIT address\n"); + } else { + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (err) { + dev_err(&pdev->dev, "DMA configuration failed\n"); + goto flr_fail; + } + } + + err = pci_request_mem_regions(pdev, nitrox_driver_name); + if (err) + goto flr_fail; + pci_set_master(pdev); + + ndev = kzalloc(sizeof(*ndev), GFP_KERNEL); + if (!ndev) { + err = -ENOMEM; + goto ndev_fail; + } + + pci_set_drvdata(pdev, ndev); + ndev->pdev = pdev; + + /* add to device list */ + nitrox_add_to_devlist(ndev); + + ndev->hw.vendor_id = pdev->vendor; + ndev->hw.device_id = pdev->device; + ndev->hw.revision_id = pdev->revision; + /* command timeout in jiffies */ + ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT); + ndev->node = dev_to_node(&pdev->dev); + if (ndev->node == NUMA_NO_NODE) + ndev->node = 0; + + ndev->bar_addr = ioremap(pci_resource_start(pdev, 0), + pci_resource_len(pdev, 0)); + if (!ndev->bar_addr) { + err = -EIO; + goto ioremap_err; + } + /* allocate command queus based on cpus, max queues are 64 */ + ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus()); + ndev->qlen = qlen; + + err = nitrox_pf_sw_init(ndev); + if (err) + goto pf_sw_fail; + + err = nitrox_pf_hw_init(ndev); + if (err) + goto pf_hw_fail; + + nitrox_debugfs_init(ndev); + + /* clear the statistics */ + atomic64_set(&ndev->stats.posted, 0); + atomic64_set(&ndev->stats.completed, 0); + atomic64_set(&ndev->stats.dropped, 0); + + atomic_set(&ndev->state, __NDEV_READY); + /* barrier to sync with other cpus */ + smp_mb__after_atomic(); + + err = nitrox_crypto_register(); + if (err) + goto crypto_fail; + + return 0; + +crypto_fail: + nitrox_debugfs_exit(ndev); + atomic_set(&ndev->state, __NDEV_NOT_READY); + /* barrier to sync with other cpus */ + smp_mb__after_atomic(); +pf_hw_fail: + nitrox_pf_sw_cleanup(ndev); +pf_sw_fail: + iounmap(ndev->bar_addr); +ioremap_err: + nitrox_remove_from_devlist(ndev); + kfree(ndev); + pci_set_drvdata(pdev, NULL); +ndev_fail: + pci_release_mem_regions(pdev); +flr_fail: + pci_disable_device(pdev); + return err; +} + +/** + * nitrox_remove - Unbind the driver from the device. + * @pdev: PCI device information struct + */ +static void nitrox_remove(struct pci_dev *pdev) +{ + struct nitrox_device *ndev = pci_get_drvdata(pdev); + + if (!ndev) + return; + + if (!refcount_dec_and_test(&ndev->refcnt)) { + dev_err(DEV(ndev), "Device refcnt not zero (%d)\n", + refcount_read(&ndev->refcnt)); + return; + } + + dev_info(DEV(ndev), "Removing Device %x:%x\n", + ndev->hw.vendor_id, ndev->hw.device_id); + + atomic_set(&ndev->state, __NDEV_NOT_READY); + /* barrier to sync with other cpus */ + smp_mb__after_atomic(); + + nitrox_remove_from_devlist(ndev); + + /* disable SR-IOV */ + nitrox_sriov_configure(pdev, 0); + nitrox_crypto_unregister(); + nitrox_debugfs_exit(ndev); + nitrox_pf_sw_cleanup(ndev); + + iounmap(ndev->bar_addr); + kfree(ndev); + + pci_set_drvdata(pdev, NULL); + pci_release_mem_regions(pdev); + pci_disable_device(pdev); +} + +static void nitrox_shutdown(struct pci_dev *pdev) +{ + pci_set_drvdata(pdev, NULL); + pci_release_mem_regions(pdev); + pci_disable_device(pdev); +} + +static struct pci_driver nitrox_driver = { + .name = nitrox_driver_name, + .id_table = nitrox_pci_tbl, + .probe = nitrox_probe, + .remove = nitrox_remove, + .shutdown = nitrox_shutdown, + .sriov_configure = nitrox_sriov_configure, +}; + +module_pci_driver(nitrox_driver); + +MODULE_AUTHOR("Srikanth Jampala <Jampala.Srikanth@cavium.com>"); +MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " "); +MODULE_LICENSE("GPL"); +MODULE_VERSION(DRIVER_VERSION); +MODULE_FIRMWARE(SE_FW); diff --git a/drivers/crypto/cavium/nitrox/nitrox_mbx.c b/drivers/crypto/cavium/nitrox/nitrox_mbx.c new file mode 100644 index 000000000..d4e06999a --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_mbx.c @@ -0,0 +1,218 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/bitmap.h> +#include <linux/workqueue.h> + +#include "nitrox_csr.h" +#include "nitrox_hal.h" +#include "nitrox_dev.h" +#include "nitrox_mbx.h" + +#define RING_TO_VFNO(_x, _y) ((_x) / (_y)) + +/* + * mbx_msg_type - Mailbox message types + */ +enum mbx_msg_type { + MBX_MSG_TYPE_NOP, + MBX_MSG_TYPE_REQ, + MBX_MSG_TYPE_ACK, + MBX_MSG_TYPE_NACK, +}; + +/* + * mbx_msg_opcode - Mailbox message opcodes + */ +enum mbx_msg_opcode { + MSG_OP_VF_MODE = 1, + MSG_OP_VF_UP, + MSG_OP_VF_DOWN, + MSG_OP_CHIPID_VFID, + MSG_OP_MCODE_INFO = 11, +}; + +struct pf2vf_work { + struct nitrox_vfdev *vfdev; + struct nitrox_device *ndev; + struct work_struct pf2vf_resp; +}; + +static inline u64 pf2vf_read_mbox(struct nitrox_device *ndev, int ring) +{ + u64 reg_addr; + + reg_addr = NPS_PKT_MBOX_VF_PF_PFDATAX(ring); + return nitrox_read_csr(ndev, reg_addr); +} + +static inline void pf2vf_write_mbox(struct nitrox_device *ndev, u64 value, + int ring) +{ + u64 reg_addr; + + reg_addr = NPS_PKT_MBOX_PF_VF_PFDATAX(ring); + nitrox_write_csr(ndev, reg_addr, value); +} + +static void pf2vf_send_response(struct nitrox_device *ndev, + struct nitrox_vfdev *vfdev) +{ + union mbox_msg msg; + + msg.value = vfdev->msg.value; + + switch (vfdev->msg.opcode) { + case MSG_OP_VF_MODE: + msg.data = ndev->mode; + break; + case MSG_OP_VF_UP: + vfdev->nr_queues = vfdev->msg.data; + atomic_set(&vfdev->state, __NDEV_READY); + break; + case MSG_OP_CHIPID_VFID: + msg.id.chipid = ndev->idx; + msg.id.vfid = vfdev->vfno; + break; + case MSG_OP_VF_DOWN: + vfdev->nr_queues = 0; + atomic_set(&vfdev->state, __NDEV_NOT_READY); + break; + case MSG_OP_MCODE_INFO: + msg.data = 0; + msg.mcode_info.count = 2; + msg.mcode_info.info = MCODE_TYPE_SE_SSL | (MCODE_TYPE_AE << 5); + msg.mcode_info.next_se_grp = 1; + msg.mcode_info.next_ae_grp = 1; + break; + default: + msg.type = MBX_MSG_TYPE_NOP; + break; + } + + if (msg.type == MBX_MSG_TYPE_NOP) + return; + + /* send ACK to VF */ + msg.type = MBX_MSG_TYPE_ACK; + pf2vf_write_mbox(ndev, msg.value, vfdev->ring); + + vfdev->msg.value = 0; + atomic64_inc(&vfdev->mbx_resp); +} + +static void pf2vf_resp_handler(struct work_struct *work) +{ + struct pf2vf_work *pf2vf_resp = container_of(work, struct pf2vf_work, + pf2vf_resp); + struct nitrox_vfdev *vfdev = pf2vf_resp->vfdev; + struct nitrox_device *ndev = pf2vf_resp->ndev; + + switch (vfdev->msg.type) { + case MBX_MSG_TYPE_REQ: + /* process the request from VF */ + pf2vf_send_response(ndev, vfdev); + break; + case MBX_MSG_TYPE_ACK: + case MBX_MSG_TYPE_NACK: + break; + } + + kfree(pf2vf_resp); +} + +void nitrox_pf2vf_mbox_handler(struct nitrox_device *ndev) +{ + DECLARE_BITMAP(csr, BITS_PER_TYPE(u64)); + struct nitrox_vfdev *vfdev; + struct pf2vf_work *pfwork; + u64 value, reg_addr; + u32 i; + int vfno; + + /* loop for VF(0..63) */ + reg_addr = NPS_PKT_MBOX_INT_LO; + value = nitrox_read_csr(ndev, reg_addr); + bitmap_from_u64(csr, value); + for_each_set_bit(i, csr, BITS_PER_TYPE(csr)) { + /* get the vfno from ring */ + vfno = RING_TO_VFNO(i, ndev->iov.max_vf_queues); + vfdev = ndev->iov.vfdev + vfno; + vfdev->ring = i; + /* fill the vf mailbox data */ + vfdev->msg.value = pf2vf_read_mbox(ndev, vfdev->ring); + pfwork = kzalloc(sizeof(*pfwork), GFP_ATOMIC); + if (!pfwork) + continue; + + pfwork->vfdev = vfdev; + pfwork->ndev = ndev; + INIT_WORK(&pfwork->pf2vf_resp, pf2vf_resp_handler); + queue_work(ndev->iov.pf2vf_wq, &pfwork->pf2vf_resp); + /* clear the corresponding vf bit */ + nitrox_write_csr(ndev, reg_addr, BIT_ULL(i)); + } + + /* loop for VF(64..127) */ + reg_addr = NPS_PKT_MBOX_INT_HI; + value = nitrox_read_csr(ndev, reg_addr); + bitmap_from_u64(csr, value); + for_each_set_bit(i, csr, BITS_PER_TYPE(csr)) { + /* get the vfno from ring */ + vfno = RING_TO_VFNO(i + 64, ndev->iov.max_vf_queues); + vfdev = ndev->iov.vfdev + vfno; + vfdev->ring = (i + 64); + /* fill the vf mailbox data */ + vfdev->msg.value = pf2vf_read_mbox(ndev, vfdev->ring); + + pfwork = kzalloc(sizeof(*pfwork), GFP_ATOMIC); + if (!pfwork) + continue; + + pfwork->vfdev = vfdev; + pfwork->ndev = ndev; + INIT_WORK(&pfwork->pf2vf_resp, pf2vf_resp_handler); + queue_work(ndev->iov.pf2vf_wq, &pfwork->pf2vf_resp); + /* clear the corresponding vf bit */ + nitrox_write_csr(ndev, reg_addr, BIT_ULL(i)); + } +} + +int nitrox_mbox_init(struct nitrox_device *ndev) +{ + struct nitrox_vfdev *vfdev; + int i; + + ndev->iov.vfdev = kcalloc(ndev->iov.num_vfs, + sizeof(struct nitrox_vfdev), GFP_KERNEL); + if (!ndev->iov.vfdev) + return -ENOMEM; + + for (i = 0; i < ndev->iov.num_vfs; i++) { + vfdev = ndev->iov.vfdev + i; + vfdev->vfno = i; + } + + /* allocate pf2vf response workqueue */ + ndev->iov.pf2vf_wq = alloc_workqueue("nitrox_pf2vf", 0, 0); + if (!ndev->iov.pf2vf_wq) { + kfree(ndev->iov.vfdev); + ndev->iov.vfdev = NULL; + return -ENOMEM; + } + /* enable pf2vf mailbox interrupts */ + enable_pf2vf_mbox_interrupts(ndev); + + return 0; +} + +void nitrox_mbox_cleanup(struct nitrox_device *ndev) +{ + /* disable pf2vf mailbox interrupts */ + disable_pf2vf_mbox_interrupts(ndev); + /* destroy workqueue */ + if (ndev->iov.pf2vf_wq) + destroy_workqueue(ndev->iov.pf2vf_wq); + + kfree(ndev->iov.vfdev); + ndev->iov.pf2vf_wq = NULL; + ndev->iov.vfdev = NULL; +} diff --git a/drivers/crypto/cavium/nitrox/nitrox_mbx.h b/drivers/crypto/cavium/nitrox/nitrox_mbx.h new file mode 100644 index 000000000..7c93d0282 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_mbx.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __NITROX_MBX_H +#define __NITROX_MBX_H + +int nitrox_mbox_init(struct nitrox_device *ndev); +void nitrox_mbox_cleanup(struct nitrox_device *ndev); +void nitrox_pf2vf_mbox_handler(struct nitrox_device *ndev); + +#endif /* __NITROX_MBX_H */ diff --git a/drivers/crypto/cavium/nitrox/nitrox_req.h b/drivers/crypto/cavium/nitrox/nitrox_req.h new file mode 100644 index 000000000..6bf088bcd --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_req.h @@ -0,0 +1,728 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __NITROX_REQ_H +#define __NITROX_REQ_H + +#include <linux/dma-mapping.h> +#include <crypto/aes.h> + +#include "nitrox_dev.h" + +#define PENDING_SIG 0xFFFFFFFFFFFFFFFFUL +#define PRIO 4001 + +typedef void (*sereq_completion_t)(void *req, int err); + +/** + * struct gphdr - General purpose Header + * @param0: first parameter. + * @param1: second parameter. + * @param2: third parameter. + * @param3: fourth parameter. + * + * Params tell the iv and enc/dec data offsets. + */ +struct gphdr { + __be16 param0; + __be16 param1; + __be16 param2; + __be16 param3; +}; + +/** + * struct se_req_ctrl - SE request information. + * @arg: Minor number of the opcode + * @ctxc: Context control. + * @unca: Uncertainity enabled. + * @info: Additional information for SE cores. + * @ctxl: Context length in bytes. + * @uddl: User defined data length + */ +union se_req_ctrl { + u64 value; + struct { + u64 raz : 22; + u64 arg : 8; + u64 ctxc : 2; + u64 unca : 1; + u64 info : 3; + u64 unc : 8; + u64 ctxl : 12; + u64 uddl : 8; + } s; +}; + +#define MAX_IV_LEN 16 + +/** + * struct se_crypto_request - SE crypto request structure. + * @opcode: Request opcode (enc/dec) + * @flags: flags from crypto subsystem + * @ctx_handle: Crypto context handle. + * @gph: GP Header + * @ctrl: Request Information. + * @orh: ORH address + * @comp: completion address + * @src: Input sglist + * @dst: Output sglist + */ +struct se_crypto_request { + u8 opcode; + gfp_t gfp; + u32 flags; + u64 ctx_handle; + + struct gphdr gph; + union se_req_ctrl ctrl; + u64 *orh; + u64 *comp; + + struct scatterlist *src; + struct scatterlist *dst; +}; + +/* Crypto opcodes */ +#define FLEXI_CRYPTO_ENCRYPT_HMAC 0x33 +#define ENCRYPT 0 +#define DECRYPT 1 + +/* IV from context */ +#define IV_FROM_CTX 0 +/* IV from Input data */ +#define IV_FROM_DPTR 1 + +/** + * cipher opcodes for firmware + */ +enum flexi_cipher { + CIPHER_NULL = 0, + CIPHER_3DES_CBC, + CIPHER_3DES_ECB, + CIPHER_AES_CBC, + CIPHER_AES_ECB, + CIPHER_AES_CFB, + CIPHER_AES_CTR, + CIPHER_AES_GCM, + CIPHER_AES_XTS, + CIPHER_AES_CCM, + CIPHER_AES_CBC_CTS, + CIPHER_AES_ECB_CTS, + CIPHER_INVALID +}; + +enum flexi_auth { + AUTH_NULL = 0, + AUTH_MD5, + AUTH_SHA1, + AUTH_SHA2_SHA224, + AUTH_SHA2_SHA256, + AUTH_SHA2_SHA384, + AUTH_SHA2_SHA512, + AUTH_GMAC, + AUTH_INVALID +}; + +/** + * struct crypto_keys - Crypto keys + * @key: Encryption key or KEY1 for AES-XTS + * @iv: Encryption IV or Tweak for AES-XTS + */ +struct crypto_keys { + union { + u8 key[AES_MAX_KEY_SIZE]; + u8 key1[AES_MAX_KEY_SIZE]; + } u; + u8 iv[AES_BLOCK_SIZE]; +}; + +/** + * struct auth_keys - Authentication keys + * @ipad: IPAD or KEY2 for AES-XTS + * @opad: OPAD or AUTH KEY if auth_input_type = 1 + */ +struct auth_keys { + union { + u8 ipad[64]; + u8 key2[64]; + } u; + u8 opad[64]; +}; + +union fc_ctx_flags { + __be64 f; + u64 fu; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 cipher_type : 4; + u64 reserved_59 : 1; + u64 aes_keylen : 2; + u64 iv_source : 1; + u64 hash_type : 4; + u64 reserved_49_51 : 3; + u64 auth_input_type: 1; + u64 mac_len : 8; + u64 reserved_0_39 : 40; +#else + u64 reserved_0_39 : 40; + u64 mac_len : 8; + u64 auth_input_type: 1; + u64 reserved_49_51 : 3; + u64 hash_type : 4; + u64 iv_source : 1; + u64 aes_keylen : 2; + u64 reserved_59 : 1; + u64 cipher_type : 4; +#endif + } w0; +}; +/** + * struct flexi_crypto_context - Crypto context + * @cipher_type: Encryption cipher type + * @aes_keylen: AES key length + * @iv_source: Encryption IV source + * @hash_type: Authentication type + * @auth_input_type: Authentication input type + * 1 - Authentication IV and KEY, microcode calculates OPAD/IPAD + * 0 - Authentication OPAD/IPAD + * @mac_len: mac length + * @crypto: Crypto keys + * @auth: Authentication keys + */ +struct flexi_crypto_context { + union fc_ctx_flags flags; + struct crypto_keys crypto; + struct auth_keys auth; +}; + +struct crypto_ctx_hdr { + struct dma_pool *pool; + dma_addr_t dma; + void *vaddr; +}; + +struct nitrox_crypto_ctx { + struct nitrox_device *ndev; + union { + u64 ctx_handle; + struct flexi_crypto_context *fctx; + } u; + struct crypto_ctx_hdr *chdr; + sereq_completion_t callback; +}; + +struct nitrox_kcrypt_request { + struct se_crypto_request creq; + u8 *src; + u8 *dst; + u8 *iv_out; +}; + +/** + * struct nitrox_aead_rctx - AEAD request context + * @nkreq: Base request context + * @cryptlen: Encryption/Decryption data length + * @assoclen: AAD length + * @srclen: Input buffer length + * @dstlen: Output buffer length + * @iv: IV data + * @ivsize: IV data length + * @flags: AEAD req flags + * @ctx_handle: Device context handle + * @src: Source sglist + * @dst: Destination sglist + * @ctrl_arg: Identifies the request type (ENCRYPT/DECRYPT) + */ +struct nitrox_aead_rctx { + struct nitrox_kcrypt_request nkreq; + unsigned int cryptlen; + unsigned int assoclen; + unsigned int srclen; + unsigned int dstlen; + u8 *iv; + int ivsize; + u32 flags; + u64 ctx_handle; + struct scatterlist *src; + struct scatterlist *dst; + u8 ctrl_arg; +}; + +/** + * struct nitrox_rfc4106_rctx - rfc4106 cipher request context + * @base: AEAD request context + * @src: Source sglist + * @dst: Destination sglist + * @assoc: AAD + */ +struct nitrox_rfc4106_rctx { + struct nitrox_aead_rctx base; + struct scatterlist src[3]; + struct scatterlist dst[3]; + u8 assoc[20]; +}; + +/** + * struct pkt_instr_hdr - Packet Instruction Header + * @g: Gather used + * When [G] is set and [GSZ] != 0, the instruction is + * indirect gather instruction. + * When [G] is set and [GSZ] = 0, the instruction is + * direct gather instruction. + * @gsz: Number of pointers in the indirect gather list + * @ihi: When set hardware duplicates the 1st 8 bytes of pkt_instr_hdr + * and adds them to the packet after the pkt_instr_hdr but before any UDD + * @ssz: Not used by the input hardware. But can become slc_store_int[SSZ] + * when [IHI] is set. + * @fsz: The number of front data bytes directly included in the + * PCIe instruction. + * @tlen: The length of the input packet in bytes, include: + * - 16B pkt_hdr + * - Inline context bytes if any, + * - UDD if any, + * - packet payload bytes + */ +union pkt_instr_hdr { + __be64 bev; + u64 value; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 raz_48_63 : 16; + u64 g : 1; + u64 gsz : 7; + u64 ihi : 1; + u64 ssz : 7; + u64 raz_30_31 : 2; + u64 fsz : 6; + u64 raz_16_23 : 8; + u64 tlen : 16; +#else + u64 tlen : 16; + u64 raz_16_23 : 8; + u64 fsz : 6; + u64 raz_30_31 : 2; + u64 ssz : 7; + u64 ihi : 1; + u64 gsz : 7; + u64 g : 1; + u64 raz_48_63 : 16; +#endif + } s; +}; + +/** + * struct pkt_hdr - Packet Input Header + * @opcode: Request opcode (Major) + * @arg: Request opcode (Minor) + * @ctxc: Context control. + * @unca: When set [UNC] is the uncertainty count for an input packet. + * The hardware uses uncertainty counts to predict + * output buffer use and avoid deadlock. + * @info: Not used by input hardware. Available for use + * during SE processing. + * @destport: The expected destination port/ring/channel for the packet. + * @unc: Uncertainty count for an input packet. + * @grp: SE group that will process the input packet. + * @ctxl: Context Length in 64-bit words. + * @uddl: User-defined data (UDD) length in bytes. + * @ctxp: Context pointer. CTXP<63,2:0> must be zero in all cases. + */ +union pkt_hdr { + __be64 bev[2]; + u64 value[2]; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 opcode : 8; + u64 arg : 8; + u64 ctxc : 2; + u64 unca : 1; + u64 raz_44 : 1; + u64 info : 3; + u64 destport : 9; + u64 unc : 8; + u64 raz_19_23 : 5; + u64 grp : 3; + u64 raz_15 : 1; + u64 ctxl : 7; + u64 uddl : 8; +#else + u64 uddl : 8; + u64 ctxl : 7; + u64 raz_15 : 1; + u64 grp : 3; + u64 raz_19_23 : 5; + u64 unc : 8; + u64 destport : 9; + u64 info : 3; + u64 raz_44 : 1; + u64 unca : 1; + u64 ctxc : 2; + u64 arg : 8; + u64 opcode : 8; +#endif + __be64 ctxp; + } s; +}; + +/** + * struct slc_store_info - Solicited Paceket Output Store Information. + * @ssz: The number of scatterlist pointers for the solicited output port + * packet. + * @rptr: The result pointer for the solicited output port packet. + * If [SSZ]=0, [RPTR] must point directly to a buffer on the remote + * host that is large enough to hold the entire output packet. + * If [SSZ]!=0, [RPTR] must point to an array of ([SSZ]+3)/4 + * sglist components at [RPTR] on the remote host. + */ +union slc_store_info { + __be64 bev[2]; + u64 value[2]; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 raz_39_63 : 25; + u64 ssz : 7; + u64 raz_0_31 : 32; +#else + u64 raz_0_31 : 32; + u64 ssz : 7; + u64 raz_39_63 : 25; +#endif + __be64 rptr; + } s; +}; + +/** + * struct nps_pkt_instr - NPS Packet Instruction of SE cores. + * @dptr0 : Input pointer points to buffer in remote host. + * @ih: Packet Instruction Header (8 bytes) + * @irh: Packet Input Header (16 bytes) + * @slc: Solicited Packet Output Store Information (16 bytes) + * @fdata: Front data + * + * 64-Byte Instruction Format + */ +struct nps_pkt_instr { + __be64 dptr0; + union pkt_instr_hdr ih; + union pkt_hdr irh; + union slc_store_info slc; + u64 fdata[2]; +}; + +/** + * struct aqmq_command_s - The 32 byte command for AE processing. + * @opcode: Request opcode + * @param1: Request control parameter 1 + * @param2: Request control parameter 2 + * @dlen: Input length + * @dptr: Input pointer points to buffer in remote host + * @rptr: Result pointer points to buffer in remote host + * @grp: AQM Group (0..7) + * @cptr: Context pointer + */ +struct aqmq_command_s { + __be16 opcode; + __be16 param1; + __be16 param2; + __be16 dlen; + __be64 dptr; + __be64 rptr; + union { + __be64 word3; +#if defined(__BIG_ENDIAN_BITFIELD) + u64 grp : 3; + u64 cptr : 61; +#else + u64 cptr : 61; + u64 grp : 3; +#endif + }; +}; + +/** + * struct ctx_hdr - Book keeping data about the crypto context + * @pool: Pool used to allocate crypto context + * @dma: Base DMA address of the crypto context + * @ctx_dma: Actual usable crypto context for NITROX + */ +struct ctx_hdr { + struct dma_pool *pool; + dma_addr_t dma; + dma_addr_t ctx_dma; +}; + +/* + * struct sglist_component - SG list component format + * @len0: The number of bytes at [PTR0] on the remote host. + * @len1: The number of bytes at [PTR1] on the remote host. + * @len2: The number of bytes at [PTR2] on the remote host. + * @len3: The number of bytes at [PTR3] on the remote host. + * @dma0: First pointer point to buffer in remote host. + * @dma1: Second pointer point to buffer in remote host. + * @dma2: Third pointer point to buffer in remote host. + * @dma3: Fourth pointer point to buffer in remote host. + */ +struct nitrox_sgcomp { + __be16 len[4]; + __be64 dma[4]; +}; + +/* + * strutct nitrox_sgtable - SG list information + * @sgmap_cnt: Number of buffers mapped + * @total_bytes: Total bytes in sglist. + * @sgcomp_len: Total sglist components length. + * @sgcomp_dma: DMA address of sglist component. + * @sg: crypto request buffer. + * @sgcomp: sglist component for NITROX. + */ +struct nitrox_sgtable { + u8 sgmap_cnt; + u16 total_bytes; + u32 sgcomp_len; + dma_addr_t sgcomp_dma; + struct scatterlist *sg; + struct nitrox_sgcomp *sgcomp; +}; + +/* Response Header Length */ +#define ORH_HLEN 8 +/* Completion bytes Length */ +#define COMP_HLEN 8 + +struct resp_hdr { + u64 *orh; + u64 *completion; +}; + +typedef void (*completion_t)(void *arg, int err); + +/** + * struct nitrox_softreq - Represents the NIROX Request. + * @response: response list entry + * @backlog: Backlog list entry + * @ndev: Device used to submit the request + * @cmdq: Command queue for submission + * @resp: Response headers + * @instr: 64B instruction + * @in: SG table for input + * @out SG table for output + * @tstamp: Request submitted time in jiffies + * @callback: callback after request completion/timeout + * @cb_arg: callback argument + */ +struct nitrox_softreq { + struct list_head response; + struct list_head backlog; + + u32 flags; + gfp_t gfp; + atomic_t status; + + struct nitrox_device *ndev; + struct nitrox_cmdq *cmdq; + + struct nps_pkt_instr instr; + struct resp_hdr resp; + struct nitrox_sgtable in; + struct nitrox_sgtable out; + + unsigned long tstamp; + + completion_t callback; + void *cb_arg; +}; + +static inline int flexi_aes_keylen(int keylen) +{ + int aes_keylen; + + switch (keylen) { + case AES_KEYSIZE_128: + aes_keylen = 1; + break; + case AES_KEYSIZE_192: + aes_keylen = 2; + break; + case AES_KEYSIZE_256: + aes_keylen = 3; + break; + default: + aes_keylen = -EINVAL; + break; + } + return aes_keylen; +} + +static inline void *alloc_req_buf(int nents, int extralen, gfp_t gfp) +{ + size_t size; + + size = sizeof(struct scatterlist) * nents; + size += extralen; + + return kzalloc(size, gfp); +} + +/** + * create_single_sg - Point SG entry to the data + * @sg: Destination SG list + * @buf: Data + * @buflen: Data length + * + * Returns next free entry in the destination SG list + **/ +static inline struct scatterlist *create_single_sg(struct scatterlist *sg, + void *buf, int buflen) +{ + sg_set_buf(sg, buf, buflen); + sg++; + return sg; +} + +/** + * create_multi_sg - Create multiple sg entries with buflen data length from + * source sglist + * @to_sg: Destination SG list + * @from_sg: Source SG list + * @buflen: Data length + * + * Returns next free entry in the destination SG list + **/ +static inline struct scatterlist *create_multi_sg(struct scatterlist *to_sg, + struct scatterlist *from_sg, + int buflen) +{ + struct scatterlist *sg = to_sg; + unsigned int sglen; + + for (; buflen && from_sg; buflen -= sglen) { + sglen = from_sg->length; + if (sglen > buflen) + sglen = buflen; + + sg_set_buf(sg, sg_virt(from_sg), sglen); + from_sg = sg_next(from_sg); + sg++; + } + + return sg; +} + +static inline void set_orh_value(u64 *orh) +{ + WRITE_ONCE(*orh, PENDING_SIG); +} + +static inline void set_comp_value(u64 *comp) +{ + WRITE_ONCE(*comp, PENDING_SIG); +} + +static inline int alloc_src_req_buf(struct nitrox_kcrypt_request *nkreq, + int nents, int ivsize) +{ + struct se_crypto_request *creq = &nkreq->creq; + + nkreq->src = alloc_req_buf(nents, ivsize, creq->gfp); + if (!nkreq->src) + return -ENOMEM; + + return 0; +} + +static inline void nitrox_creq_copy_iv(char *dst, char *src, int size) +{ + memcpy(dst, src, size); +} + +static inline struct scatterlist *nitrox_creq_src_sg(char *iv, int ivsize) +{ + return (struct scatterlist *)(iv + ivsize); +} + +static inline void nitrox_creq_set_src_sg(struct nitrox_kcrypt_request *nkreq, + int nents, int ivsize, + struct scatterlist *src, int buflen) +{ + char *iv = nkreq->src; + struct scatterlist *sg; + struct se_crypto_request *creq = &nkreq->creq; + + creq->src = nitrox_creq_src_sg(iv, ivsize); + sg = creq->src; + sg_init_table(sg, nents); + + /* Input format: + * +----+----------------+ + * | IV | SRC sg entries | + * +----+----------------+ + */ + + /* IV */ + sg = create_single_sg(sg, iv, ivsize); + /* SRC entries */ + create_multi_sg(sg, src, buflen); +} + +static inline int alloc_dst_req_buf(struct nitrox_kcrypt_request *nkreq, + int nents) +{ + int extralen = ORH_HLEN + COMP_HLEN; + struct se_crypto_request *creq = &nkreq->creq; + + nkreq->dst = alloc_req_buf(nents, extralen, creq->gfp); + if (!nkreq->dst) + return -ENOMEM; + + return 0; +} + +static inline void nitrox_creq_set_orh(struct nitrox_kcrypt_request *nkreq) +{ + struct se_crypto_request *creq = &nkreq->creq; + + creq->orh = (u64 *)(nkreq->dst); + set_orh_value(creq->orh); +} + +static inline void nitrox_creq_set_comp(struct nitrox_kcrypt_request *nkreq) +{ + struct se_crypto_request *creq = &nkreq->creq; + + creq->comp = (u64 *)(nkreq->dst + ORH_HLEN); + set_comp_value(creq->comp); +} + +static inline struct scatterlist *nitrox_creq_dst_sg(char *dst) +{ + return (struct scatterlist *)(dst + ORH_HLEN + COMP_HLEN); +} + +static inline void nitrox_creq_set_dst_sg(struct nitrox_kcrypt_request *nkreq, + int nents, int ivsize, + struct scatterlist *dst, int buflen) +{ + struct se_crypto_request *creq = &nkreq->creq; + struct scatterlist *sg; + char *iv = nkreq->src; + + creq->dst = nitrox_creq_dst_sg(nkreq->dst); + sg = creq->dst; + sg_init_table(sg, nents); + + /* Output format: + * +-----+----+----------------+-----------------+ + * | ORH | IV | DST sg entries | COMPLETION Bytes| + * +-----+----+----------------+-----------------+ + */ + + /* ORH */ + sg = create_single_sg(sg, creq->orh, ORH_HLEN); + /* IV */ + sg = create_single_sg(sg, iv, ivsize); + /* DST entries */ + sg = create_multi_sg(sg, dst, buflen); + /* COMPLETION Bytes */ + create_single_sg(sg, creq->comp, COMP_HLEN); +} + +#endif /* __NITROX_REQ_H */ diff --git a/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c b/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c new file mode 100644 index 000000000..55c18da4a --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c @@ -0,0 +1,607 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/gfp.h> +#include <linux/workqueue.h> +#include <crypto/internal/skcipher.h> + +#include "nitrox_common.h" +#include "nitrox_dev.h" +#include "nitrox_req.h" +#include "nitrox_csr.h" + +/* SLC_STORE_INFO */ +#define MIN_UDD_LEN 16 +/* PKT_IN_HDR + SLC_STORE_INFO */ +#define FDATA_SIZE 32 +/* Base destination port for the solicited requests */ +#define SOLICIT_BASE_DPORT 256 + +#define REQ_NOT_POSTED 1 +#define REQ_BACKLOG 2 +#define REQ_POSTED 3 + +/* + * Response codes from SE microcode + * 0x00 - Success + * Completion with no error + * 0x43 - ERR_GC_DATA_LEN_INVALID + * Invalid Data length if Encryption Data length is + * less than 16 bytes for AES-XTS and AES-CTS. + * 0x45 - ERR_GC_CTX_LEN_INVALID + * Invalid context length: CTXL != 23 words. + * 0x4F - ERR_GC_DOCSIS_CIPHER_INVALID + * DOCSIS support is enabled with other than + * AES/DES-CBC mode encryption. + * 0x50 - ERR_GC_DOCSIS_OFFSET_INVALID + * Authentication offset is other than 0 with + * Encryption IV source = 0. + * Authentication offset is other than 8 (DES)/16 (AES) + * with Encryption IV source = 1 + * 0x51 - ERR_GC_CRC32_INVALID_SELECTION + * CRC32 is enabled for other than DOCSIS encryption. + * 0x52 - ERR_GC_AES_CCM_FLAG_INVALID + * Invalid flag options in AES-CCM IV. + */ + +static inline int incr_index(int index, int count, int max) +{ + if ((index + count) >= max) + index = index + count - max; + else + index += count; + + return index; +} + +static void softreq_unmap_sgbufs(struct nitrox_softreq *sr) +{ + struct nitrox_device *ndev = sr->ndev; + struct device *dev = DEV(ndev); + + + dma_unmap_sg(dev, sr->in.sg, sg_nents(sr->in.sg), + DMA_BIDIRECTIONAL); + dma_unmap_single(dev, sr->in.sgcomp_dma, sr->in.sgcomp_len, + DMA_TO_DEVICE); + kfree(sr->in.sgcomp); + sr->in.sg = NULL; + sr->in.sgmap_cnt = 0; + + dma_unmap_sg(dev, sr->out.sg, sg_nents(sr->out.sg), + DMA_BIDIRECTIONAL); + dma_unmap_single(dev, sr->out.sgcomp_dma, sr->out.sgcomp_len, + DMA_TO_DEVICE); + kfree(sr->out.sgcomp); + sr->out.sg = NULL; + sr->out.sgmap_cnt = 0; +} + +static void softreq_destroy(struct nitrox_softreq *sr) +{ + softreq_unmap_sgbufs(sr); + kfree(sr); +} + +/** + * create_sg_component - create SG componets for N5 device. + * @sr: Request structure + * @sgtbl: SG table + * @map_nents: number of dma mapped entries + * + * Component structure + * + * 63 48 47 32 31 16 15 0 + * -------------------------------------- + * | LEN0 | LEN1 | LEN2 | LEN3 | + * |------------------------------------- + * | PTR0 | + * -------------------------------------- + * | PTR1 | + * -------------------------------------- + * | PTR2 | + * -------------------------------------- + * | PTR3 | + * -------------------------------------- + * + * Returns 0 if success or a negative errno code on error. + */ +static int create_sg_component(struct nitrox_softreq *sr, + struct nitrox_sgtable *sgtbl, int map_nents) +{ + struct nitrox_device *ndev = sr->ndev; + struct nitrox_sgcomp *sgcomp; + struct scatterlist *sg; + dma_addr_t dma; + size_t sz_comp; + int i, j, nr_sgcomp; + + nr_sgcomp = roundup(map_nents, 4) / 4; + + /* each component holds 4 dma pointers */ + sz_comp = nr_sgcomp * sizeof(*sgcomp); + sgcomp = kzalloc(sz_comp, sr->gfp); + if (!sgcomp) + return -ENOMEM; + + sgtbl->sgcomp = sgcomp; + + sg = sgtbl->sg; + /* populate device sg component */ + for (i = 0; i < nr_sgcomp; i++) { + for (j = 0; j < 4 && sg; j++) { + sgcomp[i].len[j] = cpu_to_be16(sg_dma_len(sg)); + sgcomp[i].dma[j] = cpu_to_be64(sg_dma_address(sg)); + sg = sg_next(sg); + } + } + /* map the device sg component */ + dma = dma_map_single(DEV(ndev), sgtbl->sgcomp, sz_comp, DMA_TO_DEVICE); + if (dma_mapping_error(DEV(ndev), dma)) { + kfree(sgtbl->sgcomp); + sgtbl->sgcomp = NULL; + return -ENOMEM; + } + + sgtbl->sgcomp_dma = dma; + sgtbl->sgcomp_len = sz_comp; + + return 0; +} + +/** + * dma_map_inbufs - DMA map input sglist and creates sglist component + * for N5 device. + * @sr: Request structure + * @req: Crypto request structre + * + * Returns 0 if successful or a negative errno code on error. + */ +static int dma_map_inbufs(struct nitrox_softreq *sr, + struct se_crypto_request *req) +{ + struct device *dev = DEV(sr->ndev); + struct scatterlist *sg; + int i, nents, ret = 0; + + nents = dma_map_sg(dev, req->src, sg_nents(req->src), + DMA_BIDIRECTIONAL); + if (!nents) + return -EINVAL; + + for_each_sg(req->src, sg, nents, i) + sr->in.total_bytes += sg_dma_len(sg); + + sr->in.sg = req->src; + sr->in.sgmap_cnt = nents; + ret = create_sg_component(sr, &sr->in, sr->in.sgmap_cnt); + if (ret) + goto incomp_err; + + return 0; + +incomp_err: + dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_BIDIRECTIONAL); + sr->in.sgmap_cnt = 0; + return ret; +} + +static int dma_map_outbufs(struct nitrox_softreq *sr, + struct se_crypto_request *req) +{ + struct device *dev = DEV(sr->ndev); + int nents, ret = 0; + + nents = dma_map_sg(dev, req->dst, sg_nents(req->dst), + DMA_BIDIRECTIONAL); + if (!nents) + return -EINVAL; + + sr->out.sg = req->dst; + sr->out.sgmap_cnt = nents; + ret = create_sg_component(sr, &sr->out, sr->out.sgmap_cnt); + if (ret) + goto outcomp_map_err; + + return 0; + +outcomp_map_err: + dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_BIDIRECTIONAL); + sr->out.sgmap_cnt = 0; + sr->out.sg = NULL; + return ret; +} + +static inline int softreq_map_iobuf(struct nitrox_softreq *sr, + struct se_crypto_request *creq) +{ + int ret; + + ret = dma_map_inbufs(sr, creq); + if (ret) + return ret; + + ret = dma_map_outbufs(sr, creq); + if (ret) + softreq_unmap_sgbufs(sr); + + return ret; +} + +static inline void backlog_list_add(struct nitrox_softreq *sr, + struct nitrox_cmdq *cmdq) +{ + INIT_LIST_HEAD(&sr->backlog); + + spin_lock_bh(&cmdq->backlog_qlock); + list_add_tail(&sr->backlog, &cmdq->backlog_head); + atomic_inc(&cmdq->backlog_count); + atomic_set(&sr->status, REQ_BACKLOG); + spin_unlock_bh(&cmdq->backlog_qlock); +} + +static inline void response_list_add(struct nitrox_softreq *sr, + struct nitrox_cmdq *cmdq) +{ + INIT_LIST_HEAD(&sr->response); + + spin_lock_bh(&cmdq->resp_qlock); + list_add_tail(&sr->response, &cmdq->response_head); + spin_unlock_bh(&cmdq->resp_qlock); +} + +static inline void response_list_del(struct nitrox_softreq *sr, + struct nitrox_cmdq *cmdq) +{ + spin_lock_bh(&cmdq->resp_qlock); + list_del(&sr->response); + spin_unlock_bh(&cmdq->resp_qlock); +} + +static struct nitrox_softreq * +get_first_response_entry(struct nitrox_cmdq *cmdq) +{ + return list_first_entry_or_null(&cmdq->response_head, + struct nitrox_softreq, response); +} + +static inline bool cmdq_full(struct nitrox_cmdq *cmdq, int qlen) +{ + if (atomic_inc_return(&cmdq->pending_count) > qlen) { + atomic_dec(&cmdq->pending_count); + /* sync with other cpus */ + smp_mb__after_atomic(); + return true; + } + /* sync with other cpus */ + smp_mb__after_atomic(); + return false; +} + +/** + * post_se_instr - Post SE instruction to Packet Input ring + * @sr: Request structure + * @cmdq: Command queue structure + * + * Returns 0 if successful or a negative error code, + * if no space in ring. + */ +static void post_se_instr(struct nitrox_softreq *sr, + struct nitrox_cmdq *cmdq) +{ + struct nitrox_device *ndev = sr->ndev; + int idx; + u8 *ent; + + spin_lock_bh(&cmdq->cmd_qlock); + + idx = cmdq->write_idx; + /* copy the instruction */ + ent = cmdq->base + (idx * cmdq->instr_size); + memcpy(ent, &sr->instr, cmdq->instr_size); + + atomic_set(&sr->status, REQ_POSTED); + response_list_add(sr, cmdq); + sr->tstamp = jiffies; + /* flush the command queue updates */ + dma_wmb(); + + /* Ring doorbell with count 1 */ + writeq(1, cmdq->dbell_csr_addr); + + cmdq->write_idx = incr_index(idx, 1, ndev->qlen); + + spin_unlock_bh(&cmdq->cmd_qlock); + + /* increment the posted command count */ + atomic64_inc(&ndev->stats.posted); +} + +static int post_backlog_cmds(struct nitrox_cmdq *cmdq) +{ + struct nitrox_device *ndev = cmdq->ndev; + struct nitrox_softreq *sr, *tmp; + int ret = 0; + + if (!atomic_read(&cmdq->backlog_count)) + return 0; + + spin_lock_bh(&cmdq->backlog_qlock); + + list_for_each_entry_safe(sr, tmp, &cmdq->backlog_head, backlog) { + /* submit until space available */ + if (unlikely(cmdq_full(cmdq, ndev->qlen))) { + ret = -ENOSPC; + break; + } + /* delete from backlog list */ + list_del(&sr->backlog); + atomic_dec(&cmdq->backlog_count); + /* sync with other cpus */ + smp_mb__after_atomic(); + + /* post the command */ + post_se_instr(sr, cmdq); + } + spin_unlock_bh(&cmdq->backlog_qlock); + + return ret; +} + +static int nitrox_enqueue_request(struct nitrox_softreq *sr) +{ + struct nitrox_cmdq *cmdq = sr->cmdq; + struct nitrox_device *ndev = sr->ndev; + + /* try to post backlog requests */ + post_backlog_cmds(cmdq); + + if (unlikely(cmdq_full(cmdq, ndev->qlen))) { + if (!(sr->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { + /* increment drop count */ + atomic64_inc(&ndev->stats.dropped); + return -ENOSPC; + } + /* add to backlog list */ + backlog_list_add(sr, cmdq); + return -EINPROGRESS; + } + post_se_instr(sr, cmdq); + + return -EINPROGRESS; +} + +/** + * nitrox_process_se_request - Send request to SE core + * @ndev: NITROX device + * @req: Crypto request + * @callback: Completion callback + * @cb_arg: Completion callback arguments + * + * Returns 0 on success, or a negative error code. + */ +int nitrox_process_se_request(struct nitrox_device *ndev, + struct se_crypto_request *req, + completion_t callback, + void *cb_arg) +{ + struct nitrox_softreq *sr; + dma_addr_t ctx_handle = 0; + int qno, ret = 0; + + if (!nitrox_ready(ndev)) + return -ENODEV; + + sr = kzalloc(sizeof(*sr), req->gfp); + if (!sr) + return -ENOMEM; + + sr->ndev = ndev; + sr->flags = req->flags; + sr->gfp = req->gfp; + sr->callback = callback; + sr->cb_arg = cb_arg; + + atomic_set(&sr->status, REQ_NOT_POSTED); + + sr->resp.orh = req->orh; + sr->resp.completion = req->comp; + + ret = softreq_map_iobuf(sr, req); + if (ret) { + kfree(sr); + return ret; + } + + /* get the context handle */ + if (req->ctx_handle) { + struct ctx_hdr *hdr; + u8 *ctx_ptr; + + ctx_ptr = (u8 *)(uintptr_t)req->ctx_handle; + hdr = (struct ctx_hdr *)(ctx_ptr - sizeof(struct ctx_hdr)); + ctx_handle = hdr->ctx_dma; + } + + /* select the queue */ + qno = smp_processor_id() % ndev->nr_queues; + + sr->cmdq = &ndev->pkt_inq[qno]; + + /* + * 64-Byte Instruction Format + * + * ---------------------- + * | DPTR0 | 8 bytes + * ---------------------- + * | PKT_IN_INSTR_HDR | 8 bytes + * ---------------------- + * | PKT_IN_HDR | 16 bytes + * ---------------------- + * | SLC_INFO | 16 bytes + * ---------------------- + * | Front data | 16 bytes + * ---------------------- + */ + + /* fill the packet instruction */ + /* word 0 */ + sr->instr.dptr0 = cpu_to_be64(sr->in.sgcomp_dma); + + /* word 1 */ + sr->instr.ih.value = 0; + sr->instr.ih.s.g = 1; + sr->instr.ih.s.gsz = sr->in.sgmap_cnt; + sr->instr.ih.s.ssz = sr->out.sgmap_cnt; + sr->instr.ih.s.fsz = FDATA_SIZE + sizeof(struct gphdr); + sr->instr.ih.s.tlen = sr->instr.ih.s.fsz + sr->in.total_bytes; + sr->instr.ih.bev = cpu_to_be64(sr->instr.ih.value); + + /* word 2 */ + sr->instr.irh.value[0] = 0; + sr->instr.irh.s.uddl = MIN_UDD_LEN; + /* context length in 64-bit words */ + sr->instr.irh.s.ctxl = (req->ctrl.s.ctxl / 8); + /* offset from solicit base port 256 */ + sr->instr.irh.s.destport = SOLICIT_BASE_DPORT + qno; + sr->instr.irh.s.ctxc = req->ctrl.s.ctxc; + sr->instr.irh.s.arg = req->ctrl.s.arg; + sr->instr.irh.s.opcode = req->opcode; + sr->instr.irh.bev[0] = cpu_to_be64(sr->instr.irh.value[0]); + + /* word 3 */ + sr->instr.irh.s.ctxp = cpu_to_be64(ctx_handle); + + /* word 4 */ + sr->instr.slc.value[0] = 0; + sr->instr.slc.s.ssz = sr->out.sgmap_cnt; + sr->instr.slc.bev[0] = cpu_to_be64(sr->instr.slc.value[0]); + + /* word 5 */ + sr->instr.slc.s.rptr = cpu_to_be64(sr->out.sgcomp_dma); + + /* + * No conversion for front data, + * It goes into payload + * put GP Header in front data + */ + sr->instr.fdata[0] = *((u64 *)&req->gph); + sr->instr.fdata[1] = 0; + + ret = nitrox_enqueue_request(sr); + if (ret == -ENOSPC) + goto send_fail; + + return ret; + +send_fail: + softreq_destroy(sr); + return ret; +} + +static inline int cmd_timeout(unsigned long tstamp, unsigned long timeout) +{ + return time_after_eq(jiffies, (tstamp + timeout)); +} + +void backlog_qflush_work(struct work_struct *work) +{ + struct nitrox_cmdq *cmdq; + + cmdq = container_of(work, struct nitrox_cmdq, backlog_qflush); + post_backlog_cmds(cmdq); +} + +static bool sr_completed(struct nitrox_softreq *sr) +{ + u64 orh = READ_ONCE(*sr->resp.orh); + unsigned long timeout = jiffies + msecs_to_jiffies(1); + + if ((orh != PENDING_SIG) && (orh & 0xff)) + return true; + + while (READ_ONCE(*sr->resp.completion) == PENDING_SIG) { + if (time_after(jiffies, timeout)) { + pr_err("comp not done\n"); + return false; + } + } + + return true; +} + +/** + * process_response_list - process completed requests + * @cmdq: Command queue structure + * + * Returns the number of responses processed. + */ +static void process_response_list(struct nitrox_cmdq *cmdq) +{ + struct nitrox_device *ndev = cmdq->ndev; + struct nitrox_softreq *sr; + int req_completed = 0, err = 0, budget; + completion_t callback; + void *cb_arg; + + /* check all pending requests */ + budget = atomic_read(&cmdq->pending_count); + + while (req_completed < budget) { + sr = get_first_response_entry(cmdq); + if (!sr) + break; + + if (atomic_read(&sr->status) != REQ_POSTED) + break; + + /* check orh and completion bytes updates */ + if (!sr_completed(sr)) { + /* request not completed, check for timeout */ + if (!cmd_timeout(sr->tstamp, ndev->timeout)) + break; + dev_err_ratelimited(DEV(ndev), + "Request timeout, orh 0x%016llx\n", + READ_ONCE(*sr->resp.orh)); + } + atomic_dec(&cmdq->pending_count); + atomic64_inc(&ndev->stats.completed); + /* sync with other cpus */ + smp_mb__after_atomic(); + /* remove from response list */ + response_list_del(sr, cmdq); + /* ORH error code */ + err = READ_ONCE(*sr->resp.orh) & 0xff; + callback = sr->callback; + cb_arg = sr->cb_arg; + softreq_destroy(sr); + if (callback) + callback(cb_arg, err); + + req_completed++; + } +} + +/* + * pkt_slc_resp_tasklet - post processing of SE responses + */ +void pkt_slc_resp_tasklet(unsigned long data) +{ + struct nitrox_q_vector *qvec = (void *)(uintptr_t)(data); + struct nitrox_cmdq *cmdq = qvec->cmdq; + union nps_pkt_slc_cnts slc_cnts; + + /* read completion count */ + slc_cnts.value = readq(cmdq->compl_cnt_csr_addr); + /* resend the interrupt if more work to do */ + slc_cnts.s.resend = 1; + + process_response_list(cmdq); + + /* + * clear the interrupt with resend bit enabled, + * MSI-X interrupt generates if Completion count > Threshold + */ + writeq(slc_cnts.value, cmdq->compl_cnt_csr_addr); + + if (atomic_read(&cmdq->backlog_count)) + schedule_work(&cmdq->backlog_qflush); +} diff --git a/drivers/crypto/cavium/nitrox/nitrox_skcipher.c b/drivers/crypto/cavium/nitrox/nitrox_skcipher.c new file mode 100644 index 000000000..248b4fff1 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_skcipher.c @@ -0,0 +1,552 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/crypto.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/printk.h> + +#include <crypto/aes.h> +#include <crypto/skcipher.h> +#include <crypto/scatterwalk.h> +#include <crypto/ctr.h> +#include <crypto/internal/des.h> +#include <crypto/xts.h> + +#include "nitrox_dev.h" +#include "nitrox_common.h" +#include "nitrox_req.h" + +struct nitrox_cipher { + const char *name; + enum flexi_cipher value; +}; + +/* + * supported cipher list + */ +static const struct nitrox_cipher flexi_cipher_table[] = { + { "null", CIPHER_NULL }, + { "cbc(des3_ede)", CIPHER_3DES_CBC }, + { "ecb(des3_ede)", CIPHER_3DES_ECB }, + { "cbc(aes)", CIPHER_AES_CBC }, + { "ecb(aes)", CIPHER_AES_ECB }, + { "cfb(aes)", CIPHER_AES_CFB }, + { "rfc3686(ctr(aes))", CIPHER_AES_CTR }, + { "xts(aes)", CIPHER_AES_XTS }, + { "cts(cbc(aes))", CIPHER_AES_CBC_CTS }, + { NULL, CIPHER_INVALID } +}; + +static enum flexi_cipher flexi_cipher_type(const char *name) +{ + const struct nitrox_cipher *cipher = flexi_cipher_table; + + while (cipher->name) { + if (!strcmp(cipher->name, name)) + break; + cipher++; + } + return cipher->value; +} + +static void free_src_sglist(struct skcipher_request *skreq) +{ + struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq); + + kfree(nkreq->src); +} + +static void free_dst_sglist(struct skcipher_request *skreq) +{ + struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq); + + kfree(nkreq->dst); +} + +static void nitrox_skcipher_callback(void *arg, int err) +{ + struct skcipher_request *skreq = arg; + + free_src_sglist(skreq); + free_dst_sglist(skreq); + if (err) { + pr_err_ratelimited("request failed status 0x%0x\n", err); + err = -EINVAL; + } + + skcipher_request_complete(skreq, err); +} + +static void nitrox_cbc_cipher_callback(void *arg, int err) +{ + struct skcipher_request *skreq = arg; + struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq); + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(skreq); + int ivsize = crypto_skcipher_ivsize(cipher); + unsigned int start = skreq->cryptlen - ivsize; + + if (err) { + nitrox_skcipher_callback(arg, err); + return; + } + + if (nkreq->creq.ctrl.s.arg == ENCRYPT) { + scatterwalk_map_and_copy(skreq->iv, skreq->dst, start, ivsize, + 0); + } else { + if (skreq->src != skreq->dst) { + scatterwalk_map_and_copy(skreq->iv, skreq->src, start, + ivsize, 0); + } else { + memcpy(skreq->iv, nkreq->iv_out, ivsize); + kfree(nkreq->iv_out); + } + } + + nitrox_skcipher_callback(arg, err); +} + +static int nitrox_skcipher_init(struct crypto_skcipher *tfm) +{ + struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(tfm); + struct crypto_ctx_hdr *chdr; + + /* get the first device */ + nctx->ndev = nitrox_get_first_device(); + if (!nctx->ndev) + return -ENODEV; + + /* allocate nitrox crypto context */ + chdr = crypto_alloc_context(nctx->ndev); + if (!chdr) { + nitrox_put_device(nctx->ndev); + return -ENOMEM; + } + + nctx->callback = nitrox_skcipher_callback; + nctx->chdr = chdr; + nctx->u.ctx_handle = (uintptr_t)((u8 *)chdr->vaddr + + sizeof(struct ctx_hdr)); + crypto_skcipher_set_reqsize(tfm, crypto_skcipher_reqsize(tfm) + + sizeof(struct nitrox_kcrypt_request)); + return 0; +} + +static int nitrox_cbc_init(struct crypto_skcipher *tfm) +{ + int err; + struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(tfm); + + err = nitrox_skcipher_init(tfm); + if (err) + return err; + + nctx->callback = nitrox_cbc_cipher_callback; + return 0; +} + +static void nitrox_skcipher_exit(struct crypto_skcipher *tfm) +{ + struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(tfm); + + /* free the nitrox crypto context */ + if (nctx->u.ctx_handle) { + struct flexi_crypto_context *fctx = nctx->u.fctx; + + memzero_explicit(&fctx->crypto, sizeof(struct crypto_keys)); + memzero_explicit(&fctx->auth, sizeof(struct auth_keys)); + crypto_free_context((void *)nctx->chdr); + } + nitrox_put_device(nctx->ndev); + + nctx->u.ctx_handle = 0; + nctx->ndev = NULL; +} + +static inline int nitrox_skcipher_setkey(struct crypto_skcipher *cipher, + int aes_keylen, const u8 *key, + unsigned int keylen) +{ + struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); + struct nitrox_crypto_ctx *nctx = crypto_tfm_ctx(tfm); + struct flexi_crypto_context *fctx; + union fc_ctx_flags *flags; + enum flexi_cipher cipher_type; + const char *name; + + name = crypto_tfm_alg_name(tfm); + cipher_type = flexi_cipher_type(name); + if (unlikely(cipher_type == CIPHER_INVALID)) { + pr_err("unsupported cipher: %s\n", name); + return -EINVAL; + } + + /* fill crypto context */ + fctx = nctx->u.fctx; + flags = &fctx->flags; + flags->f = 0; + flags->w0.cipher_type = cipher_type; + flags->w0.aes_keylen = aes_keylen; + flags->w0.iv_source = IV_FROM_DPTR; + flags->f = cpu_to_be64(*(u64 *)&flags->w0); + /* copy the key to context */ + memcpy(fctx->crypto.u.key, key, keylen); + + return 0; +} + +static int nitrox_aes_setkey(struct crypto_skcipher *cipher, const u8 *key, + unsigned int keylen) +{ + int aes_keylen; + + aes_keylen = flexi_aes_keylen(keylen); + if (aes_keylen < 0) + return -EINVAL; + return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen); +} + +static int alloc_src_sglist(struct skcipher_request *skreq, int ivsize) +{ + struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq); + int nents = sg_nents(skreq->src) + 1; + int ret; + + /* Allocate buffer to hold IV and input scatterlist array */ + ret = alloc_src_req_buf(nkreq, nents, ivsize); + if (ret) + return ret; + + nitrox_creq_copy_iv(nkreq->src, skreq->iv, ivsize); + nitrox_creq_set_src_sg(nkreq, nents, ivsize, skreq->src, + skreq->cryptlen); + + return 0; +} + +static int alloc_dst_sglist(struct skcipher_request *skreq, int ivsize) +{ + struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq); + int nents = sg_nents(skreq->dst) + 3; + int ret; + + /* Allocate buffer to hold ORH, COMPLETION and output scatterlist + * array + */ + ret = alloc_dst_req_buf(nkreq, nents); + if (ret) + return ret; + + nitrox_creq_set_orh(nkreq); + nitrox_creq_set_comp(nkreq); + nitrox_creq_set_dst_sg(nkreq, nents, ivsize, skreq->dst, + skreq->cryptlen); + + return 0; +} + +static int nitrox_skcipher_crypt(struct skcipher_request *skreq, bool enc) +{ + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(skreq); + struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(cipher); + struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq); + int ivsize = crypto_skcipher_ivsize(cipher); + struct se_crypto_request *creq; + int ret; + + creq = &nkreq->creq; + creq->flags = skreq->base.flags; + creq->gfp = (skreq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + + /* fill the request */ + creq->ctrl.value = 0; + creq->opcode = FLEXI_CRYPTO_ENCRYPT_HMAC; + creq->ctrl.s.arg = (enc ? ENCRYPT : DECRYPT); + /* param0: length of the data to be encrypted */ + creq->gph.param0 = cpu_to_be16(skreq->cryptlen); + creq->gph.param1 = 0; + /* param2: encryption data offset */ + creq->gph.param2 = cpu_to_be16(ivsize); + creq->gph.param3 = 0; + + creq->ctx_handle = nctx->u.ctx_handle; + creq->ctrl.s.ctxl = sizeof(struct flexi_crypto_context); + + ret = alloc_src_sglist(skreq, ivsize); + if (ret) + return ret; + + ret = alloc_dst_sglist(skreq, ivsize); + if (ret) { + free_src_sglist(skreq); + return ret; + } + + /* send the crypto request */ + return nitrox_process_se_request(nctx->ndev, creq, nctx->callback, + skreq); +} + +static int nitrox_cbc_decrypt(struct skcipher_request *skreq) +{ + struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq); + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(skreq); + int ivsize = crypto_skcipher_ivsize(cipher); + gfp_t flags = (skreq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + unsigned int start = skreq->cryptlen - ivsize; + + if (skreq->src != skreq->dst) + return nitrox_skcipher_crypt(skreq, false); + + nkreq->iv_out = kmalloc(ivsize, flags); + if (!nkreq->iv_out) + return -ENOMEM; + + scatterwalk_map_and_copy(nkreq->iv_out, skreq->src, start, ivsize, 0); + return nitrox_skcipher_crypt(skreq, false); +} + +static int nitrox_aes_encrypt(struct skcipher_request *skreq) +{ + return nitrox_skcipher_crypt(skreq, true); +} + +static int nitrox_aes_decrypt(struct skcipher_request *skreq) +{ + return nitrox_skcipher_crypt(skreq, false); +} + +static int nitrox_3des_setkey(struct crypto_skcipher *cipher, + const u8 *key, unsigned int keylen) +{ + return verify_skcipher_des3_key(cipher, key) ?: + nitrox_skcipher_setkey(cipher, 0, key, keylen); +} + +static int nitrox_3des_encrypt(struct skcipher_request *skreq) +{ + return nitrox_skcipher_crypt(skreq, true); +} + +static int nitrox_3des_decrypt(struct skcipher_request *skreq) +{ + return nitrox_skcipher_crypt(skreq, false); +} + +static int nitrox_aes_xts_setkey(struct crypto_skcipher *cipher, + const u8 *key, unsigned int keylen) +{ + struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); + struct nitrox_crypto_ctx *nctx = crypto_tfm_ctx(tfm); + struct flexi_crypto_context *fctx; + int aes_keylen, ret; + + ret = xts_check_key(tfm, key, keylen); + if (ret) + return ret; + + keylen /= 2; + + aes_keylen = flexi_aes_keylen(keylen); + if (aes_keylen < 0) + return -EINVAL; + + fctx = nctx->u.fctx; + /* copy KEY2 */ + memcpy(fctx->auth.u.key2, (key + keylen), keylen); + + return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen); +} + +static int nitrox_aes_ctr_rfc3686_setkey(struct crypto_skcipher *cipher, + const u8 *key, unsigned int keylen) +{ + struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); + struct nitrox_crypto_ctx *nctx = crypto_tfm_ctx(tfm); + struct flexi_crypto_context *fctx; + int aes_keylen; + + if (keylen < CTR_RFC3686_NONCE_SIZE) + return -EINVAL; + + fctx = nctx->u.fctx; + + memcpy(fctx->crypto.iv, key + (keylen - CTR_RFC3686_NONCE_SIZE), + CTR_RFC3686_NONCE_SIZE); + + keylen -= CTR_RFC3686_NONCE_SIZE; + + aes_keylen = flexi_aes_keylen(keylen); + if (aes_keylen < 0) + return -EINVAL; + return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen); +} + +static struct skcipher_alg nitrox_skciphers[] = { { + .base = { + .cra_name = "cbc(aes)", + .cra_driver_name = "n5_cbc(aes)", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = nitrox_aes_setkey, + .encrypt = nitrox_aes_encrypt, + .decrypt = nitrox_cbc_decrypt, + .init = nitrox_cbc_init, + .exit = nitrox_skcipher_exit, +}, { + .base = { + .cra_name = "ecb(aes)", + .cra_driver_name = "n5_ecb(aes)", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = nitrox_aes_setkey, + .encrypt = nitrox_aes_encrypt, + .decrypt = nitrox_aes_decrypt, + .init = nitrox_skcipher_init, + .exit = nitrox_skcipher_exit, +}, { + .base = { + .cra_name = "cfb(aes)", + .cra_driver_name = "n5_cfb(aes)", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = nitrox_aes_setkey, + .encrypt = nitrox_aes_encrypt, + .decrypt = nitrox_aes_decrypt, + .init = nitrox_skcipher_init, + .exit = nitrox_skcipher_exit, +}, { + .base = { + .cra_name = "xts(aes)", + .cra_driver_name = "n5_xts(aes)", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = nitrox_aes_xts_setkey, + .encrypt = nitrox_aes_encrypt, + .decrypt = nitrox_aes_decrypt, + .init = nitrox_skcipher_init, + .exit = nitrox_skcipher_exit, +}, { + .base = { + .cra_name = "rfc3686(ctr(aes))", + .cra_driver_name = "n5_rfc3686(ctr(aes))", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, + .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, + .ivsize = CTR_RFC3686_IV_SIZE, + .init = nitrox_skcipher_init, + .exit = nitrox_skcipher_exit, + .setkey = nitrox_aes_ctr_rfc3686_setkey, + .encrypt = nitrox_aes_encrypt, + .decrypt = nitrox_aes_decrypt, +}, { + .base = { + .cra_name = "cts(cbc(aes))", + .cra_driver_name = "n5_cts(cbc(aes))", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = nitrox_aes_setkey, + .encrypt = nitrox_aes_encrypt, + .decrypt = nitrox_aes_decrypt, + .init = nitrox_skcipher_init, + .exit = nitrox_skcipher_exit, +}, { + .base = { + .cra_name = "cbc(des3_ede)", + .cra_driver_name = "n5_cbc(des3_ede)", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + .setkey = nitrox_3des_setkey, + .encrypt = nitrox_3des_encrypt, + .decrypt = nitrox_cbc_decrypt, + .init = nitrox_cbc_init, + .exit = nitrox_skcipher_exit, +}, { + .base = { + .cra_name = "ecb(des3_ede)", + .cra_driver_name = "n5_ecb(des3_ede)", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + .setkey = nitrox_3des_setkey, + .encrypt = nitrox_3des_encrypt, + .decrypt = nitrox_3des_decrypt, + .init = nitrox_skcipher_init, + .exit = nitrox_skcipher_exit, +} + +}; + +int nitrox_register_skciphers(void) +{ + return crypto_register_skciphers(nitrox_skciphers, + ARRAY_SIZE(nitrox_skciphers)); +} + +void nitrox_unregister_skciphers(void) +{ + crypto_unregister_skciphers(nitrox_skciphers, + ARRAY_SIZE(nitrox_skciphers)); +} diff --git a/drivers/crypto/cavium/nitrox/nitrox_sriov.c b/drivers/crypto/cavium/nitrox/nitrox_sriov.c new file mode 100644 index 000000000..43287f847 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_sriov.c @@ -0,0 +1,234 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/pci.h> +#include <linux/delay.h> + +#include "nitrox_dev.h" +#include "nitrox_hal.h" +#include "nitrox_common.h" +#include "nitrox_isr.h" +#include "nitrox_mbx.h" + +/** + * num_vfs_valid - validate VF count + * @num_vfs: number of VF(s) + */ +static inline bool num_vfs_valid(int num_vfs) +{ + bool valid = false; + + switch (num_vfs) { + case 16: + case 32: + case 64: + case 128: + valid = true; + break; + } + + return valid; +} + +static inline enum vf_mode num_vfs_to_mode(int num_vfs) +{ + enum vf_mode mode = 0; + + switch (num_vfs) { + case 0: + mode = __NDEV_MODE_PF; + break; + case 16: + mode = __NDEV_MODE_VF16; + break; + case 32: + mode = __NDEV_MODE_VF32; + break; + case 64: + mode = __NDEV_MODE_VF64; + break; + case 128: + mode = __NDEV_MODE_VF128; + break; + } + + return mode; +} + +static inline int vf_mode_to_nr_queues(enum vf_mode mode) +{ + int nr_queues = 0; + + switch (mode) { + case __NDEV_MODE_PF: + nr_queues = MAX_PF_QUEUES; + break; + case __NDEV_MODE_VF16: + nr_queues = 8; + break; + case __NDEV_MODE_VF32: + nr_queues = 4; + break; + case __NDEV_MODE_VF64: + nr_queues = 2; + break; + case __NDEV_MODE_VF128: + nr_queues = 1; + break; + } + + return nr_queues; +} + +static void nitrox_pf_cleanup(struct nitrox_device *ndev) +{ + /* PF has no queues in SR-IOV mode */ + atomic_set(&ndev->state, __NDEV_NOT_READY); + /* unregister crypto algorithms */ + nitrox_crypto_unregister(); + + /* cleanup PF resources */ + nitrox_unregister_interrupts(ndev); + nitrox_common_sw_cleanup(ndev); +} + +/** + * nitrox_pf_reinit - re-initialize PF resources once SR-IOV is disabled + * @ndev: NITROX device + */ +static int nitrox_pf_reinit(struct nitrox_device *ndev) +{ + int err; + + /* allocate resources for PF */ + err = nitrox_common_sw_init(ndev); + if (err) + return err; + + err = nitrox_register_interrupts(ndev); + if (err) { + nitrox_common_sw_cleanup(ndev); + return err; + } + + /* configure the AQM queues */ + nitrox_config_aqm_rings(ndev); + + /* configure the packet queues */ + nitrox_config_pkt_input_rings(ndev); + nitrox_config_pkt_solicit_ports(ndev); + + /* set device to ready state */ + atomic_set(&ndev->state, __NDEV_READY); + + /* register crypto algorithms */ + return nitrox_crypto_register(); +} + +static void nitrox_sriov_cleanup(struct nitrox_device *ndev) +{ + /* unregister interrupts for PF in SR-IOV */ + nitrox_sriov_unregister_interrupts(ndev); + nitrox_mbox_cleanup(ndev); +} + +static int nitrox_sriov_init(struct nitrox_device *ndev) +{ + int ret; + + /* register interrupts for PF in SR-IOV */ + ret = nitrox_sriov_register_interupts(ndev); + if (ret) + return ret; + + ret = nitrox_mbox_init(ndev); + if (ret) + goto sriov_init_fail; + + return 0; + +sriov_init_fail: + nitrox_sriov_cleanup(ndev); + return ret; +} + +static int nitrox_sriov_enable(struct pci_dev *pdev, int num_vfs) +{ + struct nitrox_device *ndev = pci_get_drvdata(pdev); + int err; + + if (!num_vfs_valid(num_vfs)) { + dev_err(DEV(ndev), "Invalid num_vfs %d\n", num_vfs); + return -EINVAL; + } + + if (pci_num_vf(pdev) == num_vfs) + return num_vfs; + + err = pci_enable_sriov(pdev, num_vfs); + if (err) { + dev_err(DEV(ndev), "failed to enable PCI sriov %d\n", err); + return err; + } + dev_info(DEV(ndev), "Enabled VF(s) %d\n", num_vfs); + + ndev->mode = num_vfs_to_mode(num_vfs); + ndev->iov.num_vfs = num_vfs; + ndev->iov.max_vf_queues = vf_mode_to_nr_queues(ndev->mode); + /* set bit in flags */ + set_bit(__NDEV_SRIOV_BIT, &ndev->flags); + + /* cleanup PF resources */ + nitrox_pf_cleanup(ndev); + + /* PF SR-IOV mode initialization */ + err = nitrox_sriov_init(ndev); + if (err) + goto iov_fail; + + config_nps_core_vfcfg_mode(ndev, ndev->mode); + return num_vfs; + +iov_fail: + pci_disable_sriov(pdev); + /* clear bit in flags */ + clear_bit(__NDEV_SRIOV_BIT, &ndev->flags); + ndev->iov.num_vfs = 0; + ndev->mode = __NDEV_MODE_PF; + /* reset back to working mode in PF */ + nitrox_pf_reinit(ndev); + return err; +} + +static int nitrox_sriov_disable(struct pci_dev *pdev) +{ + struct nitrox_device *ndev = pci_get_drvdata(pdev); + + if (!test_bit(__NDEV_SRIOV_BIT, &ndev->flags)) + return 0; + + if (pci_vfs_assigned(pdev)) { + dev_warn(DEV(ndev), "VFs are attached to VM. Can't disable SR-IOV\n"); + return -EPERM; + } + pci_disable_sriov(pdev); + /* clear bit in flags */ + clear_bit(__NDEV_SRIOV_BIT, &ndev->flags); + + ndev->iov.num_vfs = 0; + ndev->iov.max_vf_queues = 0; + ndev->mode = __NDEV_MODE_PF; + + /* cleanup PF SR-IOV resources */ + nitrox_sriov_cleanup(ndev); + + config_nps_core_vfcfg_mode(ndev, ndev->mode); + + return nitrox_pf_reinit(ndev); +} + +int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs) +{ + if (!num_vfs) + return nitrox_sriov_disable(pdev); + + return nitrox_sriov_enable(pdev, num_vfs); +} diff --git a/drivers/crypto/cavium/zip/Makefile b/drivers/crypto/cavium/zip/Makefile new file mode 100644 index 000000000..020d189d7 --- /dev/null +++ b/drivers/crypto/cavium/zip/Makefile @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for Cavium's ZIP Driver. +# + +obj-$(CONFIG_CRYPTO_DEV_CAVIUM_ZIP) += thunderx_zip.o +thunderx_zip-y := zip_main.o \ + zip_device.o \ + zip_crypto.o \ + zip_mem.o \ + zip_deflate.o \ + zip_inflate.o diff --git a/drivers/crypto/cavium/zip/common.h b/drivers/crypto/cavium/zip/common.h new file mode 100644 index 000000000..54f6fb054 --- /dev/null +++ b/drivers/crypto/cavium/zip/common.h @@ -0,0 +1,222 @@ +/***********************license start************************************ + * Copyright (c) 2003-2017 Cavium, Inc. + * All rights reserved. + * + * License: one of 'Cavium License' or 'GNU General Public License Version 2' + * + * This file is provided under the terms of the Cavium License (see below) + * or under the terms of GNU General Public License, Version 2, as + * published by the Free Software Foundation. When using or redistributing + * this file, you may do so under either license. + * + * Cavium License: Redistribution and use in source and binary forms, with + * or without modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * * Neither the name of Cavium Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * This Software, including technical data, may be subject to U.S. export + * control laws, including the U.S. Export Administration Act and its + * associated regulations, and may be subject to export or import + * regulations in other countries. + * + * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" + * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS + * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH + * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY + * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT + * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) + * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A + * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET + * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE + * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES + * WITH YOU. + ***********************license end**************************************/ + +#ifndef __COMMON_H__ +#define __COMMON_H__ + +#include <linux/delay.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/seq_file.h> +#include <linux/string.h> +#include <linux/types.h> + +/* Device specific zlib function definitions */ +#include "zip_device.h" + +/* ZIP device definitions */ +#include "zip_main.h" + +/* ZIP memory allocation/deallocation related definitions */ +#include "zip_mem.h" + +/* Device specific structure definitions */ +#include "zip_regs.h" + +#define ZIP_ERROR -1 + +#define ZIP_FLUSH_FINISH 4 + +#define RAW_FORMAT 0 /* for rawpipe */ +#define ZLIB_FORMAT 1 /* for zpipe */ +#define GZIP_FORMAT 2 /* for gzpipe */ +#define LZS_FORMAT 3 /* for lzspipe */ + +/* Max number of ZIP devices supported */ +#define MAX_ZIP_DEVICES 2 + +/* Configures the number of zip queues to be used */ +#define ZIP_NUM_QUEUES 2 + +#define DYNAMIC_STOP_EXCESS 1024 + +/* Maximum buffer sizes in direct mode */ +#define MAX_INPUT_BUFFER_SIZE (64 * 1024) +#define MAX_OUTPUT_BUFFER_SIZE (64 * 1024) + +/** + * struct zip_operation - common data structure for comp and decomp operations + * @input: Next input byte is read from here + * @output: Next output byte written here + * @ctx_addr: Inflate context buffer address + * @history: Pointer to the history buffer + * @input_len: Number of bytes available at next_in + * @input_total_len: Total number of input bytes read + * @output_len: Remaining free space at next_out + * @output_total_len: Total number of bytes output so far + * @csum: Checksum value of the uncompressed data + * @flush: Flush flag + * @format: Format (depends on stream's wrap) + * @speed: Speed depends on stream's level + * @ccode: Compression code ( stream's strategy) + * @lzs_flag: Flag for LZS support + * @begin_file: Beginning of file indication for inflate + * @history_len: Size of the history data + * @end_file: Ending of the file indication for inflate + * @compcode: Completion status of the ZIP invocation + * @bytes_read: Input bytes read in current instruction + * @bits_processed: Total bits processed for entire file + * @sizeofptr: To distinguish between ILP32 and LP64 + * @sizeofzops: Optional just for padding + * + * This structure is used to maintain the required meta data for the + * comp and decomp operations. + */ +struct zip_operation { + u8 *input; + u8 *output; + u64 ctx_addr; + u64 history; + + u32 input_len; + u32 input_total_len; + + u32 output_len; + u32 output_total_len; + + u32 csum; + u32 flush; + + u32 format; + u32 speed; + u32 ccode; + u32 lzs_flag; + + u32 begin_file; + u32 history_len; + + u32 end_file; + u32 compcode; + u32 bytes_read; + u32 bits_processed; + + u32 sizeofptr; + u32 sizeofzops; +}; + +static inline int zip_poll_result(union zip_zres_s *result) +{ + int retries = 1000; + + while (!result->s.compcode) { + if (!--retries) { + pr_err("ZIP ERR: request timed out"); + return -ETIMEDOUT; + } + udelay(10); + /* + * Force re-reading of compcode which is updated + * by the ZIP coprocessor. + */ + rmb(); + } + return 0; +} + +/* error messages */ +#define zip_err(fmt, args...) pr_err("ZIP ERR:%s():%d: " \ + fmt "\n", __func__, __LINE__, ## args) + +#ifdef MSG_ENABLE +/* Enable all messages */ +#define zip_msg(fmt, args...) pr_info("ZIP_MSG:" fmt "\n", ## args) +#else +#define zip_msg(fmt, args...) +#endif + +#if defined(ZIP_DEBUG_ENABLE) && defined(MSG_ENABLE) + +#ifdef DEBUG_LEVEL + +#define FILE_NAME (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : \ + strrchr(__FILE__, '\\') ? strrchr(__FILE__, '\\') + 1 : __FILE__) + +#if DEBUG_LEVEL >= 4 + +#define zip_dbg(fmt, args...) pr_info("ZIP DBG: %s: %s() : %d: " \ + fmt "\n", FILE_NAME, __func__, __LINE__, ## args) + +#elif DEBUG_LEVEL >= 3 + +#define zip_dbg(fmt, args...) pr_info("ZIP DBG: %s: %s() : %d: " \ + fmt "\n", FILE_NAME, __func__, __LINE__, ## args) + +#elif DEBUG_LEVEL >= 2 + +#define zip_dbg(fmt, args...) pr_info("ZIP DBG: %s() : %d: " \ + fmt "\n", __func__, __LINE__, ## args) + +#else + +#define zip_dbg(fmt, args...) pr_info("ZIP DBG:" fmt "\n", ## args) + +#endif /* DEBUG LEVEL >=4 */ + +#else + +#define zip_dbg(fmt, args...) pr_info("ZIP DBG:" fmt "\n", ## args) + +#endif /* DEBUG_LEVEL */ +#else + +#define zip_dbg(fmt, args...) + +#endif /* ZIP_DEBUG_ENABLE && MSG_ENABLE*/ + +#endif diff --git a/drivers/crypto/cavium/zip/zip_crypto.c b/drivers/crypto/cavium/zip/zip_crypto.c new file mode 100644 index 000000000..1046a746d --- /dev/null +++ b/drivers/crypto/cavium/zip/zip_crypto.c @@ -0,0 +1,301 @@ +/***********************license start************************************ + * Copyright (c) 2003-2017 Cavium, Inc. + * All rights reserved. + * + * License: one of 'Cavium License' or 'GNU General Public License Version 2' + * + * This file is provided under the terms of the Cavium License (see below) + * or under the terms of GNU General Public License, Version 2, as + * published by the Free Software Foundation. When using or redistributing + * this file, you may do so under either license. + * + * Cavium License: Redistribution and use in source and binary forms, with + * or without modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * * Neither the name of Cavium Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * This Software, including technical data, may be subject to U.S. export + * control laws, including the U.S. Export Administration Act and its + * associated regulations, and may be subject to export or import + * regulations in other countries. + * + * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" + * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS + * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH + * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY + * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT + * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) + * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A + * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET + * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE + * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES + * WITH YOU. + ***********************license end**************************************/ + +#include "zip_crypto.h" + +static void zip_static_init_zip_ops(struct zip_operation *zip_ops, + int lzs_flag) +{ + zip_ops->flush = ZIP_FLUSH_FINISH; + + /* equivalent to level 6 of opensource zlib */ + zip_ops->speed = 1; + + if (!lzs_flag) { + zip_ops->ccode = 0; /* Auto Huffman */ + zip_ops->lzs_flag = 0; + zip_ops->format = ZLIB_FORMAT; + } else { + zip_ops->ccode = 3; /* LZS Encoding */ + zip_ops->lzs_flag = 1; + zip_ops->format = LZS_FORMAT; + } + zip_ops->begin_file = 1; + zip_ops->history_len = 0; + zip_ops->end_file = 1; + zip_ops->compcode = 0; + zip_ops->csum = 1; /* Adler checksum desired */ +} + +static int zip_ctx_init(struct zip_kernel_ctx *zip_ctx, int lzs_flag) +{ + struct zip_operation *comp_ctx = &zip_ctx->zip_comp; + struct zip_operation *decomp_ctx = &zip_ctx->zip_decomp; + + zip_static_init_zip_ops(comp_ctx, lzs_flag); + zip_static_init_zip_ops(decomp_ctx, lzs_flag); + + comp_ctx->input = zip_data_buf_alloc(MAX_INPUT_BUFFER_SIZE); + if (!comp_ctx->input) + return -ENOMEM; + + comp_ctx->output = zip_data_buf_alloc(MAX_OUTPUT_BUFFER_SIZE); + if (!comp_ctx->output) + goto err_comp_input; + + decomp_ctx->input = zip_data_buf_alloc(MAX_INPUT_BUFFER_SIZE); + if (!decomp_ctx->input) + goto err_comp_output; + + decomp_ctx->output = zip_data_buf_alloc(MAX_OUTPUT_BUFFER_SIZE); + if (!decomp_ctx->output) + goto err_decomp_input; + + return 0; + +err_decomp_input: + zip_data_buf_free(decomp_ctx->input, MAX_INPUT_BUFFER_SIZE); + +err_comp_output: + zip_data_buf_free(comp_ctx->output, MAX_OUTPUT_BUFFER_SIZE); + +err_comp_input: + zip_data_buf_free(comp_ctx->input, MAX_INPUT_BUFFER_SIZE); + + return -ENOMEM; +} + +static void zip_ctx_exit(struct zip_kernel_ctx *zip_ctx) +{ + struct zip_operation *comp_ctx = &zip_ctx->zip_comp; + struct zip_operation *dec_ctx = &zip_ctx->zip_decomp; + + zip_data_buf_free(comp_ctx->input, MAX_INPUT_BUFFER_SIZE); + zip_data_buf_free(comp_ctx->output, MAX_OUTPUT_BUFFER_SIZE); + + zip_data_buf_free(dec_ctx->input, MAX_INPUT_BUFFER_SIZE); + zip_data_buf_free(dec_ctx->output, MAX_OUTPUT_BUFFER_SIZE); +} + +static int zip_compress(const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen, + struct zip_kernel_ctx *zip_ctx) +{ + struct zip_operation *zip_ops = NULL; + struct zip_state *zip_state; + struct zip_device *zip = NULL; + int ret; + + if (!zip_ctx || !src || !dst || !dlen) + return -ENOMEM; + + zip = zip_get_device(zip_get_node_id()); + if (!zip) + return -ENODEV; + + zip_state = kzalloc(sizeof(*zip_state), GFP_ATOMIC); + if (!zip_state) + return -ENOMEM; + + zip_ops = &zip_ctx->zip_comp; + + zip_ops->input_len = slen; + zip_ops->output_len = *dlen; + memcpy(zip_ops->input, src, slen); + + ret = zip_deflate(zip_ops, zip_state, zip); + + if (!ret) { + *dlen = zip_ops->output_len; + memcpy(dst, zip_ops->output, *dlen); + } + kfree(zip_state); + return ret; +} + +static int zip_decompress(const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen, + struct zip_kernel_ctx *zip_ctx) +{ + struct zip_operation *zip_ops = NULL; + struct zip_state *zip_state; + struct zip_device *zip = NULL; + int ret; + + if (!zip_ctx || !src || !dst || !dlen) + return -ENOMEM; + + zip = zip_get_device(zip_get_node_id()); + if (!zip) + return -ENODEV; + + zip_state = kzalloc(sizeof(*zip_state), GFP_ATOMIC); + if (!zip_state) + return -ENOMEM; + + zip_ops = &zip_ctx->zip_decomp; + memcpy(zip_ops->input, src, slen); + + /* Work around for a bug in zlib which needs an extra bytes sometimes */ + if (zip_ops->ccode != 3) /* Not LZS Encoding */ + zip_ops->input[slen++] = 0; + + zip_ops->input_len = slen; + zip_ops->output_len = *dlen; + + ret = zip_inflate(zip_ops, zip_state, zip); + + if (!ret) { + *dlen = zip_ops->output_len; + memcpy(dst, zip_ops->output, *dlen); + } + kfree(zip_state); + return ret; +} + +/* Legacy Compress framework start */ +int zip_alloc_comp_ctx_deflate(struct crypto_tfm *tfm) +{ + struct zip_kernel_ctx *zip_ctx = crypto_tfm_ctx(tfm); + + return zip_ctx_init(zip_ctx, 0); +} + +int zip_alloc_comp_ctx_lzs(struct crypto_tfm *tfm) +{ + struct zip_kernel_ctx *zip_ctx = crypto_tfm_ctx(tfm); + + return zip_ctx_init(zip_ctx, 1); +} + +void zip_free_comp_ctx(struct crypto_tfm *tfm) +{ + struct zip_kernel_ctx *zip_ctx = crypto_tfm_ctx(tfm); + + zip_ctx_exit(zip_ctx); +} + +int zip_comp_compress(struct crypto_tfm *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen) +{ + struct zip_kernel_ctx *zip_ctx = crypto_tfm_ctx(tfm); + + return zip_compress(src, slen, dst, dlen, zip_ctx); +} + +int zip_comp_decompress(struct crypto_tfm *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen) +{ + struct zip_kernel_ctx *zip_ctx = crypto_tfm_ctx(tfm); + + return zip_decompress(src, slen, dst, dlen, zip_ctx); +} /* Legacy compress framework end */ + +/* SCOMP framework start */ +void *zip_alloc_scomp_ctx_deflate(struct crypto_scomp *tfm) +{ + int ret; + struct zip_kernel_ctx *zip_ctx; + + zip_ctx = kzalloc(sizeof(*zip_ctx), GFP_KERNEL); + if (!zip_ctx) + return ERR_PTR(-ENOMEM); + + ret = zip_ctx_init(zip_ctx, 0); + + if (ret) { + kfree_sensitive(zip_ctx); + return ERR_PTR(ret); + } + + return zip_ctx; +} + +void *zip_alloc_scomp_ctx_lzs(struct crypto_scomp *tfm) +{ + int ret; + struct zip_kernel_ctx *zip_ctx; + + zip_ctx = kzalloc(sizeof(*zip_ctx), GFP_KERNEL); + if (!zip_ctx) + return ERR_PTR(-ENOMEM); + + ret = zip_ctx_init(zip_ctx, 1); + + if (ret) { + kfree_sensitive(zip_ctx); + return ERR_PTR(ret); + } + + return zip_ctx; +} + +void zip_free_scomp_ctx(struct crypto_scomp *tfm, void *ctx) +{ + struct zip_kernel_ctx *zip_ctx = ctx; + + zip_ctx_exit(zip_ctx); + kfree_sensitive(zip_ctx); +} + +int zip_scomp_compress(struct crypto_scomp *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen, void *ctx) +{ + struct zip_kernel_ctx *zip_ctx = ctx; + + return zip_compress(src, slen, dst, dlen, zip_ctx); +} + +int zip_scomp_decompress(struct crypto_scomp *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen, void *ctx) +{ + struct zip_kernel_ctx *zip_ctx = ctx; + + return zip_decompress(src, slen, dst, dlen, zip_ctx); +} /* SCOMP framework end */ diff --git a/drivers/crypto/cavium/zip/zip_crypto.h b/drivers/crypto/cavium/zip/zip_crypto.h new file mode 100644 index 000000000..b59ddfcac --- /dev/null +++ b/drivers/crypto/cavium/zip/zip_crypto.h @@ -0,0 +1,79 @@ +/***********************license start************************************ + * Copyright (c) 2003-2017 Cavium, Inc. + * All rights reserved. + * + * License: one of 'Cavium License' or 'GNU General Public License Version 2' + * + * This file is provided under the terms of the Cavium License (see below) + * or under the terms of GNU General Public License, Version 2, as + * published by the Free Software Foundation. When using or redistributing + * this file, you may do so under either license. + * + * Cavium License: Redistribution and use in source and binary forms, with + * or without modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * * Neither the name of Cavium Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * This Software, including technical data, may be subject to U.S. export + * control laws, including the U.S. Export Administration Act and its + * associated regulations, and may be subject to export or import + * regulations in other countries. + * + * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" + * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS + * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH + * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY + * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT + * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) + * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A + * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET + * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE + * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES + * WITH YOU. + ***********************license end**************************************/ + +#ifndef __ZIP_CRYPTO_H__ +#define __ZIP_CRYPTO_H__ + +#include <linux/crypto.h> +#include <crypto/internal/scompress.h> +#include "common.h" +#include "zip_deflate.h" +#include "zip_inflate.h" + +struct zip_kernel_ctx { + struct zip_operation zip_comp; + struct zip_operation zip_decomp; +}; + +int zip_alloc_comp_ctx_deflate(struct crypto_tfm *tfm); +int zip_alloc_comp_ctx_lzs(struct crypto_tfm *tfm); +void zip_free_comp_ctx(struct crypto_tfm *tfm); +int zip_comp_compress(struct crypto_tfm *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen); +int zip_comp_decompress(struct crypto_tfm *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen); + +void *zip_alloc_scomp_ctx_deflate(struct crypto_scomp *tfm); +void *zip_alloc_scomp_ctx_lzs(struct crypto_scomp *tfm); +void zip_free_scomp_ctx(struct crypto_scomp *tfm, void *zip_ctx); +int zip_scomp_compress(struct crypto_scomp *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen, void *ctx); +int zip_scomp_decompress(struct crypto_scomp *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen, void *ctx); +#endif diff --git a/drivers/crypto/cavium/zip/zip_deflate.c b/drivers/crypto/cavium/zip/zip_deflate.c new file mode 100644 index 000000000..d7133f857 --- /dev/null +++ b/drivers/crypto/cavium/zip/zip_deflate.c @@ -0,0 +1,200 @@ +/***********************license start************************************ + * Copyright (c) 2003-2017 Cavium, Inc. + * All rights reserved. + * + * License: one of 'Cavium License' or 'GNU General Public License Version 2' + * + * This file is provided under the terms of the Cavium License (see below) + * or under the terms of GNU General Public License, Version 2, as + * published by the Free Software Foundation. When using or redistributing + * this file, you may do so under either license. + * + * Cavium License: Redistribution and use in source and binary forms, with + * or without modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * * Neither the name of Cavium Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * This Software, including technical data, may be subject to U.S. export + * control laws, including the U.S. Export Administration Act and its + * associated regulations, and may be subject to export or import + * regulations in other countries. + * + * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" + * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS + * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH + * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY + * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT + * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) + * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A + * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET + * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE + * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES + * WITH YOU. + ***********************license end**************************************/ + +#include <linux/delay.h> +#include <linux/sched.h> + +#include "common.h" +#include "zip_deflate.h" + +/* Prepares the deflate zip command */ +static int prepare_zip_command(struct zip_operation *zip_ops, + struct zip_state *s, union zip_inst_s *zip_cmd) +{ + union zip_zres_s *result_ptr = &s->result; + + memset(zip_cmd, 0, sizeof(s->zip_cmd)); + memset(result_ptr, 0, sizeof(s->result)); + + /* IWORD #0 */ + /* History gather */ + zip_cmd->s.hg = 0; + /* compression enable = 1 for deflate */ + zip_cmd->s.ce = 1; + /* sf (sync flush) */ + zip_cmd->s.sf = 1; + /* ef (end of file) */ + if (zip_ops->flush == ZIP_FLUSH_FINISH) { + zip_cmd->s.ef = 1; + zip_cmd->s.sf = 0; + } + + zip_cmd->s.cc = zip_ops->ccode; + /* ss (compression speed/storage) */ + zip_cmd->s.ss = zip_ops->speed; + + /* IWORD #1 */ + /* adler checksum */ + zip_cmd->s.adlercrc32 = zip_ops->csum; + zip_cmd->s.historylength = zip_ops->history_len; + zip_cmd->s.dg = 0; + + /* IWORD # 6 and 7 - compression input/history pointer */ + zip_cmd->s.inp_ptr_addr.s.addr = __pa(zip_ops->input); + zip_cmd->s.inp_ptr_ctl.s.length = (zip_ops->input_len + + zip_ops->history_len); + zip_cmd->s.ds = 0; + + /* IWORD # 8 and 9 - Output pointer */ + zip_cmd->s.out_ptr_addr.s.addr = __pa(zip_ops->output); + zip_cmd->s.out_ptr_ctl.s.length = zip_ops->output_len; + /* maximum number of output-stream bytes that can be written */ + zip_cmd->s.totaloutputlength = zip_ops->output_len; + + /* IWORD # 10 and 11 - Result pointer */ + zip_cmd->s.res_ptr_addr.s.addr = __pa(result_ptr); + /* Clearing completion code */ + result_ptr->s.compcode = 0; + + return 0; +} + +/** + * zip_deflate - API to offload deflate operation to hardware + * @zip_ops: Pointer to zip operation structure + * @s: Pointer to the structure representing zip state + * @zip_dev: Pointer to zip device structure + * + * This function prepares the zip deflate command and submits it to the zip + * engine for processing. + * + * Return: 0 if successful or error code + */ +int zip_deflate(struct zip_operation *zip_ops, struct zip_state *s, + struct zip_device *zip_dev) +{ + union zip_inst_s *zip_cmd = &s->zip_cmd; + union zip_zres_s *result_ptr = &s->result; + u32 queue; + + /* Prepares zip command based on the input parameters */ + prepare_zip_command(zip_ops, s, zip_cmd); + + atomic64_add(zip_ops->input_len, &zip_dev->stats.comp_in_bytes); + /* Loads zip command into command queues and rings door bell */ + queue = zip_load_instr(zip_cmd, zip_dev); + + /* Stats update for compression requests submitted */ + atomic64_inc(&zip_dev->stats.comp_req_submit); + + /* Wait for completion or error */ + zip_poll_result(result_ptr); + + /* Stats update for compression requests completed */ + atomic64_inc(&zip_dev->stats.comp_req_complete); + + zip_ops->compcode = result_ptr->s.compcode; + switch (zip_ops->compcode) { + case ZIP_CMD_NOTDONE: + zip_dbg("Zip instruction not yet completed"); + return ZIP_ERROR; + + case ZIP_CMD_SUCCESS: + zip_dbg("Zip instruction completed successfully"); + zip_update_cmd_bufs(zip_dev, queue); + break; + + case ZIP_CMD_DTRUNC: + zip_dbg("Output Truncate error"); + /* Returning ZIP_ERROR to avoid copy to user */ + return ZIP_ERROR; + + default: + zip_err("Zip instruction failed. Code:%d", zip_ops->compcode); + return ZIP_ERROR; + } + + /* Update the CRC depending on the format */ + switch (zip_ops->format) { + case RAW_FORMAT: + zip_dbg("RAW Format: %d ", zip_ops->format); + /* Get checksum from engine, need to feed it again */ + zip_ops->csum = result_ptr->s.adler32; + break; + + case ZLIB_FORMAT: + zip_dbg("ZLIB Format: %d ", zip_ops->format); + zip_ops->csum = result_ptr->s.adler32; + break; + + case GZIP_FORMAT: + zip_dbg("GZIP Format: %d ", zip_ops->format); + zip_ops->csum = result_ptr->s.crc32; + break; + + case LZS_FORMAT: + zip_dbg("LZS Format: %d ", zip_ops->format); + break; + + default: + zip_err("Unknown Format:%d\n", zip_ops->format); + } + + atomic64_add(result_ptr->s.totalbyteswritten, + &zip_dev->stats.comp_out_bytes); + + /* Update output_len */ + if (zip_ops->output_len < result_ptr->s.totalbyteswritten) { + /* Dynamic stop && strm->output_len < zipconstants[onfsize] */ + zip_err("output_len (%d) < total bytes written(%d)\n", + zip_ops->output_len, result_ptr->s.totalbyteswritten); + zip_ops->output_len = 0; + + } else { + zip_ops->output_len = result_ptr->s.totalbyteswritten; + } + + return 0; +} diff --git a/drivers/crypto/cavium/zip/zip_deflate.h b/drivers/crypto/cavium/zip/zip_deflate.h new file mode 100644 index 000000000..1d32e76ed --- /dev/null +++ b/drivers/crypto/cavium/zip/zip_deflate.h @@ -0,0 +1,62 @@ +/***********************license start************************************ + * Copyright (c) 2003-2017 Cavium, Inc. + * All rights reserved. + * + * License: one of 'Cavium License' or 'GNU General Public License Version 2' + * + * This file is provided under the terms of the Cavium License (see below) + * or under the terms of GNU General Public License, Version 2, as + * published by the Free Software Foundation. When using or redistributing + * this file, you may do so under either license. + * + * Cavium License: Redistribution and use in source and binary forms, with + * or without modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * * Neither the name of Cavium Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * This Software, including technical data, may be subject to U.S. export + * control laws, including the U.S. Export Administration Act and its + * associated regulations, and may be subject to export or import + * regulations in other countries. + * + * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" + * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS + * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH + * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY + * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT + * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) + * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A + * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET + * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE + * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES + * WITH YOU. + ***********************license end**************************************/ + +#ifndef __ZIP_DEFLATE_H__ +#define __ZIP_DEFLATE_H__ + +/** + * zip_deflate - API to offload deflate operation to hardware + * @zip_ops: Pointer to zip operation structure + * @s: Pointer to the structure representing zip state + * @zip_dev: Pointer to the structure representing zip device + * + * This function prepares the zip deflate command and submits it to the zip + * engine by ringing the doorbell. + * + * Return: 0 if successful or error code + */ +int zip_deflate(struct zip_operation *zip_ops, struct zip_state *s, + struct zip_device *zip_dev); +#endif diff --git a/drivers/crypto/cavium/zip/zip_device.c b/drivers/crypto/cavium/zip/zip_device.c new file mode 100644 index 000000000..f174ec29e --- /dev/null +++ b/drivers/crypto/cavium/zip/zip_device.c @@ -0,0 +1,202 @@ +/***********************license start************************************ + * Copyright (c) 2003-2017 Cavium, Inc. + * All rights reserved. + * + * License: one of 'Cavium License' or 'GNU General Public License Version 2' + * + * This file is provided under the terms of the Cavium License (see below) + * or under the terms of GNU General Public License, Version 2, as + * published by the Free Software Foundation. When using or redistributing + * this file, you may do so under either license. + * + * Cavium License: Redistribution and use in source and binary forms, with + * or without modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * * Neither the name of Cavium Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * This Software, including technical data, may be subject to U.S. export + * control laws, including the U.S. Export Administration Act and its + * associated regulations, and may be subject to export or import + * regulations in other countries. + * + * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" + * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS + * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH + * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY + * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT + * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) + * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A + * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET + * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE + * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES + * WITH YOU. + ***********************license end**************************************/ + +#include "common.h" +#include "zip_deflate.h" + +/** + * zip_cmd_queue_consumed - Calculates the space consumed in the command queue. + * + * @zip_dev: Pointer to zip device structure + * @queue: Queue number + * + * Return: Bytes consumed in the command queue buffer. + */ +static inline u32 zip_cmd_queue_consumed(struct zip_device *zip_dev, int queue) +{ + return ((zip_dev->iq[queue].sw_head - zip_dev->iq[queue].sw_tail) * + sizeof(u64 *)); +} + +/** + * zip_load_instr - Submits the instruction into the ZIP command queue + * @instr: Pointer to the instruction to be submitted + * @zip_dev: Pointer to ZIP device structure to which the instruction is to + * be submitted + * + * This function copies the ZIP instruction to the command queue and rings the + * doorbell to notify the engine of the instruction submission. The command + * queue is maintained in a circular fashion. When there is space for exactly + * one instruction in the queue, next chunk pointer of the queue is made to + * point to the head of the queue, thus maintaining a circular queue. + * + * Return: Queue number to which the instruction was submitted + */ +u32 zip_load_instr(union zip_inst_s *instr, + struct zip_device *zip_dev) +{ + union zip_quex_doorbell dbell; + u32 queue = 0; + u32 consumed = 0; + u64 *ncb_ptr = NULL; + union zip_nptr_s ncp; + + /* + * Distribute the instructions between the enabled queues based on + * the CPU id. + */ + if (raw_smp_processor_id() % 2 == 0) + queue = 0; + else + queue = 1; + + zip_dbg("CPU Core: %d Queue number:%d", raw_smp_processor_id(), queue); + + /* Take cmd buffer lock */ + spin_lock(&zip_dev->iq[queue].lock); + + /* + * Command Queue implementation + * 1. If there is place for new instructions, push the cmd at sw_head. + * 2. If there is place for exactly one instruction, push the new cmd + * at the sw_head. Make sw_head point to the sw_tail to make it + * circular. Write sw_head's physical address to the "Next-Chunk + * Buffer Ptr" to make it cmd_hw_tail. + * 3. Ring the door bell. + */ + zip_dbg("sw_head : %lx", zip_dev->iq[queue].sw_head); + zip_dbg("sw_tail : %lx", zip_dev->iq[queue].sw_tail); + + consumed = zip_cmd_queue_consumed(zip_dev, queue); + /* Check if there is space to push just one cmd */ + if ((consumed + 128) == (ZIP_CMD_QBUF_SIZE - 8)) { + zip_dbg("Cmd queue space available for single command"); + /* Space for one cmd, pust it and make it circular queue */ + memcpy((u8 *)zip_dev->iq[queue].sw_head, (u8 *)instr, + sizeof(union zip_inst_s)); + zip_dev->iq[queue].sw_head += 16; /* 16 64_bit words = 128B */ + + /* Now, point the "Next-Chunk Buffer Ptr" to sw_head */ + ncb_ptr = zip_dev->iq[queue].sw_head; + + zip_dbg("ncb addr :0x%lx sw_head addr :0x%lx", + ncb_ptr, zip_dev->iq[queue].sw_head - 16); + + /* Using Circular command queue */ + zip_dev->iq[queue].sw_head = zip_dev->iq[queue].sw_tail; + /* Mark this buffer for free */ + zip_dev->iq[queue].free_flag = 1; + + /* Write new chunk buffer address at "Next-Chunk Buffer Ptr" */ + ncp.u_reg64 = 0ull; + ncp.s.addr = __pa(zip_dev->iq[queue].sw_head); + *ncb_ptr = ncp.u_reg64; + zip_dbg("*ncb_ptr :0x%lx sw_head[phys] :0x%lx", + *ncb_ptr, __pa(zip_dev->iq[queue].sw_head)); + + zip_dev->iq[queue].pend_cnt++; + + } else { + zip_dbg("Enough space is available for commands"); + /* Push this cmd to cmd queue buffer */ + memcpy((u8 *)zip_dev->iq[queue].sw_head, (u8 *)instr, + sizeof(union zip_inst_s)); + zip_dev->iq[queue].sw_head += 16; /* 16 64_bit words = 128B */ + + zip_dev->iq[queue].pend_cnt++; + } + zip_dbg("sw_head :0x%lx sw_tail :0x%lx hw_tail :0x%lx", + zip_dev->iq[queue].sw_head, zip_dev->iq[queue].sw_tail, + zip_dev->iq[queue].hw_tail); + + zip_dbg(" Pushed the new cmd : pend_cnt : %d", + zip_dev->iq[queue].pend_cnt); + + /* Ring the doorbell */ + dbell.u_reg64 = 0ull; + dbell.s.dbell_cnt = 1; + zip_reg_write(dbell.u_reg64, + (zip_dev->reg_base + ZIP_QUEX_DOORBELL(queue))); + + /* Unlock cmd buffer lock */ + spin_unlock(&zip_dev->iq[queue].lock); + + return queue; +} + +/** + * zip_update_cmd_bufs - Updates the queue statistics after posting the + * instruction + * @zip_dev: Pointer to zip device structure + * @queue: Queue number + */ +void zip_update_cmd_bufs(struct zip_device *zip_dev, u32 queue) +{ + /* Take cmd buffer lock */ + spin_lock(&zip_dev->iq[queue].lock); + + /* Check if the previous buffer can be freed */ + if (zip_dev->iq[queue].free_flag == 1) { + zip_dbg("Free flag. Free cmd buffer, adjust sw head and tail"); + /* Reset the free flag */ + zip_dev->iq[queue].free_flag = 0; + + /* Point the hw_tail to start of the new chunk buffer */ + zip_dev->iq[queue].hw_tail = zip_dev->iq[queue].sw_head; + } else { + zip_dbg("Free flag not set. increment hw tail"); + zip_dev->iq[queue].hw_tail += 16; /* 16 64_bit words = 128B */ + } + + zip_dev->iq[queue].done_cnt++; + zip_dev->iq[queue].pend_cnt--; + + zip_dbg("sw_head :0x%lx sw_tail :0x%lx hw_tail :0x%lx", + zip_dev->iq[queue].sw_head, zip_dev->iq[queue].sw_tail, + zip_dev->iq[queue].hw_tail); + zip_dbg(" Got CC : pend_cnt : %d\n", zip_dev->iq[queue].pend_cnt); + + spin_unlock(&zip_dev->iq[queue].lock); +} diff --git a/drivers/crypto/cavium/zip/zip_device.h b/drivers/crypto/cavium/zip/zip_device.h new file mode 100644 index 000000000..9e18b3b93 --- /dev/null +++ b/drivers/crypto/cavium/zip/zip_device.h @@ -0,0 +1,108 @@ +/***********************license start************************************ + * Copyright (c) 2003-2017 Cavium, Inc. + * All rights reserved. + * + * License: one of 'Cavium License' or 'GNU General Public License Version 2' + * + * This file is provided under the terms of the Cavium License (see below) + * or under the terms of GNU General Public License, Version 2, as + * published by the Free Software Foundation. When using or redistributing + * this file, you may do so under either license. + * + * Cavium License: Redistribution and use in source and binary forms, with + * or without modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * * Neither the name of Cavium Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * This Software, including technical data, may be subject to U.S. export + * control laws, including the U.S. Export Administration Act and its + * associated regulations, and may be subject to export or import + * regulations in other countries. + * + * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" + * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS + * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH + * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY + * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT + * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) + * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A + * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET + * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE + * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES + * WITH YOU. + ***********************license end**************************************/ + +#ifndef __ZIP_DEVICE_H__ +#define __ZIP_DEVICE_H__ + +#include <linux/types.h> +#include "zip_main.h" + +struct sg_info { + /* + * Pointer to the input data when scatter_gather == 0 and + * pointer to the input gather list buffer when scatter_gather == 1 + */ + union zip_zptr_s *gather; + + /* + * Pointer to the output data when scatter_gather == 0 and + * pointer to the output scatter list buffer when scatter_gather == 1 + */ + union zip_zptr_s *scatter; + + /* + * Holds size of the output buffer pointed by scatter list + * when scatter_gather == 1 + */ + u64 scatter_buf_size; + + /* for gather data */ + u64 gather_enable; + + /* for scatter data */ + u64 scatter_enable; + + /* Number of gather list pointers for gather data */ + u32 gbuf_cnt; + + /* Number of scatter list pointers for scatter data */ + u32 sbuf_cnt; + + /* Buffers allocation state */ + u8 alloc_state; +}; + +/** + * struct zip_state - Structure representing the required information related + * to a command + * @zip_cmd: Pointer to zip instruction structure + * @result: Pointer to zip result structure + * @ctx: Context pointer for inflate + * @history: Decompression history pointer + * @sginfo: Scatter-gather info structure + */ +struct zip_state { + union zip_inst_s zip_cmd; + union zip_zres_s result; + union zip_zptr_s *ctx; + union zip_zptr_s *history; + struct sg_info sginfo; +}; + +#define ZIP_CONTEXT_SIZE 2048 +#define ZIP_INFLATE_HISTORY_SIZE 32768 +#define ZIP_DEFLATE_HISTORY_SIZE 32768 + +#endif diff --git a/drivers/crypto/cavium/zip/zip_inflate.c b/drivers/crypto/cavium/zip/zip_inflate.c new file mode 100644 index 000000000..7e0d73e2f --- /dev/null +++ b/drivers/crypto/cavium/zip/zip_inflate.c @@ -0,0 +1,223 @@ +/***********************license start************************************ + * Copyright (c) 2003-2017 Cavium, Inc. + * All rights reserved. + * + * License: one of 'Cavium License' or 'GNU General Public License Version 2' + * + * This file is provided under the terms of the Cavium License (see below) + * or under the terms of GNU General Public License, Version 2, as + * published by the Free Software Foundation. When using or redistributing + * this file, you may do so under either license. + * + * Cavium License: Redistribution and use in source and binary forms, with + * or without modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * * Neither the name of Cavium Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * This Software, including technical data, may be subject to U.S. export + * control laws, including the U.S. Export Administration Act and its + * associated regulations, and may be subject to export or import + * regulations in other countries. + * + * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" + * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS + * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH + * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY + * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT + * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) + * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A + * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET + * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE + * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES + * WITH YOU. + ***********************license end**************************************/ + +#include <linux/delay.h> +#include <linux/sched.h> + +#include "common.h" +#include "zip_inflate.h" + +static int prepare_inflate_zcmd(struct zip_operation *zip_ops, + struct zip_state *s, union zip_inst_s *zip_cmd) +{ + union zip_zres_s *result_ptr = &s->result; + + memset(zip_cmd, 0, sizeof(s->zip_cmd)); + memset(result_ptr, 0, sizeof(s->result)); + + /* IWORD#0 */ + + /* Decompression History Gather list - no gather list */ + zip_cmd->s.hg = 0; + /* For decompression, CE must be 0x0. */ + zip_cmd->s.ce = 0; + /* For decompression, SS must be 0x0. */ + zip_cmd->s.ss = 0; + /* For decompression, SF should always be set. */ + zip_cmd->s.sf = 1; + + /* Begin File */ + if (zip_ops->begin_file == 0) + zip_cmd->s.bf = 0; + else + zip_cmd->s.bf = 1; + + zip_cmd->s.ef = 1; + /* 0: for Deflate decompression, 3: for LZS decompression */ + zip_cmd->s.cc = zip_ops->ccode; + + /* IWORD #1*/ + + /* adler checksum */ + zip_cmd->s.adlercrc32 = zip_ops->csum; + + /* + * HISTORYLENGTH must be 0x0 for any ZIP decompress operation. + * History data is added to a decompression operation via IWORD3. + */ + zip_cmd->s.historylength = 0; + zip_cmd->s.ds = 0; + + /* IWORD # 8 and 9 - Output pointer */ + zip_cmd->s.out_ptr_addr.s.addr = __pa(zip_ops->output); + zip_cmd->s.out_ptr_ctl.s.length = zip_ops->output_len; + + /* Maximum number of output-stream bytes that can be written */ + zip_cmd->s.totaloutputlength = zip_ops->output_len; + + zip_dbg("Data Direct Input case "); + + /* IWORD # 6 and 7 - input pointer */ + zip_cmd->s.dg = 0; + zip_cmd->s.inp_ptr_addr.s.addr = __pa((u8 *)zip_ops->input); + zip_cmd->s.inp_ptr_ctl.s.length = zip_ops->input_len; + + /* IWORD # 10 and 11 - Result pointer */ + zip_cmd->s.res_ptr_addr.s.addr = __pa(result_ptr); + + /* Clearing completion code */ + result_ptr->s.compcode = 0; + + /* Returning 0 for time being.*/ + return 0; +} + +/** + * zip_inflate - API to offload inflate operation to hardware + * @zip_ops: Pointer to zip operation structure + * @s: Pointer to the structure representing zip state + * @zip_dev: Pointer to zip device structure + * + * This function prepares the zip inflate command and submits it to the zip + * engine for processing. + * + * Return: 0 if successful or error code + */ +int zip_inflate(struct zip_operation *zip_ops, struct zip_state *s, + struct zip_device *zip_dev) +{ + union zip_inst_s *zip_cmd = &s->zip_cmd; + union zip_zres_s *result_ptr = &s->result; + u32 queue; + + /* Prepare inflate zip command */ + prepare_inflate_zcmd(zip_ops, s, zip_cmd); + + atomic64_add(zip_ops->input_len, &zip_dev->stats.decomp_in_bytes); + + /* Load inflate command to zip queue and ring the doorbell */ + queue = zip_load_instr(zip_cmd, zip_dev); + + /* Decompression requests submitted stats update */ + atomic64_inc(&zip_dev->stats.decomp_req_submit); + + /* Wait for completion or error */ + zip_poll_result(result_ptr); + + /* Decompression requests completed stats update */ + atomic64_inc(&zip_dev->stats.decomp_req_complete); + + zip_ops->compcode = result_ptr->s.compcode; + switch (zip_ops->compcode) { + case ZIP_CMD_NOTDONE: + zip_dbg("Zip Instruction not yet completed\n"); + return ZIP_ERROR; + + case ZIP_CMD_SUCCESS: + zip_dbg("Zip Instruction completed successfully\n"); + break; + + case ZIP_CMD_DYNAMIC_STOP: + zip_dbg(" Dynamic stop Initiated\n"); + break; + + default: + zip_dbg("Instruction failed. Code = %d\n", zip_ops->compcode); + atomic64_inc(&zip_dev->stats.decomp_bad_reqs); + zip_update_cmd_bufs(zip_dev, queue); + return ZIP_ERROR; + } + + zip_update_cmd_bufs(zip_dev, queue); + + if ((zip_ops->ccode == 3) && (zip_ops->flush == 4) && + (zip_ops->compcode != ZIP_CMD_DYNAMIC_STOP)) + result_ptr->s.ef = 1; + + zip_ops->csum = result_ptr->s.adler32; + + atomic64_add(result_ptr->s.totalbyteswritten, + &zip_dev->stats.decomp_out_bytes); + + if (zip_ops->output_len < result_ptr->s.totalbyteswritten) { + zip_err("output_len (%d) < total bytes written (%d)\n", + zip_ops->output_len, result_ptr->s.totalbyteswritten); + zip_ops->output_len = 0; + } else { + zip_ops->output_len = result_ptr->s.totalbyteswritten; + } + + zip_ops->bytes_read = result_ptr->s.totalbytesread; + zip_ops->bits_processed = result_ptr->s.totalbitsprocessed; + zip_ops->end_file = result_ptr->s.ef; + if (zip_ops->end_file) { + switch (zip_ops->format) { + case RAW_FORMAT: + zip_dbg("RAW Format: %d ", zip_ops->format); + /* Get checksum from engine */ + zip_ops->csum = result_ptr->s.adler32; + break; + + case ZLIB_FORMAT: + zip_dbg("ZLIB Format: %d ", zip_ops->format); + zip_ops->csum = result_ptr->s.adler32; + break; + + case GZIP_FORMAT: + zip_dbg("GZIP Format: %d ", zip_ops->format); + zip_ops->csum = result_ptr->s.crc32; + break; + + case LZS_FORMAT: + zip_dbg("LZS Format: %d ", zip_ops->format); + break; + + default: + zip_err("Format error:%d\n", zip_ops->format); + } + } + + return 0; +} diff --git a/drivers/crypto/cavium/zip/zip_inflate.h b/drivers/crypto/cavium/zip/zip_inflate.h new file mode 100644 index 000000000..6b20f1799 --- /dev/null +++ b/drivers/crypto/cavium/zip/zip_inflate.h @@ -0,0 +1,62 @@ +/***********************license start************************************ + * Copyright (c) 2003-2017 Cavium, Inc. + * All rights reserved. + * + * License: one of 'Cavium License' or 'GNU General Public License Version 2' + * + * This file is provided under the terms of the Cavium License (see below) + * or under the terms of GNU General Public License, Version 2, as + * published by the Free Software Foundation. When using or redistributing + * this file, you may do so under either license. + * + * Cavium License: Redistribution and use in source and binary forms, with + * or without modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * * Neither the name of Cavium Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * This Software, including technical data, may be subject to U.S. export + * control laws, including the U.S. Export Administration Act and its + * associated regulations, and may be subject to export or import + * regulations in other countries. + * + * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" + * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS + * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH + * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY + * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT + * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) + * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A + * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET + * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE + * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES + * WITH YOU. + ***********************license end**************************************/ + +#ifndef __ZIP_INFLATE_H__ +#define __ZIP_INFLATE_H__ + +/** + * zip_inflate - API to offload inflate operation to hardware + * @zip_ops: Pointer to zip operation structure + * @s: Pointer to the structure representing zip state + * @zip_dev: Pointer to the structure representing zip device + * + * This function prepares the zip inflate command and submits it to the zip + * engine for processing. + * + * Return: 0 if successful or error code + */ +int zip_inflate(struct zip_operation *zip_ops, struct zip_state *s, + struct zip_device *zip_dev); +#endif diff --git a/drivers/crypto/cavium/zip/zip_main.c b/drivers/crypto/cavium/zip/zip_main.c new file mode 100644 index 000000000..dc5b7bf7e --- /dev/null +++ b/drivers/crypto/cavium/zip/zip_main.c @@ -0,0 +1,651 @@ +/***********************license start************************************ + * Copyright (c) 2003-2017 Cavium, Inc. + * All rights reserved. + * + * License: one of 'Cavium License' or 'GNU General Public License Version 2' + * + * This file is provided under the terms of the Cavium License (see below) + * or under the terms of GNU General Public License, Version 2, as + * published by the Free Software Foundation. When using or redistributing + * this file, you may do so under either license. + * + * Cavium License: Redistribution and use in source and binary forms, with + * or without modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * * Neither the name of Cavium Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * This Software, including technical data, may be subject to U.S. export + * control laws, including the U.S. Export Administration Act and its + * associated regulations, and may be subject to export or import + * regulations in other countries. + * + * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" + * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS + * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH + * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY + * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT + * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) + * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A + * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET + * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE + * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES + * WITH YOU. + ***********************license end**************************************/ + +#include "common.h" +#include "zip_crypto.h" + +#define DRV_NAME "ThunderX-ZIP" + +static struct zip_device *zip_dev[MAX_ZIP_DEVICES]; + +static const struct pci_device_id zip_id_table[] = { + { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDERX_ZIP) }, + { 0, } +}; + +static void zip_debugfs_init(void); +static void zip_debugfs_exit(void); +static int zip_register_compression_device(void); +static void zip_unregister_compression_device(void); + +void zip_reg_write(u64 val, u64 __iomem *addr) +{ + writeq(val, addr); +} + +u64 zip_reg_read(u64 __iomem *addr) +{ + return readq(addr); +} + +/* + * Allocates new ZIP device structure + * Returns zip_device pointer or NULL if cannot allocate memory for zip_device + */ +static struct zip_device *zip_alloc_device(struct pci_dev *pdev) +{ + struct zip_device *zip = NULL; + int idx; + + for (idx = 0; idx < MAX_ZIP_DEVICES; idx++) { + if (!zip_dev[idx]) + break; + } + + /* To ensure that the index is within the limit */ + if (idx < MAX_ZIP_DEVICES) + zip = devm_kzalloc(&pdev->dev, sizeof(*zip), GFP_KERNEL); + + if (!zip) + return NULL; + + zip_dev[idx] = zip; + zip->index = idx; + return zip; +} + +/** + * zip_get_device - Get ZIP device based on node id of cpu + * + * @node: Node id of the current cpu + * Return: Pointer to Zip device structure + */ +struct zip_device *zip_get_device(int node) +{ + if ((node < MAX_ZIP_DEVICES) && (node >= 0)) + return zip_dev[node]; + + zip_err("ZIP device not found for node id %d\n", node); + return NULL; +} + +/** + * zip_get_node_id - Get the node id of the current cpu + * + * Return: Node id of the current cpu + */ +int zip_get_node_id(void) +{ + return cpu_to_node(raw_smp_processor_id()); +} + +/* Initializes the ZIP h/w sub-system */ +static int zip_init_hw(struct zip_device *zip) +{ + union zip_cmd_ctl cmd_ctl; + union zip_constants constants; + union zip_que_ena que_ena; + union zip_quex_map que_map; + union zip_que_pri que_pri; + + union zip_quex_sbuf_addr que_sbuf_addr; + union zip_quex_sbuf_ctl que_sbuf_ctl; + + int q = 0; + + /* Enable the ZIP Engine(Core) Clock */ + cmd_ctl.u_reg64 = zip_reg_read(zip->reg_base + ZIP_CMD_CTL); + cmd_ctl.s.forceclk = 1; + zip_reg_write(cmd_ctl.u_reg64 & 0xFF, (zip->reg_base + ZIP_CMD_CTL)); + + zip_msg("ZIP_CMD_CTL : 0x%016llx", + zip_reg_read(zip->reg_base + ZIP_CMD_CTL)); + + constants.u_reg64 = zip_reg_read(zip->reg_base + ZIP_CONSTANTS); + zip->depth = constants.s.depth; + zip->onfsize = constants.s.onfsize; + zip->ctxsize = constants.s.ctxsize; + + zip_msg("depth: 0x%016llx , onfsize : 0x%016llx , ctxsize : 0x%016llx", + zip->depth, zip->onfsize, zip->ctxsize); + + /* + * Program ZIP_QUE(0..7)_SBUF_ADDR and ZIP_QUE(0..7)_SBUF_CTL to + * have the correct buffer pointer and size configured for each + * instruction queue. + */ + for (q = 0; q < ZIP_NUM_QUEUES; q++) { + que_sbuf_ctl.u_reg64 = 0ull; + que_sbuf_ctl.s.size = (ZIP_CMD_QBUF_SIZE / sizeof(u64)); + que_sbuf_ctl.s.inst_be = 0; + que_sbuf_ctl.s.stream_id = 0; + zip_reg_write(que_sbuf_ctl.u_reg64, + (zip->reg_base + ZIP_QUEX_SBUF_CTL(q))); + + zip_msg("QUEX_SBUF_CTL[%d]: 0x%016llx", q, + zip_reg_read(zip->reg_base + ZIP_QUEX_SBUF_CTL(q))); + } + + for (q = 0; q < ZIP_NUM_QUEUES; q++) { + memset(&zip->iq[q], 0x0, sizeof(struct zip_iq)); + + spin_lock_init(&zip->iq[q].lock); + + if (zip_cmd_qbuf_alloc(zip, q)) { + while (q != 0) { + q--; + zip_cmd_qbuf_free(zip, q); + } + return -ENOMEM; + } + + /* Initialize tail ptr to head */ + zip->iq[q].sw_tail = zip->iq[q].sw_head; + zip->iq[q].hw_tail = zip->iq[q].sw_head; + + /* Write the physical addr to register */ + que_sbuf_addr.u_reg64 = 0ull; + que_sbuf_addr.s.ptr = (__pa(zip->iq[q].sw_head) >> + ZIP_128B_ALIGN); + + zip_msg("QUE[%d]_PTR(PHYS): 0x%016llx", q, + (u64)que_sbuf_addr.s.ptr); + + zip_reg_write(que_sbuf_addr.u_reg64, + (zip->reg_base + ZIP_QUEX_SBUF_ADDR(q))); + + zip_msg("QUEX_SBUF_ADDR[%d]: 0x%016llx", q, + zip_reg_read(zip->reg_base + ZIP_QUEX_SBUF_ADDR(q))); + + zip_dbg("sw_head :0x%lx sw_tail :0x%lx hw_tail :0x%lx", + zip->iq[q].sw_head, zip->iq[q].sw_tail, + zip->iq[q].hw_tail); + zip_dbg("sw_head phy addr : 0x%lx", que_sbuf_addr.s.ptr); + } + + /* + * Queue-to-ZIP core mapping + * If a queue is not mapped to a particular core, it is equivalent to + * the ZIP core being disabled. + */ + que_ena.u_reg64 = 0x0ull; + /* Enabling queues based on ZIP_NUM_QUEUES */ + for (q = 0; q < ZIP_NUM_QUEUES; q++) + que_ena.s.ena |= (0x1 << q); + zip_reg_write(que_ena.u_reg64, (zip->reg_base + ZIP_QUE_ENA)); + + zip_msg("QUE_ENA : 0x%016llx", + zip_reg_read(zip->reg_base + ZIP_QUE_ENA)); + + for (q = 0; q < ZIP_NUM_QUEUES; q++) { + que_map.u_reg64 = 0ull; + /* Mapping each queue to two ZIP cores */ + que_map.s.zce = 0x3; + zip_reg_write(que_map.u_reg64, + (zip->reg_base + ZIP_QUEX_MAP(q))); + + zip_msg("QUE_MAP(%d) : 0x%016llx", q, + zip_reg_read(zip->reg_base + ZIP_QUEX_MAP(q))); + } + + que_pri.u_reg64 = 0ull; + for (q = 0; q < ZIP_NUM_QUEUES; q++) + que_pri.s.pri |= (0x1 << q); /* Higher Priority RR */ + zip_reg_write(que_pri.u_reg64, (zip->reg_base + ZIP_QUE_PRI)); + + zip_msg("QUE_PRI %016llx", zip_reg_read(zip->reg_base + ZIP_QUE_PRI)); + + return 0; +} + +static void zip_reset(struct zip_device *zip) +{ + union zip_cmd_ctl cmd_ctl; + + cmd_ctl.u_reg64 = 0x0ull; + cmd_ctl.s.reset = 1; /* Forces ZIP cores to do reset */ + zip_reg_write(cmd_ctl.u_reg64, (zip->reg_base + ZIP_CMD_CTL)); +} + +static int zip_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct device *dev = &pdev->dev; + struct zip_device *zip = NULL; + int err; + + zip = zip_alloc_device(pdev); + if (!zip) + return -ENOMEM; + + dev_info(dev, "Found ZIP device %d %x:%x on Node %d\n", zip->index, + pdev->vendor, pdev->device, dev_to_node(dev)); + + pci_set_drvdata(pdev, zip); + zip->pdev = pdev; + + err = pci_enable_device(pdev); + if (err) { + dev_err(dev, "Failed to enable PCI device"); + goto err_free_device; + } + + err = pci_request_regions(pdev, DRV_NAME); + if (err) { + dev_err(dev, "PCI request regions failed 0x%x", err); + goto err_disable_device; + } + + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); + if (err) { + dev_err(dev, "Unable to get usable 48-bit DMA configuration\n"); + goto err_release_regions; + } + + /* MAP configuration registers */ + zip->reg_base = pci_ioremap_bar(pdev, PCI_CFG_ZIP_PF_BAR0); + if (!zip->reg_base) { + dev_err(dev, "ZIP: Cannot map BAR0 CSR memory space, aborting"); + err = -ENOMEM; + goto err_release_regions; + } + + /* Initialize ZIP Hardware */ + err = zip_init_hw(zip); + if (err) + goto err_release_regions; + + /* Register with the Kernel Crypto Interface */ + err = zip_register_compression_device(); + if (err < 0) { + zip_err("ZIP: Kernel Crypto Registration failed\n"); + goto err_register; + } + + /* comp-decomp statistics are handled with debugfs interface */ + zip_debugfs_init(); + + return 0; + +err_register: + zip_reset(zip); + +err_release_regions: + if (zip->reg_base) + iounmap(zip->reg_base); + pci_release_regions(pdev); + +err_disable_device: + pci_disable_device(pdev); + +err_free_device: + pci_set_drvdata(pdev, NULL); + + /* Remove zip_dev from zip_device list, free the zip_device memory */ + zip_dev[zip->index] = NULL; + devm_kfree(dev, zip); + + return err; +} + +static void zip_remove(struct pci_dev *pdev) +{ + struct zip_device *zip = pci_get_drvdata(pdev); + int q = 0; + + if (!zip) + return; + + zip_debugfs_exit(); + + zip_unregister_compression_device(); + + if (zip->reg_base) { + zip_reset(zip); + iounmap(zip->reg_base); + } + + pci_release_regions(pdev); + pci_disable_device(pdev); + + /* + * Free Command Queue buffers. This free should be called for all + * the enabled Queues. + */ + for (q = 0; q < ZIP_NUM_QUEUES; q++) + zip_cmd_qbuf_free(zip, q); + + pci_set_drvdata(pdev, NULL); + /* remove zip device from zip device list */ + zip_dev[zip->index] = NULL; +} + +/* PCI Sub-System Interface */ +static struct pci_driver zip_driver = { + .name = DRV_NAME, + .id_table = zip_id_table, + .probe = zip_probe, + .remove = zip_remove, +}; + +/* Kernel Crypto Subsystem Interface */ + +static struct crypto_alg zip_comp_deflate = { + .cra_name = "deflate", + .cra_driver_name = "deflate-cavium", + .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, + .cra_ctxsize = sizeof(struct zip_kernel_ctx), + .cra_priority = 300, + .cra_module = THIS_MODULE, + .cra_init = zip_alloc_comp_ctx_deflate, + .cra_exit = zip_free_comp_ctx, + .cra_u = { .compress = { + .coa_compress = zip_comp_compress, + .coa_decompress = zip_comp_decompress + } } +}; + +static struct crypto_alg zip_comp_lzs = { + .cra_name = "lzs", + .cra_driver_name = "lzs-cavium", + .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, + .cra_ctxsize = sizeof(struct zip_kernel_ctx), + .cra_priority = 300, + .cra_module = THIS_MODULE, + .cra_init = zip_alloc_comp_ctx_lzs, + .cra_exit = zip_free_comp_ctx, + .cra_u = { .compress = { + .coa_compress = zip_comp_compress, + .coa_decompress = zip_comp_decompress + } } +}; + +static struct scomp_alg zip_scomp_deflate = { + .alloc_ctx = zip_alloc_scomp_ctx_deflate, + .free_ctx = zip_free_scomp_ctx, + .compress = zip_scomp_compress, + .decompress = zip_scomp_decompress, + .base = { + .cra_name = "deflate", + .cra_driver_name = "deflate-scomp-cavium", + .cra_module = THIS_MODULE, + .cra_priority = 300, + } +}; + +static struct scomp_alg zip_scomp_lzs = { + .alloc_ctx = zip_alloc_scomp_ctx_lzs, + .free_ctx = zip_free_scomp_ctx, + .compress = zip_scomp_compress, + .decompress = zip_scomp_decompress, + .base = { + .cra_name = "lzs", + .cra_driver_name = "lzs-scomp-cavium", + .cra_module = THIS_MODULE, + .cra_priority = 300, + } +}; + +static int zip_register_compression_device(void) +{ + int ret; + + ret = crypto_register_alg(&zip_comp_deflate); + if (ret < 0) { + zip_err("Deflate algorithm registration failed\n"); + return ret; + } + + ret = crypto_register_alg(&zip_comp_lzs); + if (ret < 0) { + zip_err("LZS algorithm registration failed\n"); + goto err_unregister_alg_deflate; + } + + ret = crypto_register_scomp(&zip_scomp_deflate); + if (ret < 0) { + zip_err("Deflate scomp algorithm registration failed\n"); + goto err_unregister_alg_lzs; + } + + ret = crypto_register_scomp(&zip_scomp_lzs); + if (ret < 0) { + zip_err("LZS scomp algorithm registration failed\n"); + goto err_unregister_scomp_deflate; + } + + return ret; + +err_unregister_scomp_deflate: + crypto_unregister_scomp(&zip_scomp_deflate); +err_unregister_alg_lzs: + crypto_unregister_alg(&zip_comp_lzs); +err_unregister_alg_deflate: + crypto_unregister_alg(&zip_comp_deflate); + + return ret; +} + +static void zip_unregister_compression_device(void) +{ + crypto_unregister_alg(&zip_comp_deflate); + crypto_unregister_alg(&zip_comp_lzs); + crypto_unregister_scomp(&zip_scomp_deflate); + crypto_unregister_scomp(&zip_scomp_lzs); +} + +/* + * debugfs functions + */ +#ifdef CONFIG_DEBUG_FS +#include <linux/debugfs.h> + +/* Displays ZIP device statistics */ +static int zip_stats_show(struct seq_file *s, void *unused) +{ + u64 val = 0ull; + u64 avg_chunk = 0ull, avg_cr = 0ull; + u32 q = 0; + + int index = 0; + struct zip_device *zip; + struct zip_stats *st; + + for (index = 0; index < MAX_ZIP_DEVICES; index++) { + u64 pending = 0; + + if (zip_dev[index]) { + zip = zip_dev[index]; + st = &zip->stats; + + /* Get all the pending requests */ + for (q = 0; q < ZIP_NUM_QUEUES; q++) { + val = zip_reg_read((zip->reg_base + + ZIP_DBG_QUEX_STA(q))); + pending += val >> 32 & 0xffffff; + } + + val = atomic64_read(&st->comp_req_complete); + avg_chunk = (val) ? atomic64_read(&st->comp_in_bytes) / val : 0; + + val = atomic64_read(&st->comp_out_bytes); + avg_cr = (val) ? atomic64_read(&st->comp_in_bytes) / val : 0; + seq_printf(s, " ZIP Device %d Stats\n" + "-----------------------------------\n" + "Comp Req Submitted : \t%lld\n" + "Comp Req Completed : \t%lld\n" + "Compress In Bytes : \t%lld\n" + "Compressed Out Bytes : \t%lld\n" + "Average Chunk size : \t%llu\n" + "Average Compression ratio : \t%llu\n" + "Decomp Req Submitted : \t%lld\n" + "Decomp Req Completed : \t%lld\n" + "Decompress In Bytes : \t%lld\n" + "Decompressed Out Bytes : \t%lld\n" + "Decompress Bad requests : \t%lld\n" + "Pending Req : \t%lld\n" + "---------------------------------\n", + index, + (u64)atomic64_read(&st->comp_req_submit), + (u64)atomic64_read(&st->comp_req_complete), + (u64)atomic64_read(&st->comp_in_bytes), + (u64)atomic64_read(&st->comp_out_bytes), + avg_chunk, + avg_cr, + (u64)atomic64_read(&st->decomp_req_submit), + (u64)atomic64_read(&st->decomp_req_complete), + (u64)atomic64_read(&st->decomp_in_bytes), + (u64)atomic64_read(&st->decomp_out_bytes), + (u64)atomic64_read(&st->decomp_bad_reqs), + pending); + } + } + return 0; +} + +/* Clears stats data */ +static int zip_clear_show(struct seq_file *s, void *unused) +{ + int index = 0; + + for (index = 0; index < MAX_ZIP_DEVICES; index++) { + if (zip_dev[index]) { + memset(&zip_dev[index]->stats, 0, + sizeof(struct zip_stats)); + seq_printf(s, "Cleared stats for zip %d\n", index); + } + } + + return 0; +} + +static struct zip_registers zipregs[64] = { + {"ZIP_CMD_CTL ", 0x0000ull}, + {"ZIP_THROTTLE ", 0x0010ull}, + {"ZIP_CONSTANTS ", 0x00A0ull}, + {"ZIP_QUE0_MAP ", 0x1400ull}, + {"ZIP_QUE1_MAP ", 0x1408ull}, + {"ZIP_QUE_ENA ", 0x0500ull}, + {"ZIP_QUE_PRI ", 0x0508ull}, + {"ZIP_QUE0_DONE ", 0x2000ull}, + {"ZIP_QUE1_DONE ", 0x2008ull}, + {"ZIP_QUE0_DOORBELL ", 0x4000ull}, + {"ZIP_QUE1_DOORBELL ", 0x4008ull}, + {"ZIP_QUE0_SBUF_ADDR ", 0x1000ull}, + {"ZIP_QUE1_SBUF_ADDR ", 0x1008ull}, + {"ZIP_QUE0_SBUF_CTL ", 0x1200ull}, + {"ZIP_QUE1_SBUF_CTL ", 0x1208ull}, + { NULL, 0} +}; + +/* Prints registers' contents */ +static int zip_regs_show(struct seq_file *s, void *unused) +{ + u64 val = 0; + int i = 0, index = 0; + + for (index = 0; index < MAX_ZIP_DEVICES; index++) { + if (zip_dev[index]) { + seq_printf(s, "--------------------------------\n" + " ZIP Device %d Registers\n" + "--------------------------------\n", + index); + + i = 0; + + while (zipregs[i].reg_name) { + val = zip_reg_read((zip_dev[index]->reg_base + + zipregs[i].reg_offset)); + seq_printf(s, "%s: 0x%016llx\n", + zipregs[i].reg_name, val); + i++; + } + } + } + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(zip_stats); +DEFINE_SHOW_ATTRIBUTE(zip_clear); +DEFINE_SHOW_ATTRIBUTE(zip_regs); + +/* Root directory for thunderx_zip debugfs entry */ +static struct dentry *zip_debugfs_root; + +static void zip_debugfs_init(void) +{ + if (!debugfs_initialized()) + return; + + zip_debugfs_root = debugfs_create_dir("thunderx_zip", NULL); + + /* Creating files for entries inside thunderx_zip directory */ + debugfs_create_file("zip_stats", 0444, zip_debugfs_root, NULL, + &zip_stats_fops); + + debugfs_create_file("zip_clear", 0444, zip_debugfs_root, NULL, + &zip_clear_fops); + + debugfs_create_file("zip_regs", 0444, zip_debugfs_root, NULL, + &zip_regs_fops); + +} + +static void zip_debugfs_exit(void) +{ + debugfs_remove_recursive(zip_debugfs_root); +} + +#else +static void __init zip_debugfs_init(void) { } +static void __exit zip_debugfs_exit(void) { } +#endif +/* debugfs - end */ + +module_pci_driver(zip_driver); + +MODULE_AUTHOR("Cavium Inc"); +MODULE_DESCRIPTION("Cavium Inc ThunderX ZIP Driver"); +MODULE_LICENSE("GPL v2"); +MODULE_DEVICE_TABLE(pci, zip_id_table); diff --git a/drivers/crypto/cavium/zip/zip_main.h b/drivers/crypto/cavium/zip/zip_main.h new file mode 100644 index 000000000..e1e4fa92c --- /dev/null +++ b/drivers/crypto/cavium/zip/zip_main.h @@ -0,0 +1,120 @@ +/***********************license start************************************ + * Copyright (c) 2003-2017 Cavium, Inc. + * All rights reserved. + * + * License: one of 'Cavium License' or 'GNU General Public License Version 2' + * + * This file is provided under the terms of the Cavium License (see below) + * or under the terms of GNU General Public License, Version 2, as + * published by the Free Software Foundation. When using or redistributing + * this file, you may do so under either license. + * + * Cavium License: Redistribution and use in source and binary forms, with + * or without modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * * Neither the name of Cavium Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * This Software, including technical data, may be subject to U.S. export + * control laws, including the U.S. Export Administration Act and its + * associated regulations, and may be subject to export or import + * regulations in other countries. + * + * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" + * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS + * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH + * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY + * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT + * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) + * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A + * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET + * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE + * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES + * WITH YOU. + ***********************license end**************************************/ + +#ifndef __ZIP_MAIN_H__ +#define __ZIP_MAIN_H__ + +#include "zip_device.h" +#include "zip_regs.h" + +/* PCI device IDs */ +#define PCI_DEVICE_ID_THUNDERX_ZIP 0xA01A + +/* ZIP device BARs */ +#define PCI_CFG_ZIP_PF_BAR0 0 /* Base addr for normal regs */ + +/* Maximum available zip queues */ +#define ZIP_MAX_NUM_QUEUES 8 + +#define ZIP_128B_ALIGN 7 + +/* Command queue buffer size */ +#define ZIP_CMD_QBUF_SIZE (8064 + 8) + +struct zip_registers { + char *reg_name; + u64 reg_offset; +}; + +/* ZIP Compression - Decompression stats */ +struct zip_stats { + atomic64_t comp_req_submit; + atomic64_t comp_req_complete; + atomic64_t decomp_req_submit; + atomic64_t decomp_req_complete; + atomic64_t comp_in_bytes; + atomic64_t comp_out_bytes; + atomic64_t decomp_in_bytes; + atomic64_t decomp_out_bytes; + atomic64_t decomp_bad_reqs; +}; + +/* ZIP Instruction Queue */ +struct zip_iq { + u64 *sw_head; + u64 *sw_tail; + u64 *hw_tail; + u64 done_cnt; + u64 pend_cnt; + u64 free_flag; + + /* ZIP IQ lock */ + spinlock_t lock; +}; + +/* ZIP Device */ +struct zip_device { + u32 index; + void __iomem *reg_base; + struct pci_dev *pdev; + + /* Different ZIP Constants */ + u64 depth; + u64 onfsize; + u64 ctxsize; + + struct zip_iq iq[ZIP_MAX_NUM_QUEUES]; + struct zip_stats stats; +}; + +/* Prototypes */ +struct zip_device *zip_get_device(int node_id); +int zip_get_node_id(void); +void zip_reg_write(u64 val, u64 __iomem *addr); +u64 zip_reg_read(u64 __iomem *addr); +void zip_update_cmd_bufs(struct zip_device *zip_dev, u32 queue); +u32 zip_load_instr(union zip_inst_s *instr, struct zip_device *zip_dev); + +#endif /* ZIP_MAIN_H */ diff --git a/drivers/crypto/cavium/zip/zip_mem.c b/drivers/crypto/cavium/zip/zip_mem.c new file mode 100644 index 000000000..b3e0843a9 --- /dev/null +++ b/drivers/crypto/cavium/zip/zip_mem.c @@ -0,0 +1,114 @@ +/***********************license start************************************ + * Copyright (c) 2003-2017 Cavium, Inc. + * All rights reserved. + * + * License: one of 'Cavium License' or 'GNU General Public License Version 2' + * + * This file is provided under the terms of the Cavium License (see below) + * or under the terms of GNU General Public License, Version 2, as + * published by the Free Software Foundation. When using or redistributing + * this file, you may do so under either license. + * + * Cavium License: Redistribution and use in source and binary forms, with + * or without modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * * Neither the name of Cavium Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * This Software, including technical data, may be subject to U.S. export + * control laws, including the U.S. Export Administration Act and its + * associated regulations, and may be subject to export or import + * regulations in other countries. + * + * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" + * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS + * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH + * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY + * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT + * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) + * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A + * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET + * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE + * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES + * WITH YOU. + ***********************license end**************************************/ + +#include <linux/types.h> +#include <linux/vmalloc.h> + +#include "common.h" + +/** + * zip_cmd_qbuf_alloc - Allocates a cmd buffer for ZIP Instruction Queue + * @zip: Pointer to zip device structure + * @q: Queue number to allocate bufffer to + * Return: 0 if successful, -ENOMEM otherwise + */ +int zip_cmd_qbuf_alloc(struct zip_device *zip, int q) +{ + zip->iq[q].sw_head = (u64 *)__get_free_pages((GFP_KERNEL | GFP_DMA), + get_order(ZIP_CMD_QBUF_SIZE)); + + if (!zip->iq[q].sw_head) + return -ENOMEM; + + memset(zip->iq[q].sw_head, 0, ZIP_CMD_QBUF_SIZE); + + zip_dbg("cmd_qbuf_alloc[%d] Success : %p\n", q, zip->iq[q].sw_head); + return 0; +} + +/** + * zip_cmd_qbuf_free - Frees the cmd Queue buffer + * @zip: Pointer to zip device structure + * @q: Queue number to free buffer of + */ +void zip_cmd_qbuf_free(struct zip_device *zip, int q) +{ + zip_dbg("Freeing cmd_qbuf 0x%lx\n", zip->iq[q].sw_tail); + + free_pages((u64)zip->iq[q].sw_tail, get_order(ZIP_CMD_QBUF_SIZE)); +} + +/** + * zip_data_buf_alloc - Allocates memory for a data bufffer + * @size: Size of the buffer to allocate + * Returns: Pointer to the buffer allocated + */ +u8 *zip_data_buf_alloc(u64 size) +{ + u8 *ptr; + + ptr = (u8 *)__get_free_pages((GFP_KERNEL | GFP_DMA), + get_order(size)); + + if (!ptr) + return NULL; + + memset(ptr, 0, size); + + zip_dbg("Data buffer allocation success\n"); + return ptr; +} + +/** + * zip_data_buf_free - Frees the memory of a data buffer + * @ptr: Pointer to the buffer + * @size: Buffer size + */ +void zip_data_buf_free(u8 *ptr, u64 size) +{ + zip_dbg("Freeing data buffer 0x%lx\n", ptr); + + free_pages((u64)ptr, get_order(size)); +} diff --git a/drivers/crypto/cavium/zip/zip_mem.h b/drivers/crypto/cavium/zip/zip_mem.h new file mode 100644 index 000000000..f8f2f08c4 --- /dev/null +++ b/drivers/crypto/cavium/zip/zip_mem.h @@ -0,0 +1,78 @@ +/***********************license start************************************ + * Copyright (c) 2003-2017 Cavium, Inc. + * All rights reserved. + * + * License: one of 'Cavium License' or 'GNU General Public License Version 2' + * + * This file is provided under the terms of the Cavium License (see below) + * or under the terms of GNU General Public License, Version 2, as + * published by the Free Software Foundation. When using or redistributing + * this file, you may do so under either license. + * + * Cavium License: Redistribution and use in source and binary forms, with + * or without modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * * Neither the name of Cavium Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * This Software, including technical data, may be subject to U.S. export + * control laws, including the U.S. Export Administration Act and its + * associated regulations, and may be subject to export or import + * regulations in other countries. + * + * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" + * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS + * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH + * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY + * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT + * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) + * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A + * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET + * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE + * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES + * WITH YOU. + ***********************license end**************************************/ + +#ifndef __ZIP_MEM_H__ +#define __ZIP_MEM_H__ + +/** + * zip_cmd_qbuf_free - Frees the cmd Queue buffer + * @zip: Pointer to zip device structure + * @q: Queue nmber to free buffer of + */ +void zip_cmd_qbuf_free(struct zip_device *zip, int q); + +/** + * zip_cmd_qbuf_alloc - Allocates a Chunk/cmd buffer for ZIP Inst(cmd) Queue + * @zip: Pointer to zip device structure + * @q: Queue number to allocate bufffer to + * Return: 0 if successful, 1 otherwise + */ +int zip_cmd_qbuf_alloc(struct zip_device *zip, int q); + +/** + * zip_data_buf_alloc - Allocates memory for a data bufffer + * @size: Size of the buffer to allocate + * Returns: Pointer to the buffer allocated + */ +u8 *zip_data_buf_alloc(u64 size); + +/** + * zip_data_buf_free - Frees the memory of a data buffer + * @ptr: Pointer to the buffer + * @size: Buffer size + */ +void zip_data_buf_free(u8 *ptr, u64 size); + +#endif diff --git a/drivers/crypto/cavium/zip/zip_regs.h b/drivers/crypto/cavium/zip/zip_regs.h new file mode 100644 index 000000000..874e0236c --- /dev/null +++ b/drivers/crypto/cavium/zip/zip_regs.h @@ -0,0 +1,1347 @@ +/***********************license start************************************ + * Copyright (c) 2003-2017 Cavium, Inc. + * All rights reserved. + * + * License: one of 'Cavium License' or 'GNU General Public License Version 2' + * + * This file is provided under the terms of the Cavium License (see below) + * or under the terms of GNU General Public License, Version 2, as + * published by the Free Software Foundation. When using or redistributing + * this file, you may do so under either license. + * + * Cavium License: Redistribution and use in source and binary forms, with + * or without modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * * Neither the name of Cavium Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * This Software, including technical data, may be subject to U.S. export + * control laws, including the U.S. Export Administration Act and its + * associated regulations, and may be subject to export or import + * regulations in other countries. + * + * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" + * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS + * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH + * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY + * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT + * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) + * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A + * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET + * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE + * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES + * WITH YOU. + ***********************license end**************************************/ + +#ifndef __ZIP_REGS_H__ +#define __ZIP_REGS_H__ + +/* + * Configuration and status register (CSR) address and type definitions for + * Cavium ZIP. + */ + +#include <linux/kern_levels.h> + +/* ZIP invocation result completion status codes */ +#define ZIP_CMD_NOTDONE 0x0 + +/* Successful completion. */ +#define ZIP_CMD_SUCCESS 0x1 + +/* Output truncated */ +#define ZIP_CMD_DTRUNC 0x2 + +/* Dynamic Stop */ +#define ZIP_CMD_DYNAMIC_STOP 0x3 + +/* Uncompress ran out of input data when IWORD0[EF] was set */ +#define ZIP_CMD_ITRUNC 0x4 + +/* Uncompress found the reserved block type 3 */ +#define ZIP_CMD_RBLOCK 0x5 + +/* + * Uncompress found LEN != ZIP_CMD_NLEN in an uncompressed block in the input. + */ +#define ZIP_CMD_NLEN 0x6 + +/* Uncompress found a bad code in the main Huffman codes. */ +#define ZIP_CMD_BADCODE 0x7 + +/* Uncompress found a bad code in the 19 Huffman codes encoding lengths. */ +#define ZIP_CMD_BADCODE2 0x8 + +/* Compress found a zero-length input. */ +#define ZIP_CMD_ZERO_LEN 0x9 + +/* The compress or decompress encountered an internal parity error. */ +#define ZIP_CMD_PARITY 0xA + +/* + * Uncompress found a string identifier that precedes the uncompressed data and + * decompression history. + */ +#define ZIP_CMD_FATAL 0xB + +/** + * enum zip_int_vec_e - ZIP MSI-X Vector Enumeration, enumerates the MSI-X + * interrupt vectors. + */ +enum zip_int_vec_e { + ZIP_INT_VEC_E_ECCE = 0x10, + ZIP_INT_VEC_E_FIFE = 0x11, + ZIP_INT_VEC_E_QUE0_DONE = 0x0, + ZIP_INT_VEC_E_QUE0_ERR = 0x8, + ZIP_INT_VEC_E_QUE1_DONE = 0x1, + ZIP_INT_VEC_E_QUE1_ERR = 0x9, + ZIP_INT_VEC_E_QUE2_DONE = 0x2, + ZIP_INT_VEC_E_QUE2_ERR = 0xa, + ZIP_INT_VEC_E_QUE3_DONE = 0x3, + ZIP_INT_VEC_E_QUE3_ERR = 0xb, + ZIP_INT_VEC_E_QUE4_DONE = 0x4, + ZIP_INT_VEC_E_QUE4_ERR = 0xc, + ZIP_INT_VEC_E_QUE5_DONE = 0x5, + ZIP_INT_VEC_E_QUE5_ERR = 0xd, + ZIP_INT_VEC_E_QUE6_DONE = 0x6, + ZIP_INT_VEC_E_QUE6_ERR = 0xe, + ZIP_INT_VEC_E_QUE7_DONE = 0x7, + ZIP_INT_VEC_E_QUE7_ERR = 0xf, + ZIP_INT_VEC_E_ENUM_LAST = 0x12, +}; + +/** + * union zip_zptr_addr_s - ZIP Generic Pointer Structure for ADDR. + * + * It is the generic format of pointers in ZIP_INST_S. + */ +union zip_zptr_addr_s { + u64 u_reg64; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_49_63 : 15; + u64 addr : 49; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 addr : 49; + u64 reserved_49_63 : 15; +#endif + } s; + +}; + +/** + * union zip_zptr_ctl_s - ZIP Generic Pointer Structure for CTL. + * + * It is the generic format of pointers in ZIP_INST_S. + */ +union zip_zptr_ctl_s { + u64 u_reg64; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_112_127 : 16; + u64 length : 16; + u64 reserved_67_95 : 29; + u64 fw : 1; + u64 nc : 1; + u64 data_be : 1; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 data_be : 1; + u64 nc : 1; + u64 fw : 1; + u64 reserved_67_95 : 29; + u64 length : 16; + u64 reserved_112_127 : 16; +#endif + } s; +}; + +/** + * union zip_inst_s - ZIP Instruction Structure. + * Each ZIP instruction has 16 words (they are called IWORD0 to IWORD15 within + * the structure). + */ +union zip_inst_s { + u64 u_reg64[16]; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 doneint : 1; + u64 reserved_56_62 : 7; + u64 totaloutputlength : 24; + u64 reserved_27_31 : 5; + u64 exn : 3; + u64 reserved_23_23 : 1; + u64 exbits : 7; + u64 reserved_12_15 : 4; + u64 sf : 1; + u64 ss : 2; + u64 cc : 2; + u64 ef : 1; + u64 bf : 1; + u64 ce : 1; + u64 reserved_3_3 : 1; + u64 ds : 1; + u64 dg : 1; + u64 hg : 1; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 hg : 1; + u64 dg : 1; + u64 ds : 1; + u64 reserved_3_3 : 1; + u64 ce : 1; + u64 bf : 1; + u64 ef : 1; + u64 cc : 2; + u64 ss : 2; + u64 sf : 1; + u64 reserved_12_15 : 4; + u64 exbits : 7; + u64 reserved_23_23 : 1; + u64 exn : 3; + u64 reserved_27_31 : 5; + u64 totaloutputlength : 24; + u64 reserved_56_62 : 7; + u64 doneint : 1; +#endif +#if defined(__BIG_ENDIAN_BITFIELD) + u64 historylength : 16; + u64 reserved_96_111 : 16; + u64 adlercrc32 : 32; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 adlercrc32 : 32; + u64 reserved_96_111 : 16; + u64 historylength : 16; +#endif + union zip_zptr_addr_s ctx_ptr_addr; + union zip_zptr_ctl_s ctx_ptr_ctl; + union zip_zptr_addr_s his_ptr_addr; + union zip_zptr_ctl_s his_ptr_ctl; + union zip_zptr_addr_s inp_ptr_addr; + union zip_zptr_ctl_s inp_ptr_ctl; + union zip_zptr_addr_s out_ptr_addr; + union zip_zptr_ctl_s out_ptr_ctl; + union zip_zptr_addr_s res_ptr_addr; + union zip_zptr_ctl_s res_ptr_ctl; +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_817_831 : 15; + u64 wq_ptr : 49; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 wq_ptr : 49; + u64 reserved_817_831 : 15; +#endif +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_882_895 : 14; + u64 tt : 2; + u64 reserved_874_879 : 6; + u64 grp : 10; + u64 tag : 32; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 tag : 32; + u64 grp : 10; + u64 reserved_874_879 : 6; + u64 tt : 2; + u64 reserved_882_895 : 14; +#endif +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_896_959 : 64; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 reserved_896_959 : 64; +#endif +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_960_1023 : 64; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 reserved_960_1023 : 64; +#endif + } s; +}; + +/** + * union zip_nptr_s - ZIP Instruction Next-Chunk-Buffer Pointer (NPTR) + * Structure + * + * ZIP_NPTR structure is used to chain all the zip instruction buffers + * together. ZIP instruction buffers are managed (allocated and released) by + * the software. + */ +union zip_nptr_s { + u64 u_reg64; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_49_63 : 15; + u64 addr : 49; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 addr : 49; + u64 reserved_49_63 : 15; +#endif + } s; +}; + +/** + * union zip_zptr_s - ZIP Generic Pointer Structure. + * + * It is the generic format of pointers in ZIP_INST_S. + */ +union zip_zptr_s { + u64 u_reg64[2]; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_49_63 : 15; + u64 addr : 49; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 addr : 49; + u64 reserved_49_63 : 15; +#endif +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_112_127 : 16; + u64 length : 16; + u64 reserved_67_95 : 29; + u64 fw : 1; + u64 nc : 1; + u64 data_be : 1; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 data_be : 1; + u64 nc : 1; + u64 fw : 1; + u64 reserved_67_95 : 29; + u64 length : 16; + u64 reserved_112_127 : 16; +#endif + } s; +}; + +/** + * union zip_zres_s - ZIP Result Structure + * + * The ZIP coprocessor writes the result structure after it completes the + * invocation. The result structure is exactly 24 bytes, and each invocation of + * the ZIP coprocessor produces exactly one result structure. + */ +union zip_zres_s { + u64 u_reg64[3]; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 crc32 : 32; + u64 adler32 : 32; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 adler32 : 32; + u64 crc32 : 32; +#endif +#if defined(__BIG_ENDIAN_BITFIELD) + u64 totalbyteswritten : 32; + u64 totalbytesread : 32; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 totalbytesread : 32; + u64 totalbyteswritten : 32; +#endif +#if defined(__BIG_ENDIAN_BITFIELD) + u64 totalbitsprocessed : 32; + u64 doneint : 1; + u64 reserved_155_158 : 4; + u64 exn : 3; + u64 reserved_151_151 : 1; + u64 exbits : 7; + u64 reserved_137_143 : 7; + u64 ef : 1; + + volatile u64 compcode : 8; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + + volatile u64 compcode : 8; + u64 ef : 1; + u64 reserved_137_143 : 7; + u64 exbits : 7; + u64 reserved_151_151 : 1; + u64 exn : 3; + u64 reserved_155_158 : 4; + u64 doneint : 1; + u64 totalbitsprocessed : 32; +#endif + } s; +}; + +/** + * union zip_cmd_ctl - Structure representing the register that controls + * clock and reset. + */ +union zip_cmd_ctl { + u64 u_reg64; + struct zip_cmd_ctl_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_2_63 : 62; + u64 forceclk : 1; + u64 reset : 1; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 reset : 1; + u64 forceclk : 1; + u64 reserved_2_63 : 62; +#endif + } s; +}; + +#define ZIP_CMD_CTL 0x0ull + +/** + * union zip_constants - Data structure representing the register that contains + * all of the current implementation-related parameters of the zip core in this + * chip. + */ +union zip_constants { + u64 u_reg64; + struct zip_constants_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 nexec : 8; + u64 reserved_49_55 : 7; + u64 syncflush_capable : 1; + u64 depth : 16; + u64 onfsize : 12; + u64 ctxsize : 12; + u64 reserved_1_7 : 7; + u64 disabled : 1; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 disabled : 1; + u64 reserved_1_7 : 7; + u64 ctxsize : 12; + u64 onfsize : 12; + u64 depth : 16; + u64 syncflush_capable : 1; + u64 reserved_49_55 : 7; + u64 nexec : 8; +#endif + } s; +}; + +#define ZIP_CONSTANTS 0x00A0ull + +/** + * union zip_corex_bist_status - Represents registers which have the BIST + * status of memories in zip cores. + * + * Each bit is the BIST result of an individual memory + * (per bit, 0 = pass and 1 = fail). + */ +union zip_corex_bist_status { + u64 u_reg64; + struct zip_corex_bist_status_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_53_63 : 11; + u64 bstatus : 53; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 bstatus : 53; + u64 reserved_53_63 : 11; +#endif + } s; +}; + +static inline u64 ZIP_COREX_BIST_STATUS(u64 param1) +{ + if (param1 <= 1) + return 0x0520ull + (param1 & 1) * 0x8ull; + pr_err("ZIP_COREX_BIST_STATUS: %llu\n", param1); + return 0; +} + +/** + * union zip_ctl_bist_status - Represents register that has the BIST status of + * memories in ZIP_CTL (instruction buffer, G/S pointer FIFO, input data + * buffer, output data buffers). + * + * Each bit is the BIST result of an individual memory + * (per bit, 0 = pass and 1 = fail). + */ +union zip_ctl_bist_status { + u64 u_reg64; + struct zip_ctl_bist_status_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_9_63 : 55; + u64 bstatus : 9; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 bstatus : 9; + u64 reserved_9_63 : 55; +#endif + } s; +}; + +#define ZIP_CTL_BIST_STATUS 0x0510ull + +/** + * union zip_ctl_cfg - Represents the register that controls the behavior of + * the ZIP DMA engines. + * + * It is recommended to keep default values for normal operation. Changing the + * values of the fields may be useful for diagnostics. + */ +union zip_ctl_cfg { + u64 u_reg64; + struct zip_ctl_cfg_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_52_63 : 12; + u64 ildf : 4; + u64 reserved_36_47 : 12; + u64 drtf : 4; + u64 reserved_27_31 : 5; + u64 stcf : 3; + u64 reserved_19_23 : 5; + u64 ldf : 3; + u64 reserved_2_15 : 14; + u64 busy : 1; + u64 reserved_0_0 : 1; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 reserved_0_0 : 1; + u64 busy : 1; + u64 reserved_2_15 : 14; + u64 ldf : 3; + u64 reserved_19_23 : 5; + u64 stcf : 3; + u64 reserved_27_31 : 5; + u64 drtf : 4; + u64 reserved_36_47 : 12; + u64 ildf : 4; + u64 reserved_52_63 : 12; +#endif + } s; +}; + +#define ZIP_CTL_CFG 0x0560ull + +/** + * union zip_dbg_corex_inst - Represents the registers that reflect the status + * of the current instruction that the ZIP core is executing or has executed. + * + * These registers are only for debug use. + */ +union zip_dbg_corex_inst { + u64 u_reg64; + struct zip_dbg_corex_inst_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 busy : 1; + u64 reserved_35_62 : 28; + u64 qid : 3; + u64 iid : 32; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 iid : 32; + u64 qid : 3; + u64 reserved_35_62 : 28; + u64 busy : 1; +#endif + } s; +}; + +static inline u64 ZIP_DBG_COREX_INST(u64 param1) +{ + if (param1 <= 1) + return 0x0640ull + (param1 & 1) * 0x8ull; + pr_err("ZIP_DBG_COREX_INST: %llu\n", param1); + return 0; +} + +/** + * union zip_dbg_corex_sta - Represents registers that reflect the status of + * the zip cores. + * + * They are for debug use only. + */ +union zip_dbg_corex_sta { + u64 u_reg64; + struct zip_dbg_corex_sta_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 busy : 1; + u64 reserved_37_62 : 26; + u64 ist : 5; + u64 nie : 32; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 nie : 32; + u64 ist : 5; + u64 reserved_37_62 : 26; + u64 busy : 1; +#endif + } s; +}; + +static inline u64 ZIP_DBG_COREX_STA(u64 param1) +{ + if (param1 <= 1) + return 0x0680ull + (param1 & 1) * 0x8ull; + pr_err("ZIP_DBG_COREX_STA: %llu\n", param1); + return 0; +} + +/** + * union zip_dbg_quex_sta - Represets registers that reflect status of the zip + * instruction queues. + * + * They are for debug use only. + */ +union zip_dbg_quex_sta { + u64 u_reg64; + struct zip_dbg_quex_sta_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 busy : 1; + u64 reserved_56_62 : 7; + u64 rqwc : 24; + u64 nii : 32; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 nii : 32; + u64 rqwc : 24; + u64 reserved_56_62 : 7; + u64 busy : 1; +#endif + } s; +}; + +static inline u64 ZIP_DBG_QUEX_STA(u64 param1) +{ + if (param1 <= 7) + return 0x1800ull + (param1 & 7) * 0x8ull; + pr_err("ZIP_DBG_QUEX_STA: %llu\n", param1); + return 0; +} + +/** + * union zip_ecc_ctl - Represents the register that enables ECC for each + * individual internal memory that requires ECC. + * + * For debug purpose, it can also flip one or two bits in the ECC data. + */ +union zip_ecc_ctl { + u64 u_reg64; + struct zip_ecc_ctl_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_19_63 : 45; + u64 vmem_cdis : 1; + u64 vmem_fs : 2; + u64 reserved_15_15 : 1; + u64 idf1_cdis : 1; + u64 idf1_fs : 2; + u64 reserved_11_11 : 1; + u64 idf0_cdis : 1; + u64 idf0_fs : 2; + u64 reserved_7_7 : 1; + u64 gspf_cdis : 1; + u64 gspf_fs : 2; + u64 reserved_3_3 : 1; + u64 iqf_cdis : 1; + u64 iqf_fs : 2; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 iqf_fs : 2; + u64 iqf_cdis : 1; + u64 reserved_3_3 : 1; + u64 gspf_fs : 2; + u64 gspf_cdis : 1; + u64 reserved_7_7 : 1; + u64 idf0_fs : 2; + u64 idf0_cdis : 1; + u64 reserved_11_11 : 1; + u64 idf1_fs : 2; + u64 idf1_cdis : 1; + u64 reserved_15_15 : 1; + u64 vmem_fs : 2; + u64 vmem_cdis : 1; + u64 reserved_19_63 : 45; +#endif + } s; +}; + +#define ZIP_ECC_CTL 0x0568ull + +/* NCB - zip_ecce_ena_w1c */ +union zip_ecce_ena_w1c { + u64 u_reg64; + struct zip_ecce_ena_w1c_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_37_63 : 27; + u64 dbe : 5; + u64 reserved_5_31 : 27; + u64 sbe : 5; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 sbe : 5; + u64 reserved_5_31 : 27; + u64 dbe : 5; + u64 reserved_37_63 : 27; +#endif + } s; +}; + +#define ZIP_ECCE_ENA_W1C 0x0598ull + +/* NCB - zip_ecce_ena_w1s */ +union zip_ecce_ena_w1s { + u64 u_reg64; + struct zip_ecce_ena_w1s_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_37_63 : 27; + u64 dbe : 5; + u64 reserved_5_31 : 27; + u64 sbe : 5; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 sbe : 5; + u64 reserved_5_31 : 27; + u64 dbe : 5; + u64 reserved_37_63 : 27; +#endif + } s; +}; + +#define ZIP_ECCE_ENA_W1S 0x0590ull + +/** + * union zip_ecce_int - Represents the register that contains the status of the + * ECC interrupt sources. + */ +union zip_ecce_int { + u64 u_reg64; + struct zip_ecce_int_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_37_63 : 27; + u64 dbe : 5; + u64 reserved_5_31 : 27; + u64 sbe : 5; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 sbe : 5; + u64 reserved_5_31 : 27; + u64 dbe : 5; + u64 reserved_37_63 : 27; +#endif + } s; +}; + +#define ZIP_ECCE_INT 0x0580ull + +/* NCB - zip_ecce_int_w1s */ +union zip_ecce_int_w1s { + u64 u_reg64; + struct zip_ecce_int_w1s_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_37_63 : 27; + u64 dbe : 5; + u64 reserved_5_31 : 27; + u64 sbe : 5; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 sbe : 5; + u64 reserved_5_31 : 27; + u64 dbe : 5; + u64 reserved_37_63 : 27; +#endif + } s; +}; + +#define ZIP_ECCE_INT_W1S 0x0588ull + +/* NCB - zip_fife_ena_w1c */ +union zip_fife_ena_w1c { + u64 u_reg64; + struct zip_fife_ena_w1c_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_42_63 : 22; + u64 asserts : 42; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 asserts : 42; + u64 reserved_42_63 : 22; +#endif + } s; +}; + +#define ZIP_FIFE_ENA_W1C 0x0090ull + +/* NCB - zip_fife_ena_w1s */ +union zip_fife_ena_w1s { + u64 u_reg64; + struct zip_fife_ena_w1s_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_42_63 : 22; + u64 asserts : 42; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 asserts : 42; + u64 reserved_42_63 : 22; +#endif + } s; +}; + +#define ZIP_FIFE_ENA_W1S 0x0088ull + +/* NCB - zip_fife_int */ +union zip_fife_int { + u64 u_reg64; + struct zip_fife_int_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_42_63 : 22; + u64 asserts : 42; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 asserts : 42; + u64 reserved_42_63 : 22; +#endif + } s; +}; + +#define ZIP_FIFE_INT 0x0078ull + +/* NCB - zip_fife_int_w1s */ +union zip_fife_int_w1s { + u64 u_reg64; + struct zip_fife_int_w1s_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_42_63 : 22; + u64 asserts : 42; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 asserts : 42; + u64 reserved_42_63 : 22; +#endif + } s; +}; + +#define ZIP_FIFE_INT_W1S 0x0080ull + +/** + * union zip_msix_pbax - Represents the register that is the MSI-X PBA table + * + * The bit number is indexed by the ZIP_INT_VEC_E enumeration. + */ +union zip_msix_pbax { + u64 u_reg64; + struct zip_msix_pbax_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 pend : 64; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 pend : 64; +#endif + } s; +}; + +static inline u64 ZIP_MSIX_PBAX(u64 param1) +{ + if (param1 == 0) + return 0x0000838000FF0000ull; + pr_err("ZIP_MSIX_PBAX: %llu\n", param1); + return 0; +} + +/** + * union zip_msix_vecx_addr - Represents the register that is the MSI-X vector + * table, indexed by the ZIP_INT_VEC_E enumeration. + */ +union zip_msix_vecx_addr { + u64 u_reg64; + struct zip_msix_vecx_addr_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_49_63 : 15; + u64 addr : 47; + u64 reserved_1_1 : 1; + u64 secvec : 1; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 secvec : 1; + u64 reserved_1_1 : 1; + u64 addr : 47; + u64 reserved_49_63 : 15; +#endif + } s; +}; + +static inline u64 ZIP_MSIX_VECX_ADDR(u64 param1) +{ + if (param1 <= 17) + return 0x0000838000F00000ull + (param1 & 31) * 0x10ull; + pr_err("ZIP_MSIX_VECX_ADDR: %llu\n", param1); + return 0; +} + +/** + * union zip_msix_vecx_ctl - Represents the register that is the MSI-X vector + * table, indexed by the ZIP_INT_VEC_E enumeration. + */ +union zip_msix_vecx_ctl { + u64 u_reg64; + struct zip_msix_vecx_ctl_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_33_63 : 31; + u64 mask : 1; + u64 reserved_20_31 : 12; + u64 data : 20; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 data : 20; + u64 reserved_20_31 : 12; + u64 mask : 1; + u64 reserved_33_63 : 31; +#endif + } s; +}; + +static inline u64 ZIP_MSIX_VECX_CTL(u64 param1) +{ + if (param1 <= 17) + return 0x0000838000F00008ull + (param1 & 31) * 0x10ull; + pr_err("ZIP_MSIX_VECX_CTL: %llu\n", param1); + return 0; +} + +/** + * union zip_quex_done - Represents the registers that contain the per-queue + * instruction done count. + */ +union zip_quex_done { + u64 u_reg64; + struct zip_quex_done_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_20_63 : 44; + u64 done : 20; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 done : 20; + u64 reserved_20_63 : 44; +#endif + } s; +}; + +static inline u64 ZIP_QUEX_DONE(u64 param1) +{ + if (param1 <= 7) + return 0x2000ull + (param1 & 7) * 0x8ull; + pr_err("ZIP_QUEX_DONE: %llu\n", param1); + return 0; +} + +/** + * union zip_quex_done_ack - Represents the registers on write to which will + * decrement the per-queue instructiona done count. + */ +union zip_quex_done_ack { + u64 u_reg64; + struct zip_quex_done_ack_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_20_63 : 44; + u64 done_ack : 20; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 done_ack : 20; + u64 reserved_20_63 : 44; +#endif + } s; +}; + +static inline u64 ZIP_QUEX_DONE_ACK(u64 param1) +{ + if (param1 <= 7) + return 0x2200ull + (param1 & 7) * 0x8ull; + pr_err("ZIP_QUEX_DONE_ACK: %llu\n", param1); + return 0; +} + +/** + * union zip_quex_done_ena_w1c - Represents the register which when written + * 1 to will disable the DONEINT interrupt for the queue. + */ +union zip_quex_done_ena_w1c { + u64 u_reg64; + struct zip_quex_done_ena_w1c_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_1_63 : 63; + u64 done_ena : 1; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 done_ena : 1; + u64 reserved_1_63 : 63; +#endif + } s; +}; + +static inline u64 ZIP_QUEX_DONE_ENA_W1C(u64 param1) +{ + if (param1 <= 7) + return 0x2600ull + (param1 & 7) * 0x8ull; + pr_err("ZIP_QUEX_DONE_ENA_W1C: %llu\n", param1); + return 0; +} + +/** + * union zip_quex_done_ena_w1s - Represents the register that when written 1 to + * will enable the DONEINT interrupt for the queue. + */ +union zip_quex_done_ena_w1s { + u64 u_reg64; + struct zip_quex_done_ena_w1s_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_1_63 : 63; + u64 done_ena : 1; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 done_ena : 1; + u64 reserved_1_63 : 63; +#endif + } s; +}; + +static inline u64 ZIP_QUEX_DONE_ENA_W1S(u64 param1) +{ + if (param1 <= 7) + return 0x2400ull + (param1 & 7) * 0x8ull; + pr_err("ZIP_QUEX_DONE_ENA_W1S: %llu\n", param1); + return 0; +} + +/** + * union zip_quex_done_wait - Represents the register that specifies the per + * queue interrupt coalescing settings. + */ +union zip_quex_done_wait { + u64 u_reg64; + struct zip_quex_done_wait_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_48_63 : 16; + u64 time_wait : 16; + u64 reserved_20_31 : 12; + u64 num_wait : 20; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 num_wait : 20; + u64 reserved_20_31 : 12; + u64 time_wait : 16; + u64 reserved_48_63 : 16; +#endif + } s; +}; + +static inline u64 ZIP_QUEX_DONE_WAIT(u64 param1) +{ + if (param1 <= 7) + return 0x2800ull + (param1 & 7) * 0x8ull; + pr_err("ZIP_QUEX_DONE_WAIT: %llu\n", param1); + return 0; +} + +/** + * union zip_quex_doorbell - Represents doorbell registers for the ZIP + * instruction queues. + */ +union zip_quex_doorbell { + u64 u_reg64; + struct zip_quex_doorbell_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_20_63 : 44; + u64 dbell_cnt : 20; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 dbell_cnt : 20; + u64 reserved_20_63 : 44; +#endif + } s; +}; + +static inline u64 ZIP_QUEX_DOORBELL(u64 param1) +{ + if (param1 <= 7) + return 0x4000ull + (param1 & 7) * 0x8ull; + pr_err("ZIP_QUEX_DOORBELL: %llu\n", param1); + return 0; +} + +union zip_quex_err_ena_w1c { + u64 u_reg64; + struct zip_quex_err_ena_w1c_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_5_63 : 59; + u64 mdbe : 1; + u64 nwrp : 1; + u64 nrrp : 1; + u64 irde : 1; + u64 dovf : 1; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 dovf : 1; + u64 irde : 1; + u64 nrrp : 1; + u64 nwrp : 1; + u64 mdbe : 1; + u64 reserved_5_63 : 59; +#endif + } s; +}; + +static inline u64 ZIP_QUEX_ERR_ENA_W1C(u64 param1) +{ + if (param1 <= 7) + return 0x3600ull + (param1 & 7) * 0x8ull; + pr_err("ZIP_QUEX_ERR_ENA_W1C: %llu\n", param1); + return 0; +} + +union zip_quex_err_ena_w1s { + u64 u_reg64; + struct zip_quex_err_ena_w1s_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_5_63 : 59; + u64 mdbe : 1; + u64 nwrp : 1; + u64 nrrp : 1; + u64 irde : 1; + u64 dovf : 1; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 dovf : 1; + u64 irde : 1; + u64 nrrp : 1; + u64 nwrp : 1; + u64 mdbe : 1; + u64 reserved_5_63 : 59; +#endif + } s; +}; + +static inline u64 ZIP_QUEX_ERR_ENA_W1S(u64 param1) +{ + if (param1 <= 7) + return 0x3400ull + (param1 & 7) * 0x8ull; + pr_err("ZIP_QUEX_ERR_ENA_W1S: %llu\n", param1); + return 0; +} + +/** + * union zip_quex_err_int - Represents registers that contain the per-queue + * error interrupts. + */ +union zip_quex_err_int { + u64 u_reg64; + struct zip_quex_err_int_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_5_63 : 59; + u64 mdbe : 1; + u64 nwrp : 1; + u64 nrrp : 1; + u64 irde : 1; + u64 dovf : 1; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 dovf : 1; + u64 irde : 1; + u64 nrrp : 1; + u64 nwrp : 1; + u64 mdbe : 1; + u64 reserved_5_63 : 59; +#endif + } s; +}; + +static inline u64 ZIP_QUEX_ERR_INT(u64 param1) +{ + if (param1 <= 7) + return 0x3000ull + (param1 & 7) * 0x8ull; + pr_err("ZIP_QUEX_ERR_INT: %llu\n", param1); + return 0; +} + +/* NCB - zip_que#_err_int_w1s */ +union zip_quex_err_int_w1s { + u64 u_reg64; + struct zip_quex_err_int_w1s_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_5_63 : 59; + u64 mdbe : 1; + u64 nwrp : 1; + u64 nrrp : 1; + u64 irde : 1; + u64 dovf : 1; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 dovf : 1; + u64 irde : 1; + u64 nrrp : 1; + u64 nwrp : 1; + u64 mdbe : 1; + u64 reserved_5_63 : 59; +#endif + } s; +}; + +static inline u64 ZIP_QUEX_ERR_INT_W1S(u64 param1) +{ + if (param1 <= 7) + return 0x3200ull + (param1 & 7) * 0x8ull; + pr_err("ZIP_QUEX_ERR_INT_W1S: %llu\n", param1); + return 0; +} + +/** + * union zip_quex_gcfg - Represents the registers that reflect status of the + * zip instruction queues,debug use only. + */ +union zip_quex_gcfg { + u64 u_reg64; + struct zip_quex_gcfg_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_4_63 : 60; + u64 iqb_ldwb : 1; + u64 cbw_sty : 1; + u64 l2ld_cmd : 2; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 l2ld_cmd : 2; + u64 cbw_sty : 1; + u64 iqb_ldwb : 1; + u64 reserved_4_63 : 60; +#endif + } s; +}; + +static inline u64 ZIP_QUEX_GCFG(u64 param1) +{ + if (param1 <= 7) + return 0x1A00ull + (param1 & 7) * 0x8ull; + pr_err("ZIP_QUEX_GCFG: %llu\n", param1); + return 0; +} + +/** + * union zip_quex_map - Represents the registers that control how each + * instruction queue maps to zip cores. + */ +union zip_quex_map { + u64 u_reg64; + struct zip_quex_map_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_2_63 : 62; + u64 zce : 2; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 zce : 2; + u64 reserved_2_63 : 62; +#endif + } s; +}; + +static inline u64 ZIP_QUEX_MAP(u64 param1) +{ + if (param1 <= 7) + return 0x1400ull + (param1 & 7) * 0x8ull; + pr_err("ZIP_QUEX_MAP: %llu\n", param1); + return 0; +} + +/** + * union zip_quex_sbuf_addr - Represents the registers that set the buffer + * parameters for the instruction queues. + * + * When quiescent (i.e. outstanding doorbell count is 0), it is safe to rewrite + * this register to effectively reset the command buffer state machine. + * These registers must be programmed after SW programs the corresponding + * ZIP_QUE(0..7)_SBUF_CTL. + */ +union zip_quex_sbuf_addr { + u64 u_reg64; + struct zip_quex_sbuf_addr_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_49_63 : 15; + u64 ptr : 42; + u64 off : 7; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 off : 7; + u64 ptr : 42; + u64 reserved_49_63 : 15; +#endif + } s; +}; + +static inline u64 ZIP_QUEX_SBUF_ADDR(u64 param1) +{ + if (param1 <= 7) + return 0x1000ull + (param1 & 7) * 0x8ull; + pr_err("ZIP_QUEX_SBUF_ADDR: %llu\n", param1); + return 0; +} + +/** + * union zip_quex_sbuf_ctl - Represents the registers that set the buffer + * parameters for the instruction queues. + * + * When quiescent (i.e. outstanding doorbell count is 0), it is safe to rewrite + * this register to effectively reset the command buffer state machine. + * These registers must be programmed before SW programs the corresponding + * ZIP_QUE(0..7)_SBUF_ADDR. + */ +union zip_quex_sbuf_ctl { + u64 u_reg64; + struct zip_quex_sbuf_ctl_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_45_63 : 19; + u64 size : 13; + u64 inst_be : 1; + u64 reserved_24_30 : 7; + u64 stream_id : 8; + u64 reserved_12_15 : 4; + u64 aura : 12; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 aura : 12; + u64 reserved_12_15 : 4; + u64 stream_id : 8; + u64 reserved_24_30 : 7; + u64 inst_be : 1; + u64 size : 13; + u64 reserved_45_63 : 19; +#endif + } s; +}; + +static inline u64 ZIP_QUEX_SBUF_CTL(u64 param1) +{ + if (param1 <= 7) + return 0x1200ull + (param1 & 7) * 0x8ull; + pr_err("ZIP_QUEX_SBUF_CTL: %llu\n", param1); + return 0; +} + +/** + * union zip_que_ena - Represents queue enable register + * + * If a queue is disabled, ZIP_CTL stops fetching instructions from the queue. + */ +union zip_que_ena { + u64 u_reg64; + struct zip_que_ena_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_8_63 : 56; + u64 ena : 8; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 ena : 8; + u64 reserved_8_63 : 56; +#endif + } s; +}; + +#define ZIP_QUE_ENA 0x0500ull + +/** + * union zip_que_pri - Represents the register that defines the priority + * between instruction queues. + */ +union zip_que_pri { + u64 u_reg64; + struct zip_que_pri_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_8_63 : 56; + u64 pri : 8; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 pri : 8; + u64 reserved_8_63 : 56; +#endif + } s; +}; + +#define ZIP_QUE_PRI 0x0508ull + +/** + * union zip_throttle - Represents the register that controls the maximum + * number of in-flight X2I data fetch transactions. + * + * Writing 0 to this register causes the ZIP module to temporarily suspend NCB + * accesses; it is not recommended for normal operation, but may be useful for + * diagnostics. + */ +union zip_throttle { + u64 u_reg64; + struct zip_throttle_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_6_63 : 58; + u64 ld_infl : 6; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 ld_infl : 6; + u64 reserved_6_63 : 58; +#endif + } s; +}; + +#define ZIP_THROTTLE 0x0010ull + +#endif /* _CSRS_ZIP__ */ |