From 483eb2f56657e8e7f419ab1a4fab8dce9ade8609 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sat, 27 Apr 2024 20:24:20 +0200 Subject: Adding upstream version 14.2.21. Signed-off-by: Daniel Baumann --- src/spdk/dpdk/drivers/compress/octeontx/Makefile | 30 + .../drivers/compress/octeontx/include/zip_regs.h | 711 +++++++++++++++++++++ .../dpdk/drivers/compress/octeontx/meson.build | 9 + src/spdk/dpdk/drivers/compress/octeontx/otx_zip.c | 180 ++++++ src/spdk/dpdk/drivers/compress/octeontx/otx_zip.h | 277 ++++++++ .../dpdk/drivers/compress/octeontx/otx_zip_pmd.c | 658 +++++++++++++++++++ .../octeontx/rte_pmd_octeontx_compress_version.map | 3 + 7 files changed, 1868 insertions(+) create mode 100644 src/spdk/dpdk/drivers/compress/octeontx/Makefile create mode 100644 src/spdk/dpdk/drivers/compress/octeontx/include/zip_regs.h create mode 100644 src/spdk/dpdk/drivers/compress/octeontx/meson.build create mode 100644 src/spdk/dpdk/drivers/compress/octeontx/otx_zip.c create mode 100644 src/spdk/dpdk/drivers/compress/octeontx/otx_zip.h create mode 100644 src/spdk/dpdk/drivers/compress/octeontx/otx_zip_pmd.c create mode 100644 src/spdk/dpdk/drivers/compress/octeontx/rte_pmd_octeontx_compress_version.map (limited to 'src/spdk/dpdk/drivers/compress/octeontx') diff --git a/src/spdk/dpdk/drivers/compress/octeontx/Makefile b/src/spdk/dpdk/drivers/compress/octeontx/Makefile new file mode 100644 index 00000000..f34424c8 --- /dev/null +++ b/src/spdk/dpdk/drivers/compress/octeontx/Makefile @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2018 Cavium, Inc + +include $(RTE_SDK)/mk/rte.vars.mk + +# library name +LIB = librte_pmd_octeontx_zip.a + +# library version +LIBABIVER := 1 + +# build flags +CFLAGS += $(WERROR_FLAGS) +CFLAGS += -O3 +CFLAGS += -DALLOW_EXPERIMENTAL_API +CFLAGS += -I$(RTE_SDK)/drivers/compress/octeontx/include + +# external library include paths +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_compressdev +LDLIBS += -lrte_pci -lrte_bus_pci + +# library source files +SRCS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_ZIPVF) += otx_zip_pmd.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_ZIPVF) += otx_zip.c + +# versioning export map +EXPORT_MAP := rte_pmd_octeontx_compress_version.map + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/src/spdk/dpdk/drivers/compress/octeontx/include/zip_regs.h b/src/spdk/dpdk/drivers/compress/octeontx/include/zip_regs.h new file mode 100644 index 00000000..1e74db43 --- /dev/null +++ b/src/spdk/dpdk/drivers/compress/octeontx/include/zip_regs.h @@ -0,0 +1,711 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 Cavium, Inc + */ + +#ifndef _RTE_OCTEONTX_ZIP_REGS_H_ +#define _RTE_OCTEONTX_ZIP_REGS_H_ + + +/** + * Enumeration zip_cc + * + * ZIP compression coding Enumeration + * Enumerates ZIP_INST_S[CC]. + */ +enum { + ZIP_CC_DEFAULT = 0, + ZIP_CC_DYN_HUFF, + ZIP_CC_FIXED_HUFF, + ZIP_CC_LZS +} zip_cc; + +/** + * Register (NCB) zip_vq#_ena + * + * ZIP VF Queue Enable Register + * If a queue is disabled, ZIP CTL stops fetching instructions from the queue. + */ +typedef union { + uint64_t u; + struct zip_vqx_ena_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + uint64_t reserved_1_63 : 63; + uint64_t ena : 1; +#else /* Word 0 - Little Endian */ + uint64_t ena : 1; + uint64_t reserved_1_63 : 63; +#endif /* Word 0 - End */ + } s; + /* struct zip_vqx_ena_s cn; */ +} zip_vqx_ena_t; + +/** + * Register (NCB) zip_vq#_sbuf_addr + * + * ZIP VF Queue Starting Buffer Address Registers + * These registers set the buffer parameters for the instruction queues. + * When quiescent (i.e. + * outstanding doorbell count is 0), it is safe to rewrite this register + * to effectively reset the + * command buffer state machine. + * These registers must be programmed after software programs the + * corresponding ZIP_QUE()_SBUF_CTL. + */ +typedef union { + uint64_t u; + struct zip_vqx_sbuf_addr_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + uint64_t reserved_49_63 : 15; + uint64_t ptr : 42; + uint64_t off : 7; +#else /* Word 0 - Little Endian */ + uint64_t off : 7; + uint64_t ptr : 42; + uint64_t reserved_49_63 : 15; +#endif /* Word 0 - End */ + } s; + /* struct zip_vqx_sbuf_addr_s cn; */ +} zip_vqx_sbuf_addr_t; + +/** + * Register (NCB) zip_que#_doorbell + * + * ZIP Queue Doorbell Registers + * Doorbells for the ZIP instruction queues. + */ +typedef union { + uint64_t u; + struct zip_quex_doorbell_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + uint64_t reserved_20_63 : 44; + uint64_t dbell_cnt : 20; +#else /* Word 0 - Little Endian */ + uint64_t dbell_cnt : 20; + uint64_t reserved_20_63 : 44; +#endif /* Word 0 - End */ + } s; + /* struct zip_quex_doorbell_s cn; */ +} zip_quex_doorbell_t; + +/** + * Structure zip_nptr_s + * + * ZIP Instruction Next-Chunk-Buffer Pointer (NPTR) Structure + * This structure is used to chain all the ZIP instruction buffers + * together. ZIP instruction buffers are managed + * (allocated and released) by software. + */ +union zip_nptr_s { + uint64_t u; + struct zip_nptr_s_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + uint64_t addr : 64; +#else /* Word 0 - Little Endian */ + uint64_t addr : 64; +#endif /* Word 0 - End */ + } s; + /* struct zip_nptr_s_s cn83xx; */ +}; + +/** + * generic ptr address + */ +union zip_zptr_addr_s { + /** This field can be used to set/clear all bits, or do bitwise + * operations over the entire structure. + */ + uint64_t u; + /** generic ptr address */ + struct { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + uint64_t addr : 64; +#else /* Word 0 - Little Endian */ + uint64_t addr : 64; +#endif /* Word 0 - End */ + } s; +}; + +/** + * generic ptr ctl + */ +union zip_zptr_ctl_s { + /** This field can be used to set/clear all bits, or do bitwise + * operations over the entire structure. + */ + uint64_t u; + /** generic ptr ctl */ + struct { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 1 - Big Endian */ + uint64_t reserved_112_127 : 16; + uint64_t length : 16; + uint64_t reserved_67_95 : 29; + uint64_t fw : 1; + uint64_t nc : 1; + uint64_t data_be : 1; +#else /* Word 1 - Little Endian */ + uint64_t data_be : 1; + uint64_t nc : 1; + uint64_t fw : 1; + uint64_t reserved_67_95 : 29; + uint64_t length : 16; + uint64_t reserved_112_127 : 16; +#endif /* Word 1 - End */ + } s; + +}; + +/** + * Structure zip_inst_s + * + * ZIP Instruction Structure + * Each ZIP instruction has 16 words (they are called IWORD0 to IWORD15 + * within the structure). + */ +union zip_inst_s { + /** This field can be used to set/clear all bits, or do bitwise + * operations over the entire structure. + */ + uint64_t u[16]; + /** ZIP Instruction Structure */ + struct zip_inst_s_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + /** Done interrupt */ + uint64_t doneint : 1; + /** reserved */ + uint64_t reserved_56_62 : 7; + /** Total output length */ + uint64_t totaloutputlength : 24; + /** reserved */ + uint64_t reserved_27_31 : 5; + /** EXNUM */ + uint64_t exn : 3; + /** HASH IV */ + uint64_t iv : 1; + /** EXBITS */ + uint64_t exbits : 7; + /** Hash more-in-file */ + uint64_t hmif : 1; + /** Hash Algorithm and enable */ + uint64_t halg : 3; + /** Sync flush*/ + uint64_t sf : 1; + /** Compression speed/storage */ + uint64_t ss : 2; + /** Compression coding */ + uint64_t cc : 2; + /** End of input data */ + uint64_t ef : 1; + /** Beginning of file */ + uint64_t bf : 1; + // uint64_t reserved_3_4 : 2; + /** Comp/decomp operation */ + uint64_t op : 2; + /** Data sactter */ + uint64_t ds : 1; + /** Data gather */ + uint64_t dg : 1; + /** History gather */ + uint64_t hg : 1; +#else /* Word 0 - Little Endian */ + uint64_t hg : 1; + uint64_t dg : 1; + uint64_t ds : 1; + //uint64_t reserved_3_4 : 2; + uint64_t op : 2; + uint64_t bf : 1; + uint64_t ef : 1; + uint64_t cc : 2; + uint64_t ss : 2; + uint64_t sf : 1; + uint64_t halg : 3; + uint64_t hmif : 1; + uint64_t exbits : 7; + uint64_t iv : 1; + uint64_t exn : 3; + uint64_t reserved_27_31 : 5; + uint64_t totaloutputlength : 24; + uint64_t reserved_56_62 : 7; + uint64_t doneint : 1; + +#endif /* Word 0 - End */ + +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 1 - Big Endian */ + /** History length */ + uint64_t historylength : 16; + /** reserved */ + uint64_t reserved_96_111 : 16; + /** adler/crc32 checksum*/ + uint64_t adlercrc32 : 32; +#else /* Word 1 - Little Endian */ + uint64_t adlercrc32 : 32; + uint64_t reserved_96_111 : 16; + uint64_t historylength : 16; +#endif /* Word 1 - End */ + +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 2 - Big Endian */ + /** Decompression Context Pointer Address */ + union zip_zptr_addr_s ctx_ptr_addr; +#else /* Word 2 - Little Endian */ + union zip_zptr_addr_s ctx_ptr_addr; +#endif /* Word 2 - End */ + +#if defined(__BIG_ENDIAN_BITFIELD) + /** Decompression Context Pointer Control */ + union zip_zptr_ctl_s ctx_ptr_ctl; +#else /* Word 3 - Little Endian */ + union zip_zptr_ctl_s ctx_ptr_ctl; +#endif /* Word 3 - End */ + +#if defined(__BIG_ENDIAN_BITFIELD) + /** Decompression history pointer address */ + union zip_zptr_addr_s his_ptr_addr; +#else /* Word 4 - Little Endian */ + union zip_zptr_addr_s his_ptr_addr; +#endif /* Word 4 - End */ + +#if defined(__BIG_ENDIAN_BITFIELD) + /** Decompression history pointer control */ + union zip_zptr_ctl_s his_ptr_ctl; +#else /* Word 5 - Little Endian */ + union zip_zptr_ctl_s his_ptr_ctl; +#endif /* Word 5 - End */ + +#if defined(__BIG_ENDIAN_BITFIELD) + /** Input and compression history pointer address */ + union zip_zptr_addr_s inp_ptr_addr; +#else /* Word 6 - Little Endian */ + union zip_zptr_addr_s inp_ptr_addr; +#endif /* Word 6 - End */ + +#if defined(__BIG_ENDIAN_BITFIELD) + /** Input and compression history pointer control */ + union zip_zptr_ctl_s inp_ptr_ctl; +#else /* Word 7 - Little Endian */ + union zip_zptr_ctl_s inp_ptr_ctl; +#endif /* Word 7 - End */ + +#if defined(__BIG_ENDIAN_BITFIELD) + /** Output pointer address */ + union zip_zptr_addr_s out_ptr_addr; +#else /* Word 8 - Little Endian */ + union zip_zptr_addr_s out_ptr_addr; +#endif /* Word 8 - End */ + +#if defined(__BIG_ENDIAN_BITFIELD) + /** Output pointer control */ + union zip_zptr_ctl_s out_ptr_ctl; +#else /* Word 9 - Little Endian */ + union zip_zptr_ctl_s out_ptr_ctl; +#endif /* Word 9 - End */ + +#if defined(__BIG_ENDIAN_BITFIELD) + /** Result pointer address */ + union zip_zptr_addr_s res_ptr_addr; +#else /* Word 10 - Little Endian */ + union zip_zptr_addr_s res_ptr_addr; +#endif /* Word 10 - End */ + +#if defined(__BIG_ENDIAN_BITFIELD) + /** Result pointer control */ + union zip_zptr_ctl_s res_ptr_ctl; +#else /* Word 11 - Little Endian */ + union zip_zptr_ctl_s res_ptr_ctl; +#endif /* Word 11 - End */ + +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 12 - Big Endian */ + /** reserved */ + uint64_t reserved_812_831 : 20; + /** SSO guest group */ + uint64_t ggrp : 10; + /** SSO tag type */ + uint64_t tt : 2; + /** SSO tag */ + uint64_t tag : 32; +#else /* Word 12 - Little Endian */ + uint64_t tag : 32; + uint64_t tt : 2; + uint64_t ggrp : 10; + uint64_t reserved_812_831 : 20; +#endif /* Word 12 - End */ + +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 13 - Big Endian */ + /** Work queue entry pointer */ + uint64_t wq_ptr : 64; +#else /* Word 13 - Little Endian */ + uint64_t wq_ptr : 64; +#endif /* Word 13 - End */ + +#if defined(__BIG_ENDIAN_BITFIELD) + /** reserved */ + uint64_t reserved_896_959 : 64; +#else /* Word 14 - Little Endian */ + uint64_t reserved_896_959 : 64; +#endif /* Word 14 - End */ +#if defined(__BIG_ENDIAN_BITFIELD) + /** Hash structure pointer */ + uint64_t hash_ptr : 64; +#else /* Word 15 - Little Endian */ + uint64_t hash_ptr : 64; +#endif /* Word 15 - End */ + } /** ZIP 88xx Instruction Structure */zip88xx; + + /** ZIP Instruction Structure */ + struct zip_inst_s_cn83xx { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + /** Done interrupt */ + uint64_t doneint : 1; + /** reserved */ + uint64_t reserved_56_62 : 7; + /** Total output length */ + uint64_t totaloutputlength : 24; + /** reserved */ + uint64_t reserved_27_31 : 5; + /** EXNUM */ + uint64_t exn : 3; + /** HASH IV */ + uint64_t iv : 1; + /** EXBITS */ + uint64_t exbits : 7; + /** Hash more-in-file */ + uint64_t hmif : 1; + /** Hash Algorithm and enable */ + uint64_t halg : 3; + /** Sync flush*/ + uint64_t sf : 1; + /** Compression speed/storage */ + uint64_t ss : 2; + /** Compression coding */ + uint64_t cc : 2; + /** End of input data */ + uint64_t ef : 1; + /** Beginning of file */ + uint64_t bf : 1; + /** Comp/decomp operation */ + uint64_t op : 2; + /** Data sactter */ + uint64_t ds : 1; + /** Data gather */ + uint64_t dg : 1; + /** History gather */ + uint64_t hg : 1; +#else /* Word 0 - Little Endian */ + uint64_t hg : 1; + uint64_t dg : 1; + uint64_t ds : 1; + uint64_t op : 2; + uint64_t bf : 1; + uint64_t ef : 1; + uint64_t cc : 2; + uint64_t ss : 2; + uint64_t sf : 1; + uint64_t halg : 3; + uint64_t hmif : 1; + uint64_t exbits : 7; + uint64_t iv : 1; + uint64_t exn : 3; + uint64_t reserved_27_31 : 5; + uint64_t totaloutputlength : 24; + uint64_t reserved_56_62 : 7; + uint64_t doneint : 1; +#endif /* Word 0 - End */ +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 1 - Big Endian */ + /** History length */ + uint64_t historylength : 16; + /** reserved */ + uint64_t reserved_96_111 : 16; + /** adler/crc32 checksum*/ + uint64_t adlercrc32 : 32; +#else /* Word 1 - Little Endian */ + uint64_t adlercrc32 : 32; + uint64_t reserved_96_111 : 16; + uint64_t historylength : 16; +#endif /* Word 1 - End */ +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 2 - Big Endian */ + /** Decompression Context Pointer Address */ + union zip_zptr_addr_s ctx_ptr_addr; +#else /* Word 2 - Little Endian */ + union zip_zptr_addr_s ctx_ptr_addr; +#endif /* Word 2 - End */ +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 3 - Big Endian */ + /** Decompression Context Pointer Control */ + union zip_zptr_ctl_s ctx_ptr_ctl; +#else /* Word 3 - Little Endian */ + union zip_zptr_ctl_s ctx_ptr_ctl; +#endif /* Word 3 - End */ +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 4 - Big Endian */ + /** Decompression history pointer address */ + union zip_zptr_addr_s his_ptr_addr; +#else /* Word 4 - Little Endian */ + union zip_zptr_addr_s his_ptr_addr; +#endif /* Word 4 - End */ +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 5 - Big Endian */ + /** Decompression history pointer control */ + union zip_zptr_ctl_s his_ptr_ctl; +#else /* Word 5 - Little Endian */ + union zip_zptr_ctl_s his_ptr_ctl; +#endif /* Word 5 - End */ +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 6 - Big Endian */ + /** Input and compression history pointer address */ + union zip_zptr_addr_s inp_ptr_addr; +#else /* Word 6 - Little Endian */ + union zip_zptr_addr_s inp_ptr_addr; +#endif /* Word 6 - End */ +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 7 - Big Endian */ + /** Input and compression history pointer control */ + union zip_zptr_ctl_s inp_ptr_ctl; +#else /* Word 7 - Little Endian */ + union zip_zptr_ctl_s inp_ptr_ctl; +#endif /* Word 7 - End */ +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 8 - Big Endian */ + /** Output pointer address */ + union zip_zptr_addr_s out_ptr_addr; +#else /* Word 8 - Little Endian */ + union zip_zptr_addr_s out_ptr_addr; +#endif /* Word 8 - End */ +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 9 - Big Endian */ + /** Output pointer control */ + union zip_zptr_ctl_s out_ptr_ctl; +#else /* Word 9 - Little Endian */ + union zip_zptr_ctl_s out_ptr_ctl; +#endif /* Word 9 - End */ +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 10 - Big Endian */ + /** Result pointer address */ + union zip_zptr_addr_s res_ptr_addr; +#else /* Word 10 - Little Endian */ + union zip_zptr_addr_s res_ptr_addr; +#endif /* Word 10 - End */ +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 11 - Big Endian */ + /** Result pointer control */ + union zip_zptr_ctl_s res_ptr_ctl; +#else /* Word 11 - Little Endian */ + union zip_zptr_ctl_s res_ptr_ctl; +#endif /* Word 11 - End */ +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 12 - Big Endian */ + /** reserved */ + uint64_t reserved_812_831 : 20; + /** SSO guest group */ + uint64_t ggrp : 10; + /** SSO tag type */ + uint64_t tt : 2; + /** SSO tag */ + uint64_t tag : 32; +#else /* Word 12 - Little Endian */ + uint64_t tag : 32; + uint64_t tt : 2; + uint64_t ggrp : 10; + uint64_t reserved_812_831 : 20; +#endif /* Word 12 - End */ +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 13 - Big Endian */ + /** Work queue entry pointer */ + uint64_t wq_ptr : 64; +#else /* Word 13 - Little Endian */ + uint64_t wq_ptr : 64; +#endif /* Word 13 - End */ +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 14 - Big Endian */ + /** reserved */ + uint64_t reserved_896_959 : 64; +#else /* Word 14 - Little Endian */ + uint64_t reserved_896_959 : 64; +#endif /* Word 14 - End */ +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 15 - Big Endian */ + /** Hash structure pointer */ + uint64_t hash_ptr : 64; +#else /* Word 15 - Little Endian */ + uint64_t hash_ptr : 64; +#endif /* Word 15 - End */ + } /** ZIP 83xx Instruction Structure */s; +}; + +/** + * Structure zip_zres_s + * + * ZIP Result Structure + * The ZIP coprocessor writes the result structure after it completes the + * invocation. The result structure is exactly 24 bytes, and each invocation + * of the ZIP coprocessor produces exactly one result structure. + */ +union zip_zres_s { + /** This field can be used to set/clear all bits, or do bitwise + * operations over the entire structure. + */ + uint64_t u[8]; + /** ZIP Result Structure */ + struct zip_zres_s_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + /** crc32 checksum of uncompressed stream */ + uint64_t crc32 : 32; + /** adler32 checksum of uncompressed stream*/ + uint64_t adler32 : 32; +#else /* Word 0 - Little Endian */ + uint64_t adler32 : 32; + uint64_t crc32 : 32; +#endif /* Word 0 - End */ +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 1 - Big Endian */ + /** Total numer of Bytes produced in output stream */ + uint64_t totalbyteswritten : 32; + /** Total number of bytes processed from the input stream */ + uint64_t totalbytesread : 32; +#else /* Word 1 - Little Endian */ + uint64_t totalbytesread : 32; + uint64_t totalbyteswritten : 32; +#endif /* Word 1 - End */ +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 2 - Big Endian */ + /** Total number of compressed input bits + * consumed to decompress all blocks in the file + */ + uint64_t totalbitsprocessed : 32; + /** Done interrupt*/ + uint64_t doneint : 1; + /** reserved */ + uint64_t reserved_155_158 : 4; + /** EXNUM */ + uint64_t exn : 3; + /** reserved */ + uint64_t reserved_151 : 1; + /** EXBITS */ + uint64_t exbits : 7; + /** reserved */ + uint64_t reserved_137_143 : 7; + /** End of file */ + uint64_t ef : 1; + /** Completion/error code */ + uint64_t compcode : 8; +#else /* Word 2 - Little Endian */ + uint64_t compcode : 8; + uint64_t ef : 1; + uint64_t reserved_137_143 : 7; + uint64_t exbits : 7; + uint64_t reserved_151 : 1; + uint64_t exn : 3; + uint64_t reserved_155_158 : 4; + uint64_t doneint : 1; + uint64_t totalbitsprocessed : 32; +#endif /* Word 2 - End */ +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 3 - Big Endian */ + /** reserved */ + uint64_t reserved_253_255 : 3; + /** Hash length in bytes */ + uint64_t hshlen : 61; +#else /* Word 3 - Little Endian */ + uint64_t hshlen : 61; + uint64_t reserved_253_255 : 3; +#endif /* Word 3 - End */ +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 4 - Big Endian */ + /** Double-word 0 of computed hash */ + uint64_t hash0 : 64; +#else /* Word 4 - Little Endian */ + uint64_t hash0 : 64; +#endif /* Word 4 - End */ +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 5 - Big Endian */ + /** Double-word 1 of computed hash */ + uint64_t hash1 : 64; +#else /* Word 5 - Little Endian */ + uint64_t hash1 : 64; +#endif /* Word 5 - End */ +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 6 - Big Endian */ + /** Double-word 2 of computed hash */ + uint64_t hash2 : 64; +#else /* Word 6 - Little Endian */ + uint64_t hash2 : 64; +#endif /* Word 6 - End */ +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 7 - Big Endian */ + /** Double-word 3 of computed hash */ + uint64_t hash3 : 64; +#else /* Word 7 - Little Endian */ + uint64_t hash3 : 64; +#endif /* Word 7 - End */ + } /** ZIP Result Structure */s; + + /* struct zip_zres_s_s cn83xx; */ +}; + +/** + * Structure zip_zptr_s + * + * ZIP Generic Pointer Structure + * This structure is the generic format of pointers in ZIP_INST_S. + */ +union zip_zptr_s { + /** This field can be used to set/clear all bits, or do bitwise + * operations over the entire structure. + */ + uint64_t u[2]; + /** ZIP Generic Pointer Structure */ + struct zip_zptr_s_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + /** Pointer to Data or scatter-gather list */ + uint64_t addr : 64; +#else /* Word 0 - Little Endian */ + uint64_t addr : 64; +#endif /* Word 0 - End */ +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 1 - Big Endian */ + /** reserved */ + uint64_t reserved_112_127 : 16; + /** Length of Data or scatter-gather list*/ + uint64_t length : 16; + /** reserved */ + uint64_t reserved_67_95 : 29; + /** Full-block write */ + uint64_t fw : 1; + /** No cache allocation */ + uint64_t nc : 1; + /** reserved */ + uint64_t data_be : 1; +#else /* Word 1 - Little Endian */ + uint64_t data_be : 1; + uint64_t nc : 1; + uint64_t fw : 1; + uint64_t reserved_67_95 : 29; + uint64_t length : 16; + uint64_t reserved_112_127 : 16; +#endif /* Word 1 - End */ + } /** ZIP Generic Pointer Structure */s; +}; + +/** + * Enumeration zip_comp_e + * + * ZIP Completion Enumeration + * Enumerates the values of ZIP_ZRES_S[COMPCODE]. + */ +#define ZIP_COMP_E_NOTDONE (0) +#define ZIP_COMP_E_SUCCESS (1) +#define ZIP_COMP_E_DTRUNC (2) +#define ZIP_COMP_E_DSTOP (3) +#define ZIP_COMP_E_ITRUNC (4) +#define ZIP_COMP_E_RBLOCK (5) +#define ZIP_COMP_E_NLEN (6) +#define ZIP_COMP_E_BADCODE (7) +#define ZIP_COMP_E_BADCODE2 (8) +#define ZIP_COMP_E_ZERO_LEN (9) +#define ZIP_COMP_E_PARITY (0xa) +#define ZIP_COMP_E_FATAL (0xb) +#define ZIP_COMP_E_TIMEOUT (0xc) +#define ZIP_COMP_E_INSTR_ERR (0xd) +#define ZIP_COMP_E_HCTX_ERR (0xe) +#define ZIP_COMP_E_STOP (3) + +/** + * Enumeration zip_op_e + * + * ZIP Operation Enumeration + * Enumerates ZIP_INST_S[OP]. + * Internal: + */ +#define ZIP_OP_E_DECOMP (0) +#define ZIP_OP_E_NOCOMP (1) +#define ZIP_OP_E_COMP (2) + +/** + * Enumeration zip compression levels + * + * ZIP Compression Level Enumeration + * Enumerates ZIP_INST_S[SS]. + * Internal: + */ +#define ZIP_COMP_E_LEVEL_MAX (0) +#define ZIP_COMP_E_LEVEL_MED (1) +#define ZIP_COMP_E_LEVEL_LOW (2) +#define ZIP_COMP_E_LEVEL_MIN (3) + +#endif /* _RTE_ZIP_REGS_H_ */ diff --git a/src/spdk/dpdk/drivers/compress/octeontx/meson.build b/src/spdk/dpdk/drivers/compress/octeontx/meson.build new file mode 100644 index 00000000..7cd202d0 --- /dev/null +++ b/src/spdk/dpdk/drivers/compress/octeontx/meson.build @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2018 Cavium, Inc + +name = 'octeontx_compress' +sources = files('otx_zip.c', 'otx_zip_pmd.c') +allow_experimental_apis = true +includes += include_directories('include') +deps += ['mempool_octeontx', 'bus_pci'] +ext_deps += dep diff --git a/src/spdk/dpdk/drivers/compress/octeontx/otx_zip.c b/src/spdk/dpdk/drivers/compress/octeontx/otx_zip.c new file mode 100644 index 00000000..a9046ff3 --- /dev/null +++ b/src/spdk/dpdk/drivers/compress/octeontx/otx_zip.c @@ -0,0 +1,180 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 Cavium, Inc + */ + +#include "otx_zip.h" + +uint64_t +zip_reg_read64(uint8_t *hw_addr, uint64_t offset) +{ + uint8_t *base = hw_addr; + return *(volatile uint64_t *)(base + offset); +} + +void +zip_reg_write64(uint8_t *hw_addr, uint64_t offset, uint64_t val) +{ + uint8_t *base = hw_addr; + *(uint64_t *)(base + offset) = val; +} + +static void +zip_q_enable(struct zipvf_qp *qp) +{ + zip_vqx_ena_t que_ena; + + /*ZIP VFx command queue init*/ + que_ena.u = 0ull; + que_ena.s.ena = 1; + + zip_reg_write64(qp->vf->vbar0, ZIP_VQ_ENA, que_ena.u); + rte_wmb(); +} + +/* initialize given qp on zip device */ +int +zipvf_q_init(struct zipvf_qp *qp) +{ + zip_vqx_sbuf_addr_t que_sbuf_addr; + + uint64_t size; + void *cmdq_addr; + uint64_t iova; + struct zipvf_cmdq *cmdq = &qp->cmdq; + struct zip_vf *vf = qp->vf; + + /* allocate and setup instruction queue */ + size = ZIP_MAX_CMDQ_SIZE; + size = ZIP_ALIGN_ROUNDUP(size, ZIP_CMDQ_ALIGN); + + cmdq_addr = rte_zmalloc(qp->name, size, ZIP_CMDQ_ALIGN); + if (cmdq_addr == NULL) + return -1; + + cmdq->sw_head = (uint64_t *)cmdq_addr; + cmdq->va = (uint8_t *)cmdq_addr; + iova = rte_mem_virt2iova(cmdq_addr); + + cmdq->iova = iova; + + que_sbuf_addr.u = 0ull; + que_sbuf_addr.s.ptr = (cmdq->iova >> 7); + zip_reg_write64(vf->vbar0, ZIP_VQ_SBUF_ADDR, que_sbuf_addr.u); + + zip_q_enable(qp); + + memset(cmdq->va, 0, ZIP_MAX_CMDQ_SIZE); + rte_spinlock_init(&cmdq->qlock); + + return 0; +} + +int +zipvf_q_term(struct zipvf_qp *qp) +{ + struct zipvf_cmdq *cmdq = &qp->cmdq; + zip_vqx_ena_t que_ena; + struct zip_vf *vf = qp->vf; + + if (cmdq->va != NULL) { + memset(cmdq->va, 0, ZIP_MAX_CMDQ_SIZE); + rte_free(cmdq->va); + } + + /*Disabling the ZIP queue*/ + que_ena.u = 0ull; + zip_reg_write64(vf->vbar0, ZIP_VQ_ENA, que_ena.u); + + return 0; +} + +void +zipvf_push_command(struct zipvf_qp *qp, union zip_inst_s *cmd) +{ + zip_quex_doorbell_t dbell; + union zip_nptr_s ncp; + uint64_t *ncb_ptr; + struct zipvf_cmdq *cmdq = &qp->cmdq; + void *reg_base = qp->vf->vbar0; + + /*Held queue lock*/ + rte_spinlock_lock(&(cmdq->qlock)); + + /* Check space availability in zip cmd queue */ + if ((((cmdq->sw_head - (uint64_t *)cmdq->va) * sizeof(uint64_t *)) + + ZIP_CMD_SIZE) == (ZIP_MAX_CMDQ_SIZE - ZIP_MAX_NCBP_SIZE)) { + /*Last buffer of the command queue*/ + memcpy((uint8_t *)cmdq->sw_head, + (uint8_t *)cmd, + sizeof(union zip_inst_s)); + /* move pointer to next loc in unit of 64-bit word */ + cmdq->sw_head += ZIP_CMD_SIZE_WORDS; + + /* now, point the "Next-Chunk Buffer Ptr" to sw_head */ + ncb_ptr = cmdq->sw_head; + /* Pointing head again to cmdqueue base*/ + cmdq->sw_head = (uint64_t *)cmdq->va; + + ncp.u = 0ull; + ncp.s.addr = cmdq->iova; + *ncb_ptr = ncp.u; + } else { + /*Enough buffers available in the command queue*/ + memcpy((uint8_t *)cmdq->sw_head, + (uint8_t *)cmd, + sizeof(union zip_inst_s)); + cmdq->sw_head += ZIP_CMD_SIZE_WORDS; + } + + rte_wmb(); + + /* Ringing ZIP VF doorbell */ + dbell.u = 0ull; + dbell.s.dbell_cnt = 1; + zip_reg_write64(reg_base, ZIP_VQ_DOORBELL, dbell.u); + + rte_spinlock_unlock(&(cmdq->qlock)); +} + +int +zipvf_create(struct rte_compressdev *compressdev) +{ + struct rte_pci_device *pdev = RTE_DEV_TO_PCI(compressdev->device); + struct zip_vf *zipvf = NULL; + char *dev_name = compressdev->data->name; + void *vbar0; + uint64_t reg; + + if (pdev->mem_resource[0].phys_addr == 0ULL) + return -EIO; + + vbar0 = pdev->mem_resource[0].addr; + if (!vbar0) { + ZIP_PMD_ERR("Failed to map BAR0 of %s", dev_name); + return -ENODEV; + } + + zipvf = (struct zip_vf *)(compressdev->data->dev_private); + + if (!zipvf) + return -ENOMEM; + + zipvf->vbar0 = vbar0; + reg = zip_reg_read64(zipvf->vbar0, ZIP_VF_PF_MBOXX(0)); + /* Storing domain in local to ZIP VF */ + zipvf->dom_sdom = reg; + zipvf->pdev = pdev; + zipvf->max_nb_queue_pairs = ZIP_MAX_VF_QUEUE; + return 0; +} + +int +zipvf_destroy(struct rte_compressdev *compressdev) +{ + struct zip_vf *vf = (struct zip_vf *)(compressdev->data->dev_private); + + /* Rewriting the domain_id in ZIP_VF_MBOX for app rerun */ + zip_reg_write64(vf->vbar0, ZIP_VF_PF_MBOXX(0), vf->dom_sdom); + + return 0; +} diff --git a/src/spdk/dpdk/drivers/compress/octeontx/otx_zip.h b/src/spdk/dpdk/drivers/compress/octeontx/otx_zip.h new file mode 100644 index 00000000..99a38d00 --- /dev/null +++ b/src/spdk/dpdk/drivers/compress/octeontx/otx_zip.h @@ -0,0 +1,277 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 Cavium, Inc + */ + +#ifndef _RTE_OCTEONTX_ZIP_VF_H_ +#define _RTE_OCTEONTX_ZIP_VF_H_ + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +int octtx_zip_logtype_driver; + +/* ZIP VF Control/Status registers (CSRs): */ +/* VF_BAR0: */ +#define ZIP_VQ_ENA (0x10) +#define ZIP_VQ_SBUF_ADDR (0x20) +#define ZIP_VF_PF_MBOXX(x) (0x400 | (x)<<3) +#define ZIP_VQ_DOORBELL (0x1000) + +/**< Vendor ID */ +#define PCI_VENDOR_ID_CAVIUM 0x177D +/**< PCI device id of ZIP VF */ +#define PCI_DEVICE_ID_OCTEONTX_ZIPVF 0xA037 + +/* maxmum number of zip vf devices */ +#define ZIP_MAX_VFS 8 + +/* max size of one chunk */ +#define ZIP_MAX_CHUNK_SIZE 8192 + +/* each instruction is fixed 128 bytes */ +#define ZIP_CMD_SIZE 128 + +#define ZIP_CMD_SIZE_WORDS (ZIP_CMD_SIZE >> 3) /* 16 64_bit words */ + +/* size of next chunk buffer pointer */ +#define ZIP_MAX_NCBP_SIZE 8 + +/* size of instruction queue in units of instruction size */ +#define ZIP_MAX_NUM_CMDS ((ZIP_MAX_CHUNK_SIZE - ZIP_MAX_NCBP_SIZE) / \ + ZIP_CMD_SIZE) /* 63 */ + +/* size of instruct queue in bytes */ +#define ZIP_MAX_CMDQ_SIZE ((ZIP_MAX_NUM_CMDS * ZIP_CMD_SIZE) + \ + ZIP_MAX_NCBP_SIZE)/* ~8072ull */ + +#define ZIP_BUF_SIZE 256 + +#define ZIP_SGPTR_ALIGN 16 +#define ZIP_CMDQ_ALIGN 128 +#define MAX_SG_LEN ((ZIP_BUF_SIZE - ZIP_SGPTR_ALIGN) / sizeof(void *)) + +/**< ZIP PMD specified queue pairs */ +#define ZIP_MAX_VF_QUEUE 1 + +#define ZIP_ALIGN_ROUNDUP(x, _align) \ + ((_align) * (((x) + (_align) - 1) / (_align))) + +/**< ZIP PMD device name */ +#define COMPRESSDEV_NAME_ZIP_PMD compress_octeonx + +#define ZIP_PMD_LOG(level, fmt, args...) \ + rte_log(RTE_LOG_ ## level, \ + octtx_zip_logtype_driver, "%s(): "fmt "\n", \ + __func__, ##args) + +#define ZIP_PMD_INFO(fmt, args...) \ + ZIP_PMD_LOG(INFO, fmt, ## args) +#define ZIP_PMD_ERR(fmt, args...) \ + ZIP_PMD_LOG(ERR, fmt, ## args) + +/* resources required to process stream */ +enum { + RES_BUF = 0, + CMD_BUF, + HASH_CTX_BUF, + DECOMP_CTX_BUF, + IN_DATA_BUF, + OUT_DATA_BUF, + HISTORY_DATA_BUF, + MAX_BUFS_PER_STREAM +} NUM_BUFS_PER_STREAM; + +struct zip_stream; +struct zipvf_qp; + +/* Algorithm handler function prototype */ +typedef int (*comp_func_t)(struct rte_comp_op *op, + struct zipvf_qp *qp, struct zip_stream *zstrm); + +/** + * ZIP private stream structure + */ +struct zip_stream { + union zip_inst_s *inst; + /* zip instruction pointer */ + comp_func_t func; + /* function to process comp operation */ + void *bufs[MAX_BUFS_PER_STREAM]; +} _rte_cache_aligned; + + +/** + * ZIP instruction Queue + */ +struct zipvf_cmdq { + rte_spinlock_t qlock; + /* queue lock */ + uint64_t *sw_head; + /* pointer to start of 8-byte word length queue-head */ + uint8_t *va; + /* pointer to instruction queue virtual address */ + rte_iova_t iova; + /* iova addr of cmdq head*/ +}; + +/** + * ZIP device queue structure + */ +struct zipvf_qp { + struct zipvf_cmdq cmdq; + /* Hardware instruction queue structure */ + struct rte_ring *processed_pkts; + /* Ring for placing processed packets */ + struct rte_compressdev_stats qp_stats; + /* Queue pair statistics */ + uint16_t id; + /* Queue Pair Identifier */ + const char *name; + /* Unique Queue Pair Name */ + struct zip_vf *vf; + /* pointer to device, queue belongs to */ +} __rte_cache_aligned; + +/** + * ZIP VF device structure. + */ +struct zip_vf { + int vfid; + /* vf index */ + struct rte_pci_device *pdev; + /* pci device */ + void *vbar0; + /* CSR base address for underlying BAR0 VF.*/ + uint64_t dom_sdom; + /* Storing mbox domain and subdomain id for app rerun*/ + uint32_t max_nb_queue_pairs; + /* pointer to device qps */ + struct rte_mempool *zip_mp; + /* pointer to pools */ +} __rte_cache_aligned; + + +static inline void +zipvf_prepare_in_buf(struct zip_stream *zstrm, struct rte_comp_op *op) +{ + uint32_t offset, inlen; + struct rte_mbuf *m_src; + union zip_inst_s *inst = zstrm->inst; + + inlen = op->src.length; + offset = op->src.offset; + m_src = op->m_src; + + /* Prepare direct input data pointer */ + inst->s.dg = 0; + inst->s.inp_ptr_addr.s.addr = + rte_pktmbuf_iova_offset(m_src, offset); + inst->s.inp_ptr_ctl.s.length = inlen; +} + +static inline void +zipvf_prepare_out_buf(struct zip_stream *zstrm, struct rte_comp_op *op) +{ + uint32_t offset; + struct rte_mbuf *m_dst; + union zip_inst_s *inst = zstrm->inst; + + offset = op->dst.offset; + m_dst = op->m_dst; + + /* Prepare direct input data pointer */ + inst->s.ds = 0; + inst->s.out_ptr_addr.s.addr = + rte_pktmbuf_iova_offset(m_dst, offset); + inst->s.totaloutputlength = rte_pktmbuf_pkt_len(m_dst) - + op->dst.offset; + inst->s.out_ptr_ctl.s.length = inst->s.totaloutputlength; +} + +static inline void +zipvf_prepare_cmd_stateless(struct rte_comp_op *op, struct zip_stream *zstrm) +{ + union zip_inst_s *inst = zstrm->inst; + + /* set flush flag to always 1*/ + inst->s.ef = 1; + + if (inst->s.op == ZIP_OP_E_DECOMP) + inst->s.sf = 1; + else + inst->s.sf = 0; + + /* Set input checksum */ + inst->s.adlercrc32 = op->input_chksum; + + /* Prepare gather buffers */ + zipvf_prepare_in_buf(zstrm, op); + zipvf_prepare_out_buf(zstrm, op); +} + +#ifdef ZIP_DBG +static inline void +zip_dump_instruction(void *inst) +{ + union zip_inst_s *cmd83 = (union zip_inst_s *)inst; + printf("####### START ########\n"); + printf("doneint:%d totaloutputlength:%d\n", cmd83->s.doneint, + cmd83->s.totaloutputlength); + printf("exnum:%d iv:%d exbits:%d hmif:%d halg:%d\n", cmd83->s.exn, + cmd83->s.iv, cmd83->s.exbits, cmd83->s.hmif, cmd83->s.halg); + printf("flush:%d speed:%d cc:%d\n", cmd83->s.sf, + cmd83->s.ss, cmd83->s.cc); + printf("eof:%d bof:%d op:%d dscatter:%d dgather:%d hgather:%d\n", + cmd83->s.ef, cmd83->s.bf, cmd83->s.op, cmd83->s.ds, + cmd83->s.dg, cmd83->s.hg); + printf("historylength:%d adler32:%d\n", cmd83->s.historylength, + cmd83->s.adlercrc32); + printf("ctx_ptr.addr:0x%"PRIx64"\n", cmd83->s.ctx_ptr_addr.s.addr); + printf("ctx_ptr.len:%d\n", cmd83->s.ctx_ptr_ctl.s.length); + printf("history_ptr.addr:0x%"PRIx64"\n", cmd83->s.his_ptr_addr.s.addr); + printf("history_ptr.len:%d\n", cmd83->s.his_ptr_ctl.s.length); + printf("inp_ptr.addr:0x%"PRIx64"\n", cmd83->s.inp_ptr_addr.s.addr); + printf("inp_ptr.len:%d\n", cmd83->s.inp_ptr_ctl.s.length); + printf("out_ptr.addr:0x%"PRIx64"\n", cmd83->s.out_ptr_addr.s.addr); + printf("out_ptr.len:%d\n", cmd83->s.out_ptr_ctl.s.length); + printf("result_ptr.len:%d\n", cmd83->s.res_ptr_ctl.s.length); + printf("####### END ########\n"); +} +#endif + +int +zipvf_create(struct rte_compressdev *compressdev); + +int +zipvf_destroy(struct rte_compressdev *compressdev); + +int +zipvf_q_init(struct zipvf_qp *qp); + +int +zipvf_q_term(struct zipvf_qp *qp); + +void +zipvf_push_command(struct zipvf_qp *qp, union zip_inst_s *zcmd); + +int +zip_process_op(struct rte_comp_op *op, + struct zipvf_qp *qp, + struct zip_stream *zstrm); + +uint64_t +zip_reg_read64(uint8_t *hw_addr, uint64_t offset); + +void +zip_reg_write64(uint8_t *hw_addr, uint64_t offset, uint64_t val); + +#endif /* _RTE_ZIP_VF_H_ */ diff --git a/src/spdk/dpdk/drivers/compress/octeontx/otx_zip_pmd.c b/src/spdk/dpdk/drivers/compress/octeontx/otx_zip_pmd.c new file mode 100644 index 00000000..9d13f933 --- /dev/null +++ b/src/spdk/dpdk/drivers/compress/octeontx/otx_zip_pmd.c @@ -0,0 +1,658 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 Cavium, Inc + */ + +#include + +#include +#include +#include +#include + +#include "otx_zip.h" + +static const struct rte_compressdev_capabilities + octtx_zip_pmd_capabilities[] = { + { .algo = RTE_COMP_ALGO_DEFLATE, + /* Deflate */ + .comp_feature_flags = RTE_COMP_FF_HUFFMAN_FIXED | + RTE_COMP_FF_HUFFMAN_DYNAMIC, + /* Non sharable Priv XFORM and Stateless */ + .window_size = { + .min = 1, + .max = 14, + .increment = 1 + /* size supported 2^1 to 2^14 */ + }, + }, + RTE_COMP_END_OF_CAPABILITIES_LIST() +}; + +/* + * Reset session to default state for next set of stateless operation + */ +static inline void +reset_stream(struct zip_stream *z_stream) +{ + union zip_inst_s *inst = (union zip_inst_s *)(z_stream->inst); + + inst->s.bf = 1; + inst->s.ef = 0; +} + +int +zip_process_op(struct rte_comp_op *op, + struct zipvf_qp *qp, + struct zip_stream *zstrm) +{ + union zip_inst_s *inst = zstrm->inst; + volatile union zip_zres_s *zresult = NULL; + + + if ((op->m_src->nb_segs > 1) || (op->m_dst->nb_segs > 1) || + (op->src.offset > rte_pktmbuf_pkt_len(op->m_src)) || + (op->dst.offset > rte_pktmbuf_pkt_len(op->m_dst))) { + op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; + ZIP_PMD_ERR("Segmented packet is not supported\n"); + return 0; + } + + zipvf_prepare_cmd_stateless(op, zstrm); + + zresult = (union zip_zres_s *)zstrm->bufs[RES_BUF]; + zresult->s.compcode = 0; + +#ifdef ZIP_DBG + zip_dump_instruction(inst); +#endif + + /* Submit zip command */ + zipvf_push_command(qp, (void *)inst); + + /* Check and Process results in sync mode */ + do { + } while (!zresult->s.compcode); + + if (zresult->s.compcode == ZIP_COMP_E_SUCCESS) { + op->status = RTE_COMP_OP_STATUS_SUCCESS; + } else { + /* FATAL error cannot do anything */ + ZIP_PMD_ERR("operation failed with error code:%d\n", + zresult->s.compcode); + if (zresult->s.compcode == ZIP_COMP_E_DSTOP) + op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED; + else + op->status = RTE_COMP_OP_STATUS_ERROR; + } + + ZIP_PMD_INFO("written %d\n", zresult->s.totalbyteswritten); + + /* Update op stats */ + switch (op->status) { + case RTE_COMP_OP_STATUS_SUCCESS: + op->consumed = zresult->s.totalbytesread; + /* Fall-through */ + case RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED: + op->produced = zresult->s.totalbyteswritten; + break; + default: + ZIP_PMD_ERR("stats not updated for status:%d\n", + op->status); + break; + } + /* zstream is reset irrespective of result */ + reset_stream(zstrm); + + zresult->s.compcode = ZIP_COMP_E_NOTDONE; + return 0; +} + +/** Parse xform parameters and setup a stream */ +static int +zip_set_stream_parameters(struct rte_compressdev *dev, + const struct rte_comp_xform *xform, + struct zip_stream *z_stream) +{ + int ret; + union zip_inst_s *inst; + struct zip_vf *vf = (struct zip_vf *)dev->data->dev_private; + void *res; + + /* Allocate resources required by a stream */ + ret = rte_mempool_get_bulk(vf->zip_mp, + z_stream->bufs, MAX_BUFS_PER_STREAM); + if (ret < 0) + return -1; + + /* get one command buffer from pool and set up */ + inst = (union zip_inst_s *)z_stream->bufs[CMD_BUF]; + res = z_stream->bufs[RES_BUF]; + + memset(inst->u, 0, sizeof(inst->u)); + + /* set bf for only first ops of stream */ + inst->s.bf = 1; + + if (xform->type == RTE_COMP_COMPRESS) { + inst->s.op = ZIP_OP_E_COMP; + + switch (xform->compress.deflate.huffman) { + case RTE_COMP_HUFFMAN_DEFAULT: + inst->s.cc = ZIP_CC_DEFAULT; + break; + case RTE_COMP_HUFFMAN_FIXED: + inst->s.cc = ZIP_CC_FIXED_HUFF; + break; + case RTE_COMP_HUFFMAN_DYNAMIC: + inst->s.cc = ZIP_CC_DYN_HUFF; + break; + default: + ret = -1; + goto err; + } + + switch (xform->compress.level) { + case RTE_COMP_LEVEL_MIN: + inst->s.ss = ZIP_COMP_E_LEVEL_MIN; + break; + case RTE_COMP_LEVEL_MAX: + inst->s.ss = ZIP_COMP_E_LEVEL_MAX; + break; + case RTE_COMP_LEVEL_NONE: + ZIP_PMD_ERR("Compression level not supported"); + ret = -1; + goto err; + default: + /* for any value between min and max , choose + * PMD default. + */ + inst->s.ss = ZIP_COMP_E_LEVEL_MED; /** PMD default **/ + break; + } + } else if (xform->type == RTE_COMP_DECOMPRESS) { + inst->s.op = ZIP_OP_E_DECOMP; + /* from HRM, + * For DEFLATE decompression, [CC] must be 0x0. + * For decompression, [SS] must be 0x0 + */ + inst->s.cc = 0; + /* Speed bit should not be set for decompression */ + inst->s.ss = 0; + /* decompression context is supported only for STATEFUL + * operations. Currently we support STATELESS ONLY so + * skip setting of ctx pointer + */ + + } else { + ZIP_PMD_ERR("\nxform type not supported"); + ret = -1; + goto err; + } + + inst->s.res_ptr_addr.s.addr = rte_mempool_virt2iova(res); + inst->s.res_ptr_ctl.s.length = 0; + + z_stream->inst = inst; + z_stream->func = zip_process_op; + + return 0; + +err: + rte_mempool_put_bulk(vf->zip_mp, + (void *)&(z_stream->bufs[0]), + MAX_BUFS_PER_STREAM); + + return ret; +} + +/** Configure device */ +static int +zip_pmd_config(struct rte_compressdev *dev, + struct rte_compressdev_config *config) +{ + int nb_streams; + char res_pool[RTE_MEMZONE_NAMESIZE]; + struct zip_vf *vf; + struct rte_mempool *zip_buf_mp; + + if (!config || !dev) + return -EIO; + + vf = (struct zip_vf *)(dev->data->dev_private); + + /* create pool with maximum numbers of resources + * required by streams + */ + + /* use common pool for non-shareable priv_xform and stream */ + nb_streams = config->max_nb_priv_xforms + config->max_nb_streams; + + snprintf(res_pool, RTE_MEMZONE_NAMESIZE, "octtx_zip_res_pool%u", + dev->data->dev_id); + + /** TBD Should we use the per core object cache for stream resources */ + zip_buf_mp = rte_mempool_create( + res_pool, + nb_streams * MAX_BUFS_PER_STREAM, + ZIP_BUF_SIZE, + 0, + 0, + NULL, + NULL, + NULL, + NULL, + SOCKET_ID_ANY, + 0); + + if (zip_buf_mp == NULL) { + ZIP_PMD_ERR( + "Failed to create buf mempool octtx_zip_res_pool%u", + dev->data->dev_id); + return -1; + } + + vf->zip_mp = zip_buf_mp; + + return 0; +} + +/** Start device */ +static int +zip_pmd_start(__rte_unused struct rte_compressdev *dev) +{ + return 0; +} + +/** Stop device */ +static void +zip_pmd_stop(__rte_unused struct rte_compressdev *dev) +{ + +} + +/** Close device */ +static int +zip_pmd_close(struct rte_compressdev *dev) +{ + if (dev == NULL) + return -1; + + struct zip_vf *vf = (struct zip_vf *)dev->data->dev_private; + rte_mempool_free(vf->zip_mp); + + return 0; +} + +/** Get device statistics */ +static void +zip_pmd_stats_get(struct rte_compressdev *dev, + struct rte_compressdev_stats *stats) +{ + int qp_id; + + for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { + struct zipvf_qp *qp = dev->data->queue_pairs[qp_id]; + + stats->enqueued_count += qp->qp_stats.enqueued_count; + stats->dequeued_count += qp->qp_stats.dequeued_count; + + stats->enqueue_err_count += qp->qp_stats.enqueue_err_count; + stats->dequeue_err_count += qp->qp_stats.dequeue_err_count; + } +} + +/** Reset device statistics */ +static void +zip_pmd_stats_reset(struct rte_compressdev *dev) +{ + int qp_id; + + for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { + struct zipvf_qp *qp = dev->data->queue_pairs[qp_id]; + memset(&qp->qp_stats, 0, sizeof(qp->qp_stats)); + } +} + +/** Get device info */ +static void +zip_pmd_info_get(struct rte_compressdev *dev, + struct rte_compressdev_info *dev_info) +{ + struct zip_vf *vf = (struct zip_vf *)dev->data->dev_private; + + if (dev_info != NULL) { + dev_info->driver_name = dev->device->driver->name; + dev_info->feature_flags = dev->feature_flags; + dev_info->capabilities = octtx_zip_pmd_capabilities; + dev_info->max_nb_queue_pairs = vf->max_nb_queue_pairs; + } +} + +/** Release queue pair */ +static int +zip_pmd_qp_release(struct rte_compressdev *dev, uint16_t qp_id) +{ + struct zipvf_qp *qp = dev->data->queue_pairs[qp_id]; + + if (qp != NULL) { + zipvf_q_term(qp); + + if (qp->processed_pkts) + rte_ring_free(qp->processed_pkts); + + rte_free(qp); + dev->data->queue_pairs[qp_id] = NULL; + } + return 0; +} + +/** Create a ring to place process packets on */ +static struct rte_ring * +zip_pmd_qp_create_processed_pkts_ring(struct zipvf_qp *qp, + unsigned int ring_size, int socket_id) +{ + struct rte_ring *r; + + r = rte_ring_lookup(qp->name); + if (r) { + if (rte_ring_get_size(r) >= ring_size) { + ZIP_PMD_INFO("Reusing existing ring %s for processed" + " packets", qp->name); + return r; + } + + ZIP_PMD_ERR("Unable to reuse existing ring %s for processed" + " packets", qp->name); + return NULL; + } + + return rte_ring_create(qp->name, ring_size, socket_id, + RING_F_EXACT_SZ); +} + +/** Setup a queue pair */ +static int +zip_pmd_qp_setup(struct rte_compressdev *dev, uint16_t qp_id, + uint32_t max_inflight_ops, int socket_id) +{ + struct zipvf_qp *qp = NULL; + struct zip_vf *vf; + char *name; + int ret; + + if (!dev) + return -1; + + vf = (struct zip_vf *) (dev->data->dev_private); + + /* Free memory prior to re-allocation if needed. */ + if (dev->data->queue_pairs[qp_id] != NULL) { + ZIP_PMD_INFO("Using existing queue pair %d ", qp_id); + return 0; + } + + name = rte_malloc(NULL, RTE_COMPRESSDEV_NAME_MAX_LEN, 0); + snprintf(name, RTE_COMPRESSDEV_NAME_MAX_LEN, + "zip_pmd_%u_qp_%u", + dev->data->dev_id, qp_id); + + /* Allocate the queue pair data structure. */ + qp = rte_zmalloc_socket(name, sizeof(*qp), + RTE_CACHE_LINE_SIZE, socket_id); + if (qp == NULL) + return (-ENOMEM); + + qp->name = name; + + /* Create completion queue upto max_inflight_ops */ + qp->processed_pkts = zip_pmd_qp_create_processed_pkts_ring(qp, + max_inflight_ops, socket_id); + if (qp->processed_pkts == NULL) + goto qp_setup_cleanup; + + qp->id = qp_id; + qp->vf = vf; + + ret = zipvf_q_init(qp); + if (ret < 0) + goto qp_setup_cleanup; + + dev->data->queue_pairs[qp_id] = qp; + + memset(&qp->qp_stats, 0, sizeof(qp->qp_stats)); + return 0; + +qp_setup_cleanup: + if (qp->processed_pkts) + rte_ring_free(qp->processed_pkts); + if (qp) + rte_free(qp); + return -1; +} + +static int +zip_pmd_stream_create(struct rte_compressdev *dev, + const struct rte_comp_xform *xform, void **stream) +{ + int ret; + struct zip_stream *strm = NULL; + + strm = rte_malloc(NULL, + sizeof(struct zip_stream), 0); + + if (strm == NULL) + return (-ENOMEM); + + ret = zip_set_stream_parameters(dev, xform, strm); + if (ret < 0) { + ZIP_PMD_ERR("failed configure xform parameters"); + rte_free(strm); + return ret; + } + *stream = strm; + return 0; +} + +static int +zip_pmd_stream_free(struct rte_compressdev *dev, void *stream) +{ + struct zip_vf *vf = (struct zip_vf *) (dev->data->dev_private); + struct zip_stream *z_stream; + + if (stream == NULL) + return 0; + + z_stream = (struct zip_stream *)stream; + + /* Free resources back to pool */ + rte_mempool_put_bulk(vf->zip_mp, + (void *)&(z_stream->bufs[0]), + MAX_BUFS_PER_STREAM); + + /* Zero out the whole structure */ + memset(stream, 0, sizeof(struct zip_stream)); + rte_free(stream); + + return 0; +} + + +static uint16_t +zip_pmd_enqueue_burst_sync(void *queue_pair, + struct rte_comp_op **ops, uint16_t nb_ops) +{ + struct zipvf_qp *qp = queue_pair; + struct rte_comp_op *op; + struct zip_stream *zstrm; + int i, ret = 0; + uint16_t enqd = 0; + + for (i = 0; i < nb_ops; i++) { + op = ops[i]; + + if (op->op_type == RTE_COMP_OP_STATEFUL) { + op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; + } else { + /* process stateless ops */ + zstrm = (struct zip_stream *)op->private_xform; + if (unlikely(zstrm == NULL)) + op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; + else + ret = zstrm->func(op, qp, zstrm); + } + + /* Whatever is out of op, put it into completion queue with + * its status + */ + if (!ret) + ret = rte_ring_enqueue(qp->processed_pkts, (void *)op); + + if (unlikely(ret < 0)) { + /* increment count if failed to enqueue op*/ + qp->qp_stats.enqueue_err_count++; + } else { + qp->qp_stats.enqueued_count++; + enqd++; + } + } + return enqd; +} + +static uint16_t +zip_pmd_dequeue_burst_sync(void *queue_pair, + struct rte_comp_op **ops, uint16_t nb_ops) +{ + struct zipvf_qp *qp = queue_pair; + + unsigned int nb_dequeued = 0; + + nb_dequeued = rte_ring_dequeue_burst(qp->processed_pkts, + (void **)ops, nb_ops, NULL); + qp->qp_stats.dequeued_count += nb_dequeued; + + return nb_dequeued; +} + +struct rte_compressdev_ops octtx_zip_pmd_ops = { + .dev_configure = zip_pmd_config, + .dev_start = zip_pmd_start, + .dev_stop = zip_pmd_stop, + .dev_close = zip_pmd_close, + + .stats_get = zip_pmd_stats_get, + .stats_reset = zip_pmd_stats_reset, + + .dev_infos_get = zip_pmd_info_get, + + .queue_pair_setup = zip_pmd_qp_setup, + .queue_pair_release = zip_pmd_qp_release, + + .private_xform_create = zip_pmd_stream_create, + .private_xform_free = zip_pmd_stream_free, + .stream_create = NULL, + .stream_free = NULL +}; + +static int +zip_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, + struct rte_pci_device *pci_dev) +{ + int ret = 0; + char compressdev_name[RTE_COMPRESSDEV_NAME_MAX_LEN]; + struct rte_compressdev *compressdev; + struct rte_compressdev_pmd_init_params init_params = { + "", + rte_socket_id(), + }; + + ZIP_PMD_INFO("vendor_id=0x%x device_id=0x%x", + (unsigned int)pci_dev->id.vendor_id, + (unsigned int)pci_dev->id.device_id); + + rte_pci_device_name(&pci_dev->addr, compressdev_name, + sizeof(compressdev_name)); + + compressdev = rte_compressdev_pmd_create(compressdev_name, + &pci_dev->device, sizeof(struct zip_vf), &init_params); + if (compressdev == NULL) { + ZIP_PMD_ERR("driver %s: create failed", init_params.name); + return -ENODEV; + } + + /* + * create only if proc_type is primary. + */ + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + /* create vf dev with given pmd dev id */ + ret = zipvf_create(compressdev); + if (ret < 0) { + ZIP_PMD_ERR("Device creation failed"); + rte_compressdev_pmd_destroy(compressdev); + return ret; + } + } + + compressdev->dev_ops = &octtx_zip_pmd_ops; + /* register rx/tx burst functions for data path */ + compressdev->dequeue_burst = zip_pmd_dequeue_burst_sync; + compressdev->enqueue_burst = zip_pmd_enqueue_burst_sync; + compressdev->feature_flags = RTE_COMPDEV_FF_HW_ACCELERATED; + return ret; +} + +static int +zip_pci_remove(struct rte_pci_device *pci_dev) +{ + struct rte_compressdev *compressdev; + char compressdev_name[RTE_COMPRESSDEV_NAME_MAX_LEN]; + + if (pci_dev == NULL) { + ZIP_PMD_ERR(" Invalid PCI Device\n"); + return -EINVAL; + } + rte_pci_device_name(&pci_dev->addr, compressdev_name, + sizeof(compressdev_name)); + + compressdev = rte_compressdev_pmd_get_named_dev(compressdev_name); + if (compressdev == NULL) + return -ENODEV; + + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + if (zipvf_destroy(compressdev) < 0) + return -ENODEV; + } + return rte_compressdev_pmd_destroy(compressdev); +} + +static struct rte_pci_id pci_id_octtx_zipvf_table[] = { + { + RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, + PCI_DEVICE_ID_OCTEONTX_ZIPVF), + }, + { + .device_id = 0 + }, +}; + +/** + * Structure that represents a PCI driver + */ +static struct rte_pci_driver octtx_zip_pmd = { + .id_table = pci_id_octtx_zipvf_table, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING, + .probe = zip_pci_probe, + .remove = zip_pci_remove, +}; + +RTE_PMD_REGISTER_PCI(COMPRESSDEV_NAME_ZIP_PMD, octtx_zip_pmd); +RTE_PMD_REGISTER_PCI_TABLE(COMPRESSDEV_NAME_ZIP_PMD, pci_id_octtx_zipvf_table); + +RTE_INIT(octtx_zip_init_log); + +static void +octtx_zip_init_log(void) +{ + octtx_zip_logtype_driver = rte_log_register("pmd.compress.octeontx"); + if (octtx_zip_logtype_driver >= 0) + rte_log_set_level(octtx_zip_logtype_driver, RTE_LOG_INFO); +} diff --git a/src/spdk/dpdk/drivers/compress/octeontx/rte_pmd_octeontx_compress_version.map b/src/spdk/dpdk/drivers/compress/octeontx/rte_pmd_octeontx_compress_version.map new file mode 100644 index 00000000..ad6e191e --- /dev/null +++ b/src/spdk/dpdk/drivers/compress/octeontx/rte_pmd_octeontx_compress_version.map @@ -0,0 +1,3 @@ +DPDK_18.08 { + local: *; +}; -- cgit v1.2.3